Adding upstream version 14.2.21.upstream/14.2.21upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

43 files changed, 15364 insertions, 0 deletions

diff --git a/src/test/rgw/CMakeLists.txt b/src/test/rgw/CMakeLists.txt
new file mode 100644
index 00000000..addbf75f
--- /dev/null
+++ b/src/test/rgw/CMakeLists.txt
@@ -0,0 +1,181 @@
+
+if(WITH_RADOSGW_AMQP_ENDPOINT)
+  # amqp mock library
+  set(amqp_mock_src
+    amqp_mock.cc amqp_url.c)
+  add_library(amqp_mock STATIC ${amqp_mock_src})
+endif()
+
+if(WITH_RADOSGW_KAFKA_ENDPOINT)
+  # kafka stub library
+  set(kafka_stub_src
+    kafka_stub.cc)
+  add_library(kafka_stub STATIC ${kafka_stub_src})
+endif()
+
+#unittest_rgw_bencode
+add_executable(unittest_rgw_bencode test_rgw_bencode.cc)
+add_ceph_unittest(unittest_rgw_bencode)
+target_link_libraries(unittest_rgw_bencode ${rgw_libs})
+
+#unitttest_rgw_period_history
+add_executable(unittest_rgw_period_history test_rgw_period_history.cc)
+add_ceph_unittest(unittest_rgw_period_history)
+target_link_libraries(unittest_rgw_period_history ${rgw_libs})
+
+# unitttest_rgw_compression
+add_executable(unittest_rgw_compression
+  test_rgw_compression.cc
+  $<TARGET_OBJECTS:unit-main>)
+add_ceph_unittest(unittest_rgw_compression)
+target_link_libraries(unittest_rgw_compression ${rgw_libs})
+
+# unitttest_http_manager
+add_executable(unittest_http_manager test_http_manager.cc)
+add_ceph_unittest(unittest_http_manager)
+target_link_libraries(unittest_http_manager ${rgw_libs})
+
+# unitttest_rgw_reshard_wait
+add_executable(unittest_rgw_reshard_wait test_rgw_reshard_wait.cc)
+add_ceph_unittest(unittest_rgw_reshard_wait)
+target_link_libraries(unittest_rgw_reshard_wait ${rgw_libs})
+
+set(test_rgw_a_src
+  test_rgw_common.cc)
+add_library(test_rgw_a STATIC ${test_rgw_a_src})
+
+# ceph_test_rgw_manifest
+set(test_rgw_manifest_srcs test_rgw_manifest.cc)
+add_executable(ceph_test_rgw_manifest
+  ${test_rgw_manifest_srcs}
+  )
+target_link_libraries(ceph_test_rgw_manifest
+  ${rgw_libs}
+  test_rgw_a
+  cls_rgw_client
+  cls_lock_client
+  cls_refcount_client
+  cls_log_client
+  cls_timeindex_client
+  cls_version_client
+  cls_user_client
+  librados
+  global
+  ${BLKID_LIBRARIES}
+  ${CURL_LIBRARIES}
+  ${EXPAT_LIBRARIES}
+  ${CMAKE_DL_LIBS}
+  ${UNITTEST_LIBS}
+  ${CRYPTO_LIBS})
+
+set(test_rgw_obj_srcs test_rgw_obj.cc)
+add_executable(ceph_test_rgw_obj
+  ${test_rgw_obj_srcs}
+  )
+target_link_libraries(ceph_test_rgw_obj
+  ${rgw_libs}
+  test_rgw_a
+  cls_rgw_client
+  cls_lock_client
+  cls_refcount_client
+  cls_log_client
+  cls_version_client
+  cls_user_client
+  librados
+  global
+  ${CURL_LIBRARIES}
+  ${EXPAT_LIBRARIES}
+  ${CMAKE_DL_LIBS}
+  ${UNITTEST_LIBS}
+  ${CRYPTO_LIBS}
+  )
+
+set(test_rgw_crypto_srcs test_rgw_crypto.cc)
+add_executable(unittest_rgw_crypto
+  ${test_rgw_crypto_srcs}
+  )
+add_ceph_unittest(unittest_rgw_crypto)
+target_link_libraries(unittest_rgw_crypto
+  ${rgw_libs}
+  cls_rgw_client
+  cls_lock_client
+  cls_refcount_client
+  cls_log_client
+  cls_version_client
+  cls_user_client
+  librados
+  global
+  ${CURL_LIBRARIES}
+  ${EXPAT_LIBRARIES}
+  ${CMAKE_DL_LIBS}
+  ${UNITTEST_LIBS}
+  ${CRYPTO_LIBS}
+  )
+
+add_executable(unittest_rgw_putobj test_rgw_putobj.cc)
+add_ceph_unittest(unittest_rgw_putobj)
+target_link_libraries(unittest_rgw_putobj rgw_a ${UNITTEST_LIBS})
+
+add_executable(ceph_test_rgw_throttle
+  test_rgw_throttle.cc
+  $<TARGET_OBJECTS:unit-main>)
+target_link_libraries(ceph_test_rgw_throttle ${rgw_libs}
+  librados global ${UNITTEST_LIBS})
+
+add_executable(unittest_rgw_iam_policy test_rgw_iam_policy.cc)
+add_ceph_unittest(unittest_rgw_iam_policy)
+target_link_libraries(unittest_rgw_iam_policy
+  ${rgw_libs}
+  cls_rgw_client
+  cls_lock_client
+  cls_refcount_client
+  cls_log_client
+  cls_version_client
+  cls_user_client
+  librados
+  global
+  ${CURL_LIBRARIES}
+  ${EXPAT_LIBRARIES}
+  ${CMAKE_DL_LIBS}
+  ${UNITTEST_LIBS}
+  ${CRYPTO_LIBS}
+  )
+
+add_executable(unittest_rgw_string test_rgw_string.cc)
+add_ceph_unittest(unittest_rgw_string)
+
+# unitttest_rgw_dmclock_queue
+add_executable(unittest_rgw_dmclock_scheduler test_rgw_dmclock_scheduler.cc $<TARGET_OBJECTS:unit-main>)
+add_ceph_unittest(unittest_rgw_dmclock_scheduler)
+
+target_link_libraries(unittest_rgw_dmclock_scheduler radosgw_a dmclock)
+if(WITH_BOOST_CONTEXT)
+  target_compile_definitions(unittest_rgw_dmclock_scheduler PRIVATE BOOST_COROUTINES_NO_DEPRECATION_WARNING)
+  target_link_libraries(unittest_rgw_dmclock_scheduler Boost::coroutine Boost::context)
+endif()
+
+if(WITH_RADOSGW_AMQP_ENDPOINT)
+  add_executable(unittest_rgw_amqp test_rgw_amqp.cc)
+  add_ceph_unittest(unittest_rgw_amqp)
+  target_link_libraries(unittest_rgw_amqp ${rgw_libs})
+endif()
+
+# unittest_rgw_xml
+add_executable(unittest_rgw_xml test_rgw_xml.cc)
+add_ceph_unittest(unittest_rgw_xml)
+
+target_link_libraries(unittest_rgw_xml ${rgw_libs} ${EXPAT_LIBRARIES})
+
+# unittest_rgw_arn
+add_executable(unittest_rgw_arn test_rgw_arn.cc)
+add_ceph_unittest(unittest_rgw_arn)
+
+target_link_libraries(unittest_rgw_arn ${rgw_libs})
+
+# unittest_rgw_url
+add_executable(unittest_rgw_url test_rgw_url.cc)
+add_ceph_unittest(unittest_rgw_url)
+
+target_link_libraries(unittest_rgw_url ${rgw_libs})
+add_ceph_test(test-ceph-diff-sorted.sh
+  ${CMAKE_CURRENT_SOURCE_DIR}/test-ceph-diff-sorted.sh)
diff --git a/src/test/rgw/amqp_mock.cc b/src/test/rgw/amqp_mock.cc
new file mode 100644
index 00000000..e37151e4
--- /dev/null
+++ b/src/test/rgw/amqp_mock.cc
@@ -0,0 +1,347 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "amqp_mock.h"
+#include <amqp.h>
+#include <amqp_tcp_socket.h>
+#include <string>
+#include <stdarg.h>
+#include <mutex>
+#include <boost/lockfree/queue.hpp>
+
+namespace amqp_mock {
+
+std::mutex set_valid_lock;
+int VALID_PORT(5672);
+std::string VALID_HOST("localhost");
+std::string VALID_VHOST("/");
+std::string VALID_USER("guest");
+std::string VALID_PASSWORD("guest");
+
+void set_valid_port(int port) {
+  std::lock_guard<std::mutex> lock(set_valid_lock);
+  VALID_PORT = port;
+}
+
+void set_valid_host(const std::string& host) {
+  std::lock_guard<std::mutex> lock(set_valid_lock);
+  VALID_HOST = host;
+}
+
+void set_valid_vhost(const std::string& vhost) {
+  std::lock_guard<std::mutex> lock(set_valid_lock);
+  VALID_VHOST = vhost;
+}
+
+void set_valid_user(const std::string& user, const std::string& password) {
+  std::lock_guard<std::mutex> lock(set_valid_lock);
+  VALID_USER = user;
+  VALID_PASSWORD = password;
+}
+
+std::atomic<unsigned> g_tag_skip = 0;
+std::atomic<int> g_multiple = 0;
+
+void set_multiple(unsigned tag_skip) {
+    g_multiple = 1;
+    g_tag_skip = tag_skip;
+}
+
+void reset_multiple() {
+    g_multiple = 0;
+    g_tag_skip = 0;
+}
+
+bool FAIL_NEXT_WRITE(false);
+bool FAIL_NEXT_READ(false);
+bool REPLY_ACK(true);
+}
+
+using namespace amqp_mock;
+
+struct amqp_connection_state_t_ {
+  amqp_socket_t* socket;
+  amqp_channel_open_ok_t* channel1;
+  amqp_channel_open_ok_t* channel2;
+  amqp_exchange_declare_ok_t* exchange;
+  amqp_queue_declare_ok_t* queue;
+  amqp_confirm_select_ok_t* confirm;
+  amqp_basic_consume_ok_t* consume;
+  bool login_called;
+  boost::lockfree::queue<amqp_basic_ack_t> ack_list;
+  boost::lockfree::queue<amqp_basic_nack_t> nack_list;
+  std::atomic<uint64_t> delivery_tag;
+  amqp_rpc_reply_t reply;
+  amqp_basic_ack_t ack;
+  amqp_basic_nack_t nack;
+  // ctor
+  amqp_connection_state_t_() : 
+    socket(nullptr), 
+    channel1(nullptr),
+    channel2(nullptr),
+    exchange(nullptr),
+    queue(nullptr),
+    confirm(nullptr),
+    consume(nullptr),
+    login_called(false),
+    ack_list(1024),
+    nack_list(1024),
+    delivery_tag(1) {
+      reply.reply_type = AMQP_RESPONSE_NONE;
+    }
+};
+
+struct amqp_socket_t_ {
+  bool open_called;
+  // ctor
+  amqp_socket_t_() : open_called(false) {
+  }
+};
+
+amqp_connection_state_t AMQP_CALL amqp_new_connection(void) {
+  auto s = new amqp_connection_state_t_;
+  return s;
+}
+
+int amqp_destroy_connection(amqp_connection_state_t state) {
+  delete state->socket;
+  delete state->channel1;
+  delete state->channel2;
+  delete state->exchange;
+  delete state->queue;
+  delete state->confirm;
+  delete state->consume;
+  delete state;
+  return 0;
+}
+
+amqp_socket_t* amqp_tcp_socket_new(amqp_connection_state_t state) {
+  state->socket = new amqp_socket_t;
+  return state->socket;
+}
+
+int amqp_socket_open(amqp_socket_t *self, const char *host, int port) {
+  if (!self) {
+    return -1;
+  }
+  {
+    std::lock_guard<std::mutex> lock(set_valid_lock);
+    if (std::string(host) != VALID_HOST) {
+      return -2;
+    } 
+    if (port != VALID_PORT) {
+      return -3;
+    }
+  }
+  self->open_called = true;
+  return 0;
+}
+
+amqp_rpc_reply_t amqp_login(
+    amqp_connection_state_t state, 
+    char const *vhost, 
+    int channel_max,
+    int frame_max, 
+    int heartbeat, 
+    amqp_sasl_method_enum sasl_method, ...) {
+  state->reply.reply_type = AMQP_RESPONSE_SERVER_EXCEPTION;
+  state->reply.library_error = 0;
+  state->reply.reply.decoded = nullptr;
+  state->reply.reply.id = 0;
+  if (std::string(vhost) != VALID_VHOST) {
+    return state->reply;
+  }
+  if (sasl_method != AMQP_SASL_METHOD_PLAIN) {
+      return state->reply;
+  }
+  va_list args;
+  va_start(args, sasl_method);
+  char* user = va_arg(args, char*);
+  char* password = va_arg(args, char*);
+  va_end(args);
+  if (std::string(user) != VALID_USER) {
+    return state->reply;
+  }
+  if (std::string(password) != VALID_PASSWORD) {
+    return state->reply;
+  }
+  state->reply.reply_type = AMQP_RESPONSE_NORMAL;
+  state->login_called = true;
+  return state->reply;
+}
+
+amqp_channel_open_ok_t* amqp_channel_open(amqp_connection_state_t state, amqp_channel_t channel) {
+  state->reply.reply_type = AMQP_RESPONSE_NORMAL;
+  if (state->channel1 == nullptr) {
+    state->channel1 = new amqp_channel_open_ok_t;
+    return state->channel1;
+  }
+
+  state->channel2 = new amqp_channel_open_ok_t;
+  return state->channel2;
+}
+
+amqp_exchange_declare_ok_t* amqp_exchange_declare(
+    amqp_connection_state_t state, 
+    amqp_channel_t channel,
+    amqp_bytes_t exchange, 
+    amqp_bytes_t type, 
+    amqp_boolean_t passive,
+    amqp_boolean_t durable, 
+    amqp_boolean_t auto_delete, 
+    amqp_boolean_t internal,
+    amqp_table_t arguments) {
+  state->exchange = new amqp_exchange_declare_ok_t;
+  state->reply.reply_type = AMQP_RESPONSE_NORMAL;
+  return state->exchange;
+}
+
+amqp_rpc_reply_t amqp_get_rpc_reply(amqp_connection_state_t state) {
+  return state->reply;
+}
+
+int amqp_basic_publish(
+    amqp_connection_state_t state, 
+    amqp_channel_t channel,
+    amqp_bytes_t exchange, 
+    amqp_bytes_t routing_key, 
+    amqp_boolean_t mandatory,
+    amqp_boolean_t immediate, 
+    struct amqp_basic_properties_t_ const *properties,
+    amqp_bytes_t body) {
+  // make sure that all calls happened before publish
+  if (state->socket && state->socket->open_called &&
+      state->login_called && state->channel1 && state->channel2 && state->exchange &&
+      !FAIL_NEXT_WRITE) {
+    state->reply.reply_type = AMQP_RESPONSE_NORMAL;
+    if (properties) {
+      if (REPLY_ACK) {
+        state->ack_list.push(amqp_basic_ack_t{state->delivery_tag++, 0});
+      } else {
+        state->nack_list.push(amqp_basic_nack_t{state->delivery_tag++, 0});
+      }
+    }
+    return AMQP_STATUS_OK;
+  }
+  return AMQP_STATUS_CONNECTION_CLOSED;
+}
+
+const amqp_table_t amqp_empty_table = {0, NULL};
+const amqp_bytes_t amqp_empty_bytes = {0, NULL};
+
+const char* amqp_error_string2(int code) {
+  static const char* str = "mock error";
+  return str;
+}
+
+char const* amqp_method_name(amqp_method_number_t methodNumber) {
+  static const char* str = "mock method";
+  return str;
+}
+
+amqp_queue_declare_ok_t* amqp_queue_declare(
+    amqp_connection_state_t state, amqp_channel_t channel, amqp_bytes_t queue,
+    amqp_boolean_t passive, amqp_boolean_t durable, amqp_boolean_t exclusive,
+    amqp_boolean_t auto_delete, amqp_table_t arguments) {
+  state->queue = new amqp_queue_declare_ok_t;
+  static const char* str = "tmp-queue";
+  state->queue->queue = amqp_cstring_bytes(str);
+  state->reply.reply_type = AMQP_RESPONSE_NORMAL;
+  return state->queue;
+}
+
+amqp_confirm_select_ok_t* amqp_confirm_select(amqp_connection_state_t state, amqp_channel_t channel) {
+  state->confirm = new amqp_confirm_select_ok_t;
+  state->reply.reply_type = AMQP_RESPONSE_NORMAL;
+  return state->confirm;
+}
+
+int amqp_simple_wait_frame_noblock(amqp_connection_state_t state, amqp_frame_t *decoded_frame, struct timeval* tv) {
+  if (state->socket && state->socket->open_called &&
+      state->login_called && state->channel1 && state->channel2 && state->exchange &&
+      state->queue && state->consume && state->confirm && !FAIL_NEXT_READ) {
+    // "wait" for queue
+    usleep(tv->tv_sec*1000000+tv->tv_usec);
+    // read from queue
+    if (g_multiple) {
+      // pop multiples and reply once at the end
+      for (auto i = 0U; i < g_tag_skip; ++i) {
+        if (REPLY_ACK && !state->ack_list.pop(state->ack)) {
+          // queue is empty
+          return AMQP_STATUS_TIMEOUT;
+        } else if (!REPLY_ACK && !state->nack_list.pop(state->nack)) {
+          // queue is empty
+          return AMQP_STATUS_TIMEOUT;
+        }
+      }
+      if (REPLY_ACK) {
+        state->ack.multiple = g_multiple;
+        decoded_frame->payload.method.id = AMQP_BASIC_ACK_METHOD;
+        decoded_frame->payload.method.decoded = &state->ack;
+      } else {
+        state->nack.multiple = g_multiple;
+        decoded_frame->payload.method.id = AMQP_BASIC_NACK_METHOD;
+        decoded_frame->payload.method.decoded = &state->nack;
+      }
+      decoded_frame->frame_type = AMQP_FRAME_METHOD;
+      state->reply.reply_type = AMQP_RESPONSE_NORMAL;
+      reset_multiple();
+      return AMQP_STATUS_OK;
+    }
+    // pop replies one by one
+    if (REPLY_ACK && state->ack_list.pop(state->ack)) {
+      state->ack.multiple = g_multiple;
+      decoded_frame->frame_type = AMQP_FRAME_METHOD;
+      decoded_frame->payload.method.id = AMQP_BASIC_ACK_METHOD;
+      decoded_frame->payload.method.decoded = &state->ack;
+      state->reply.reply_type = AMQP_RESPONSE_NORMAL;
+      return AMQP_STATUS_OK;
+    } else if (!REPLY_ACK && state->nack_list.pop(state->nack)) {
+      state->nack.multiple = g_multiple;
+      decoded_frame->frame_type = AMQP_FRAME_METHOD;
+      decoded_frame->payload.method.id = AMQP_BASIC_NACK_METHOD;
+      decoded_frame->payload.method.decoded = &state->nack;
+      state->reply.reply_type = AMQP_RESPONSE_NORMAL;
+      return AMQP_STATUS_OK;
+    } else {
+      // queue is empty
+      return AMQP_STATUS_TIMEOUT;
+    }
+  }
+  return AMQP_STATUS_CONNECTION_CLOSED;
+}
+
+amqp_basic_consume_ok_t* amqp_basic_consume(
+    amqp_connection_state_t state, amqp_channel_t channel, amqp_bytes_t queue,
+    amqp_bytes_t consumer_tag, amqp_boolean_t no_local, amqp_boolean_t no_ack,
+    amqp_boolean_t exclusive, amqp_table_t arguments) {
+  state->consume = new amqp_basic_consume_ok_t;
+  state->reply.reply_type = AMQP_RESPONSE_NORMAL;
+  return state->consume;
+}
+
+// amqp_parse_url() is linked via the actual rabbitmq-c library code. see: amqp_url.c
+
+// following functions are the actual implementation copied from rabbitmq-c library
+
+#include <string.h>
+
+amqp_bytes_t amqp_cstring_bytes(const char* cstr) {
+  amqp_bytes_t result;
+  result.len = strlen(cstr);
+  result.bytes = (void *)cstr;
+  return result;
+}
+
+void amqp_bytes_free(amqp_bytes_t bytes) { free(bytes.bytes); }
+
+amqp_bytes_t amqp_bytes_malloc_dup(amqp_bytes_t src) {
+  amqp_bytes_t result;
+  result.len = src.len;
+  result.bytes = malloc(src.len);
+  if (result.bytes != NULL) {
+    memcpy(result.bytes, src.bytes, src.len);
+  }
+  return result;
+}
+
diff --git a/src/test/rgw/amqp_mock.h b/src/test/rgw/amqp_mock.h
new file mode 100644
index 00000000..94fdfddd
--- /dev/null
+++ b/src/test/rgw/amqp_mock.h
@@ -0,0 +1,19 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+#pragma once
+
+#include <string>
+
+namespace amqp_mock {
+void set_valid_port(int port);
+void set_valid_host(const std::string& host);
+void set_valid_vhost(const std::string& vhost);
+void set_valid_user(const std::string& user, const std::string& password);
+void set_multiple(unsigned tag);
+void reset_multiple();
+  
+extern bool FAIL_NEXT_WRITE;        // default "false"
+extern bool FAIL_NEXT_READ;         // default "false"
+extern bool REPLY_ACK;              // default "true"
+}
+
diff --git a/src/test/rgw/amqp_url.c b/src/test/rgw/amqp_url.c
new file mode 100644
index 00000000..071f4390
--- /dev/null
+++ b/src/test/rgw/amqp_url.c
@@ -0,0 +1,219 @@
+/*
+ * ***** BEGIN LICENSE BLOCK *****
+ * Version: MIT
+ *
+ * Portions created by Alan Antonuk are Copyright (c) 2012-2013
+ * Alan Antonuk. All Rights Reserved.
+ *
+ * Portions created by VMware are Copyright (c) 2007-2012 VMware, Inc.
+ * All Rights Reserved.
+ *
+ * Portions created by Tony Garnock-Jones are Copyright (c) 2009-2010
+ * VMware, Inc. and Tony Garnock-Jones. All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use, copy,
+ * modify, merge, publish, distribute, sublicense, and/or sell copies
+ * of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ * ***** END LICENSE BLOCK *****
+ */
+
+// this version of the file is slightly modified from the original one
+// as it is only used to mock amqp libraries
+
+#ifdef _MSC_VER
+#define _CRT_SECURE_NO_WARNINGS
+#endif
+
+#include "amqp.h"
+#include <limits.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+void amqp_default_connection_info(struct amqp_connection_info *ci) {
+  /* Apply defaults */
+  ci->user = "guest";
+  ci->password = "guest";
+  ci->host = "localhost";
+  ci->port = 5672;
+  ci->vhost = "/";
+  ci->ssl = 0;
+}
+
+/* Scan for the next delimiter, handling percent-encodings on the way. */
+static char find_delim(char **pp, int colon_and_at_sign_are_delims) {
+  char *from = *pp;
+  char *to = from;
+
+  for (;;) {
+    char ch = *from++;
+
+    switch (ch) {
+      case ':':
+      case '@':
+        if (!colon_and_at_sign_are_delims) {
+          *to++ = ch;
+          break;
+        }
+
+      /* fall through */
+      case 0:
+      case '/':
+      case '?':
+      case '#':
+      case '[':
+      case ']':
+        *to = 0;
+        *pp = from;
+        return ch;
+
+      case '%': {
+        unsigned int val;
+        int chars;
+        int res = sscanf(from, "%2x%n", &val, &chars);
+
+        if (res == EOF || res < 1 || chars != 2 || val > CHAR_MAX)
+        /* Return a surprising delimiter to
+           force an error. */
+        {
+          return '%';
+        }
+
+        *to++ = (char)val;
+        from += 2;
+        break;
+      }
+
+      default:
+        *to++ = ch;
+        break;
+    }
+  }
+}
+
+/* Parse an AMQP URL into its component parts. */
+int amqp_parse_url(char *url, struct amqp_connection_info *parsed) {
+  int res = AMQP_STATUS_BAD_URL;
+  char delim;
+  char *start;
+  char *host;
+  char *port = NULL;
+
+  amqp_default_connection_info(parsed);
+
+  /* check the prefix */
+  if (!strncmp(url, "amqp://", 7)) {
+    /* do nothing */
+  } else if (!strncmp(url, "amqps://", 8)) {
+    parsed->port = 5671;
+    parsed->ssl = 1;
+  } else {
+    goto out;
+  }
+
+  host = start = url += (parsed->ssl ? 8 : 7);
+  delim = find_delim(&url, 1);
+
+  if (delim == ':') {
+    /* The colon could be introducing the port or the
+       password part of the userinfo.  We don't know yet,
+       so stash the preceding component. */
+    port = start = url;
+    delim = find_delim(&url, 1);
+  }
+
+  if (delim == '@') {
+    /* What might have been the host and port were in fact
+       the username and password */
+    parsed->user = host;
+    if (port) {
+      parsed->password = port;
+    }
+
+    port = NULL;
+    host = start = url;
+    delim = find_delim(&url, 1);
+  }
+
+  if (delim == '[') {
+    /* IPv6 address.  The bracket should be the first
+       character in the host. */
+    if (host != start || *host != 0) {
+      goto out;
+    }
+
+    start = url;
+    delim = find_delim(&url, 0);
+
+    if (delim != ']') {
+      goto out;
+    }
+
+    parsed->host = start;
+    start = url;
+    delim = find_delim(&url, 1);
+
+    /* Closing bracket should be the last character in the
+       host. */
+    if (*start != 0) {
+      goto out;
+    }
+  } else {
+    /* If we haven't seen the host yet, this is it. */
+    if (*host != 0) {
+      parsed->host = host;
+    }
+  }
+
+  if (delim == ':') {
+    port = start = url;
+    delim = find_delim(&url, 1);
+  }
+
+  if (port) {
+    char *end;
+    long portnum = strtol(port, &end, 10);
+
+    if (port == end || *end != 0 || portnum < 0 || portnum > 65535) {
+      goto out;
+    }
+
+    parsed->port = portnum;
+  }
+
+  if (delim == '/') {
+    start = url;
+    delim = find_delim(&url, 1);
+
+    if (delim != 0) {
+      goto out;
+    }
+
+    parsed->vhost = start;
+    res = AMQP_STATUS_OK;
+  } else if (delim == 0) {
+    res = AMQP_STATUS_OK;
+  }
+
+/* Any other delimiter is bad, and we will return AMQP_STATUS_BAD_AMQP_URL. */
+
+out:
+  return res;
+}
diff --git a/src/test/rgw/kafka_stub.cc b/src/test/rgw/kafka_stub.cc
new file mode 100644
index 00000000..6125a94c
--- /dev/null
+++ b/src/test/rgw/kafka_stub.cc
@@ -0,0 +1,68 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include <librdkafka/rdkafka.h>
+
+const char *rd_kafka_topic_name(const rd_kafka_topic_t *rkt) {
+    return "";
+}
+
+rd_kafka_resp_err_t rd_kafka_last_error() {
+    return rd_kafka_resp_err_t();
+}
+
+const char *rd_kafka_err2str(rd_kafka_resp_err_t err) {
+    return "";
+}
+
+rd_kafka_conf_t *rd_kafka_conf_new() {
+    return nullptr;
+}
+
+rd_kafka_conf_res_t rd_kafka_conf_set(rd_kafka_conf_t *conf,
+				       const char *name,
+				       const char *value,
+				       char *errstr, size_t errstr_size) {
+    return rd_kafka_conf_res_t();
+}
+
+void rd_kafka_conf_set_dr_msg_cb(rd_kafka_conf_t *conf,
+                                  void (*dr_msg_cb) (rd_kafka_t *rk,
+                                                     const rd_kafka_message_t *
+                                                     rkmessage,
+                                                     void *opaque)) {}
+
+void rd_kafka_conf_set_opaque(rd_kafka_conf_t *conf, void *opaque) {}
+
+rd_kafka_t *rd_kafka_new(rd_kafka_type_t type, rd_kafka_conf_t *conf,
+                      char *errstr, size_t errstr_size) {
+    return nullptr;
+}
+
+void rd_kafka_conf_destroy(rd_kafka_conf_t *conf) {}
+
+rd_kafka_resp_err_t rd_kafka_flush (rd_kafka_t *rk, int timeout_ms) {
+    return rd_kafka_resp_err_t();
+}
+
+void rd_kafka_destroy(rd_kafka_t *rk) {}
+
+rd_kafka_topic_t *rd_kafka_topic_new(rd_kafka_t *rk, const char *topic,
+                              rd_kafka_topic_conf_t *conf) {
+    return nullptr;
+}
+
+int rd_kafka_produce(rd_kafka_topic_t *rkt, int32_t partition,
+		      int msgflags,
+		      void *payload, size_t len,
+		      const void *key, size_t keylen,
+		      void *msg_opaque) {
+    return 0;
+}
+
+int rd_kafka_poll(rd_kafka_t *rk, int timeout_ms) {
+    return 0;
+}
+
+void rd_kafka_topic_destroy(rd_kafka_topic_t *rkt) {}
+
diff --git a/src/test/rgw/rgw_multi/__init__.py b/src/test/rgw/rgw_multi/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/src/test/rgw/rgw_multi/__init__.py
diff --git a/src/test/rgw/rgw_multi/conn.py b/src/test/rgw/rgw_multi/conn.py
new file mode 100644
index 00000000..b03db367
--- /dev/null
+++ b/src/test/rgw/rgw_multi/conn.py
@@ -0,0 +1,30 @@
+import boto
+import boto.s3.connection
+
+
+def get_gateway_connection(gateway, credentials):
+    """ connect to the given gateway """
+    if gateway.connection is None:
+        gateway.connection = boto.connect_s3(
+                aws_access_key_id = credentials.access_key,
+                aws_secret_access_key = credentials.secret,
+                host = gateway.host,
+                port = gateway.port,
+                is_secure = False,
+                calling_format = boto.s3.connection.OrdinaryCallingFormat())
+    return gateway.connection
+
+def get_gateway_secure_connection(gateway, credentials):
+    """ secure connect to the given gateway """
+    if gateway.ssl_port == 0:
+        return None
+    if gateway.secure_connection is None:
+        gateway.secure_connection = boto.connect_s3(
+            aws_access_key_id = credentials.access_key,
+            aws_secret_access_key = credentials.secret,
+            host = gateway.host,
+            port = gateway.ssl_port,
+            is_secure = True,
+            validate_certs=False,
+            calling_format = boto.s3.connection.OrdinaryCallingFormat())
+    return gateway.secure_connection
diff --git a/src/test/rgw/rgw_multi/multisite.py b/src/test/rgw/rgw_multi/multisite.py
new file mode 100644
index 00000000..94f01129
--- /dev/null
+++ b/src/test/rgw/rgw_multi/multisite.py
@@ -0,0 +1,388 @@
+from abc import ABCMeta, abstractmethod
+from six import StringIO
+
+import json
+
+from .conn import get_gateway_connection, get_gateway_secure_connection
+
+class Cluster:
+    """ interface to run commands against a distinct ceph cluster """
+    __metaclass__ = ABCMeta
+
+    @abstractmethod
+    def admin(self, args = None, **kwargs):
+        """ execute a radosgw-admin command """
+        pass
+
+class Gateway:
+    """ interface to control a single radosgw instance """
+    __metaclass__ = ABCMeta
+
+    def __init__(self, host = None, port = None, cluster = None, zone = None, ssl_port = 0):
+        self.host = host
+        self.port = port
+        self.cluster = cluster
+        self.zone = zone
+        self.connection = None
+        self.secure_connection = None
+        self.ssl_port = ssl_port
+
+    @abstractmethod
+    def start(self, args = []):
+        """ start the gateway with the given args """
+        pass
+
+    @abstractmethod
+    def stop(self):
+        """ stop the gateway """
+        pass
+
+    def endpoint(self):
+        return 'http://%s:%d' % (self.host, self.port)
+
+class SystemObject:
+    """ interface for system objects, represented in json format and
+    manipulated with radosgw-admin commands """
+    __metaclass__ = ABCMeta
+
+    def __init__(self, data = None, uuid = None):
+        self.data = data
+        self.id = uuid
+        if data:
+            self.load_from_json(data)
+
+    @abstractmethod
+    def build_command(self, command):
+        """ return the command line for the given command, including arguments
+        to specify this object """
+        pass
+
+    @abstractmethod
+    def load_from_json(self, data):
+        """ update internal state based on json data """
+        pass
+
+    def command(self, cluster, cmd, args = None, **kwargs):
+        """ run the given command and return the output and retcode """
+        args = self.build_command(cmd) + (args or [])
+        return cluster.admin(args, **kwargs)
+
+    def json_command(self, cluster, cmd, args = None, **kwargs):
+        """ run the given command, parse the output and return the resulting
+        data and retcode """
+        s, r = self.command(cluster, cmd, args or [], **kwargs)
+        if r == 0:
+            output = s[s.find('{'):] # trim extra output before json
+            data = json.loads(output)
+            self.load_from_json(data)
+            self.data = data
+        return self.data, r
+
+    # mixins for supported commands
+    class Create(object):
+        def create(self, cluster, args = None, **kwargs):
+            """ create the object with the given arguments """
+            return self.json_command(cluster, 'create', args, **kwargs)
+
+    class Delete(object):
+        def delete(self, cluster, args = None, **kwargs):
+            """ delete the object """
+            # not json_command() because delete has no output
+            _, r = self.command(cluster, 'delete', args, **kwargs)
+            if r == 0:
+                self.data = None
+            return r
+
+    class Get(object):
+        def get(self, cluster, args = None, **kwargs):
+            """ read the object from storage """
+            kwargs['read_only'] = True
+            return self.json_command(cluster, 'get', args, **kwargs)
+
+    class Set(object):
+        def set(self, cluster, data, args = None, **kwargs):
+            """ set the object by json """
+            kwargs['stdin'] = StringIO(json.dumps(data))
+            return self.json_command(cluster, 'set', args, **kwargs)
+
+    class Modify(object):
+        def modify(self, cluster, args = None, **kwargs):
+            """ modify the object with the given arguments """
+            return self.json_command(cluster, 'modify', args, **kwargs)
+
+    class CreateDelete(Create, Delete): pass
+    class GetSet(Get, Set): pass
+
+class Zone(SystemObject, SystemObject.CreateDelete, SystemObject.GetSet, SystemObject.Modify):
+    def __init__(self, name, zonegroup = None, cluster = None, data = None, zone_id = None, gateways = None):
+        self.name = name
+        self.zonegroup = zonegroup
+        self.cluster = cluster
+        self.gateways = gateways or []
+        super(Zone, self).__init__(data, zone_id)
+
+    def zone_arg(self):
+        """ command-line argument to specify this zone """
+        return ['--rgw-zone', self.name]
+
+    def zone_args(self):
+        """ command-line arguments to specify this zone/zonegroup/realm """
+        args = self.zone_arg()
+        if self.zonegroup:
+            args += self.zonegroup.zonegroup_args()
+        return args
+
+    def build_command(self, command):
+        """ build a command line for the given command and args """
+        return ['zone', command] + self.zone_args()
+
+    def load_from_json(self, data):
+        """ load the zone from json """
+        self.id = data['id']
+        self.name = data['name']
+
+    def start(self, args = None):
+        """ start all gateways """
+        for g in self.gateways:
+            g.start(args)
+
+    def stop(self):
+        """ stop all gateways """
+        for g in self.gateways:
+            g.stop()
+
+    def period(self):
+        return self.zonegroup.period if self.zonegroup else None
+
+    def realm(self):
+        return self.zonegroup.realm() if self.zonegroup else None
+
+    def is_read_only(self):
+        return False
+
+    def tier_type(self):
+        raise NotImplementedError
+
+    def has_buckets(self):
+        return True
+
+    def get_conn(self, credentials):
+        return ZoneConn(self, credentials) # not implemented, but can be used
+
+class ZoneConn(object):
+    def __init__(self, zone, credentials):
+        self.zone = zone
+        self.name = zone.name
+        """ connect to the zone's first gateway """
+        if isinstance(credentials, list):
+            self.credentials = credentials[0]
+        else:
+            self.credentials = credentials
+
+        if self.zone.gateways is not None:
+            self.conn = get_gateway_connection(self.zone.gateways[0], self.credentials)
+            self.secure_conn = get_gateway_secure_connection(self.zone.gateways[0], self.credentials)
+
+    def get_connection(self):
+        return self.conn
+
+    def get_bucket(self, bucket_name, credentials):
+        raise NotImplementedError
+
+    def check_bucket_eq(self, zone, bucket_name):
+        raise NotImplementedError
+
+class ZoneGroup(SystemObject, SystemObject.CreateDelete, SystemObject.GetSet, SystemObject.Modify):
+    def __init__(self, name, period = None, data = None, zonegroup_id = None, zones = None, master_zone  = None):
+        self.name = name
+        self.period = period
+        self.zones = zones or []
+        self.master_zone = master_zone
+        super(ZoneGroup, self).__init__(data, zonegroup_id)
+        self.rw_zones = []
+        self.ro_zones = []
+        self.zones_by_type = {}
+        for z in self.zones:
+            if z.is_read_only():
+                self.ro_zones.append(z)
+            else:
+                self.rw_zones.append(z)
+
+    def zonegroup_arg(self):
+        """ command-line argument to specify this zonegroup """
+        return ['--rgw-zonegroup', self.name]
+
+    def zonegroup_args(self):
+        """ command-line arguments to specify this zonegroup/realm """
+        args = self.zonegroup_arg()
+        realm = self.realm()
+        if realm:
+            args += realm.realm_arg()
+        return args
+
+    def build_command(self, command):
+        """ build a command line for the given command and args """
+        return ['zonegroup', command] + self.zonegroup_args()
+
+    def zone_by_id(self, zone_id):
+        """ return the matching zone by id """
+        for zone in self.zones:
+            if zone.id == zone_id:
+                return zone
+        return None
+
+    def load_from_json(self, data):
+        """ load the zonegroup from json """
+        self.id = data['id']
+        self.name = data['name']
+        master_id = data['master_zone']
+        if not self.master_zone or master_id != self.master_zone.id:
+            self.master_zone = self.zone_by_id(master_id)
+
+    def add(self, cluster, zone, args = None, **kwargs):
+        """ add an existing zone to the zonegroup """
+        args = zone.zone_arg() + (args or [])
+        data, r = self.json_command(cluster, 'add', args, **kwargs)
+        if r == 0:
+            zone.zonegroup = self
+            self.zones.append(zone)
+        return data, r
+
+    def remove(self, cluster, zone, args = None, **kwargs):
+        """ remove an existing zone from the zonegroup """
+        args = zone.zone_arg() + (args or [])
+        data, r = self.json_command(cluster, 'remove', args, **kwargs)
+        if r == 0:
+            zone.zonegroup = None
+            self.zones.remove(zone)
+        return data, r
+
+    def realm(self):
+        return self.period.realm if self.period else None
+
+class Period(SystemObject, SystemObject.Get):
+    def __init__(self, realm = None, data = None, period_id = None, zonegroups = None, master_zonegroup = None):
+        self.realm = realm
+        self.zonegroups = zonegroups or []
+        self.master_zonegroup = master_zonegroup
+        super(Period, self).__init__(data, period_id)
+
+    def zonegroup_by_id(self, zonegroup_id):
+        """ return the matching zonegroup by id """
+        for zonegroup in self.zonegroups:
+            if zonegroup.id == zonegroup_id:
+                return zonegroup
+        return None
+
+    def build_command(self, command):
+        """ build a command line for the given command and args """
+        return ['period', command]
+
+    def load_from_json(self, data):
+        """ load the period from json """
+        self.id = data['id']
+        master_id = data['master_zonegroup']
+        if not self.master_zonegroup or master_id != self.master_zonegroup.id:
+            self.master_zonegroup = self.zonegroup_by_id(master_id)
+
+    def update(self, zone, args = None, **kwargs):
+        """ run 'radosgw-admin period update' on the given zone """
+        assert(zone.cluster)
+        args = zone.zone_args() + (args or [])
+        if kwargs.pop('commit', False):
+            args.append('--commit')
+        return self.json_command(zone.cluster, 'update', args, **kwargs)
+
+    def commit(self, zone, args = None, **kwargs):
+        """ run 'radosgw-admin period commit' on the given zone """
+        assert(zone.cluster)
+        args = zone.zone_args() + (args or [])
+        return self.json_command(zone.cluster, 'commit', args, **kwargs)
+
+class Realm(SystemObject, SystemObject.CreateDelete, SystemObject.GetSet):
+    def __init__(self, name, period = None, data = None, realm_id = None):
+        self.name = name
+        self.current_period = period
+        super(Realm, self).__init__(data, realm_id)
+
+    def realm_arg(self):
+        """ return the command-line arguments that specify this realm """
+        return ['--rgw-realm', self.name]
+
+    def build_command(self, command):
+        """ build a command line for the given command and args """
+        return ['realm', command] + self.realm_arg()
+
+    def load_from_json(self, data):
+        """ load the realm from json """
+        self.id = data['id']
+
+    def pull(self, cluster, gateway, credentials, args = [], **kwargs):
+        """ pull an existing realm from the given gateway """
+        args += ['--url', gateway.endpoint()]
+        args += credentials.credential_args()
+        return self.json_command(cluster, 'pull', args, **kwargs)
+
+    def master_zonegroup(self):
+        """ return the current period's master zonegroup """
+        if self.current_period is None:
+            return None
+        return self.current_period.master_zonegroup
+
+    def meta_master_zone(self):
+        """ return the current period's metadata master zone """
+        zonegroup = self.master_zonegroup()
+        if zonegroup is None:
+            return None
+        return zonegroup.master_zone
+
+class Credentials:
+    def __init__(self, access_key, secret):
+        self.access_key = access_key
+        self.secret = secret
+
+    def credential_args(self):
+        return ['--access-key', self.access_key, '--secret', self.secret]
+
+class User(SystemObject):
+    def __init__(self, uid, data = None, name = None, credentials = None, tenant = None):
+        self.name = name
+        self.credentials = credentials or []
+        self.tenant = tenant
+        super(User, self).__init__(data, uid)
+
+    def user_arg(self):
+        """ command-line argument to specify this user """
+        args = ['--uid', self.id]
+        if self.tenant:
+            args += ['--tenant', self.tenant]
+        return args
+
+    def build_command(self, command):
+        """ build a command line for the given command and args """
+        return ['user', command] + self.user_arg()
+
+    def load_from_json(self, data):
+        """ load the user from json """
+        self.id = data['user_id']
+        self.name = data['display_name']
+        self.credentials = [Credentials(k['access_key'], k['secret_key']) for k in data['keys']]
+
+    def create(self, zone, args = None, **kwargs):
+        """ create the user with the given arguments """
+        assert(zone.cluster)
+        args = zone.zone_args() + (args or [])
+        return self.json_command(zone.cluster, 'create', args, **kwargs)
+
+    def info(self, zone, args = None, **kwargs):
+        """ read the user from storage """
+        assert(zone.cluster)
+        args = zone.zone_args() + (args or [])
+        kwargs['read_only'] = True
+        return self.json_command(zone.cluster, 'info', args, **kwargs)
+
+    def delete(self, zone, args = None, **kwargs):
+        """ delete the user """
+        assert(zone.cluster)
+        args = zone.zone_args() + (args or [])
+        return self.command(zone.cluster, 'delete', args, **kwargs)
diff --git a/src/test/rgw/rgw_multi/tests.py b/src/test/rgw/rgw_multi/tests.py
new file mode 100644
index 00000000..201b2815
--- /dev/null
+++ b/src/test/rgw/rgw_multi/tests.py
@@ -0,0 +1,1322 @@
+import json
+import random
+import string
+import sys
+import time
+import logging
+import errno
+
+try:
+    from itertools import izip_longest as zip_longest
+except ImportError:
+    from itertools import zip_longest
+from itertools import combinations
+from six import StringIO
+from six.moves import range
+
+import boto
+import boto.s3.connection
+from boto.s3.website import WebsiteConfiguration
+from boto.s3.cors import CORSConfiguration
+
+from nose.tools import eq_ as eq
+from nose.plugins.attrib import attr
+from nose.plugins.skip import SkipTest
+
+from .multisite import Zone, ZoneGroup, Credentials
+
+from .conn import get_gateway_connection
+from .tools import assert_raises
+
+class Config:
+    """ test configuration """
+    def __init__(self, **kwargs):
+        # by default, wait up to 5 minutes before giving up on a sync checkpoint
+        self.checkpoint_retries = kwargs.get('checkpoint_retries', 60)
+        self.checkpoint_delay = kwargs.get('checkpoint_delay', 5)
+        # allow some time for realm reconfiguration after changing master zone
+        self.reconfigure_delay = kwargs.get('reconfigure_delay', 5)
+        self.tenant = kwargs.get('tenant', '')
+
+# rgw multisite tests, written against the interfaces provided in rgw_multi.
+# these tests must be initialized and run by another module that provides
+# implementations of these interfaces by calling init_multi()
+realm = None
+user = None
+config = None
+def init_multi(_realm, _user, _config=None):
+    global realm
+    realm = _realm
+    global user
+    user = _user
+    global config
+    config = _config or Config()
+    realm_meta_checkpoint(realm)
+
+def get_user():
+    return user.id if user is not None else ''
+
+def get_tenant():
+    return config.tenant if config is not None and config.tenant is not None else ''
+
+def get_realm():
+    return realm
+
+log = logging.getLogger('rgw_multi.tests')
+
+num_buckets = 0
+run_prefix=''.join(random.choice(string.ascii_lowercase) for _ in range(6))
+
+def get_zone_connection(zone, credentials):
+    """ connect to the zone's first gateway """
+    if isinstance(credentials, list):
+        credentials = credentials[0]
+    return get_gateway_connection(zone.gateways[0], credentials)
+
+def mdlog_list(zone, period = None):
+    cmd = ['mdlog', 'list']
+    if period:
+        cmd += ['--period', period]
+    (mdlog_json, _) = zone.cluster.admin(cmd, read_only=True)
+    return json.loads(mdlog_json)
+
+def meta_sync_status(zone):
+    while True:
+        cmd = ['metadata', 'sync', 'status'] + zone.zone_args()
+        meta_sync_status_json, retcode = zone.cluster.admin(cmd, check_retcode=False, read_only=True)
+        if retcode == 0:
+            break
+        assert(retcode == 2) # ENOENT
+        time.sleep(5)
+
+def mdlog_autotrim(zone):
+    zone.cluster.admin(['mdlog', 'autotrim'])
+
+def datalog_list(zone, period = None):
+    cmd = ['datalog', 'list']
+    (datalog_json, _) = zone.cluster.admin(cmd, read_only=True)
+    return json.loads(datalog_json)
+
+def datalog_autotrim(zone):
+    zone.cluster.admin(['datalog', 'autotrim'])
+
+def bilog_list(zone, bucket, args = None):
+    cmd = ['bilog', 'list', '--bucket', bucket] + (args or [])
+    cmd += ['--tenant', config.tenant, '--uid', user.name] if config.tenant else []
+    bilog, _ = zone.cluster.admin(cmd, read_only=True)
+    return json.loads(bilog)
+
+def bilog_autotrim(zone, args = None):
+    zone.cluster.admin(['bilog', 'autotrim'] + (args or []))
+
+def parse_meta_sync_status(meta_sync_status_json):
+    log.debug('current meta sync status=%s', meta_sync_status_json)
+    sync_status = json.loads(meta_sync_status_json)
+
+    sync_info = sync_status['sync_status']['info']
+    global_sync_status = sync_info['status']
+    num_shards = sync_info['num_shards']
+    period = sync_info['period']
+    realm_epoch = sync_info['realm_epoch']
+
+    sync_markers=sync_status['sync_status']['markers']
+    log.debug('sync_markers=%s', sync_markers)
+    assert(num_shards == len(sync_markers))
+
+    markers={}
+    for i in range(num_shards):
+        # get marker, only if it's an incremental marker for the same realm epoch
+        if realm_epoch > sync_markers[i]['val']['realm_epoch'] or sync_markers[i]['val']['state'] == 0:
+            markers[i] = ''
+        else:
+            markers[i] = sync_markers[i]['val']['marker']
+
+    return period, realm_epoch, num_shards, markers
+
+def meta_sync_status(zone):
+    for _ in range(config.checkpoint_retries):
+        cmd = ['metadata', 'sync', 'status'] + zone.zone_args()
+        meta_sync_status_json, retcode = zone.cluster.admin(cmd, check_retcode=False, read_only=True)
+        if retcode == 0:
+            return parse_meta_sync_status(meta_sync_status_json)
+        assert(retcode == 2) # ENOENT
+        time.sleep(config.checkpoint_delay)
+
+    assert False, 'failed to read metadata sync status for zone=%s' % zone.name
+
+def meta_master_log_status(master_zone):
+    cmd = ['mdlog', 'status'] + master_zone.zone_args()
+    mdlog_status_json, retcode = master_zone.cluster.admin(cmd, read_only=True)
+    mdlog_status = json.loads(mdlog_status_json)
+
+    markers = {i: s['marker'] for i, s in enumerate(mdlog_status)}
+    log.debug('master meta markers=%s', markers)
+    return markers
+
+def compare_meta_status(zone, log_status, sync_status):
+    if len(log_status) != len(sync_status):
+        log.error('len(log_status)=%d, len(sync_status)=%d', len(log_status), len(sync_status))
+        return False
+
+    msg = ''
+    for i, l, s in zip(log_status, log_status.values(), sync_status.values()):
+        if l > s:
+            if len(msg):
+                msg += ', '
+            msg += 'shard=' + str(i) + ' master=' + l + ' target=' + s
+
+    if len(msg) > 0:
+        log.warning('zone %s behind master: %s', zone.name, msg)
+        return False
+
+    return True
+
+def zone_meta_checkpoint(zone, meta_master_zone = None, master_status = None):
+    if not meta_master_zone:
+        meta_master_zone = zone.realm().meta_master_zone()
+    if not master_status:
+        master_status = meta_master_log_status(meta_master_zone)
+
+    current_realm_epoch = realm.current_period.data['realm_epoch']
+
+    log.info('starting meta checkpoint for zone=%s', zone.name)
+
+    for _ in range(config.checkpoint_retries):
+        period, realm_epoch, num_shards, sync_status = meta_sync_status(zone)
+        if realm_epoch < current_realm_epoch:
+            log.warning('zone %s is syncing realm epoch=%d, behind current realm epoch=%d',
+                        zone.name, realm_epoch, current_realm_epoch)
+        else:
+            log.debug('log_status=%s', master_status)
+            log.debug('sync_status=%s', sync_status)
+            if compare_meta_status(zone, master_status, sync_status):
+                log.info('finish meta checkpoint for zone=%s', zone.name)
+                return
+
+        time.sleep(config.checkpoint_delay)
+    assert False, 'failed meta checkpoint for zone=%s' % zone.name
+
+def zonegroup_meta_checkpoint(zonegroup, meta_master_zone = None, master_status = None):
+    if not meta_master_zone:
+        meta_master_zone = zonegroup.realm().meta_master_zone()
+    if not master_status:
+        master_status = meta_master_log_status(meta_master_zone)
+
+    for zone in zonegroup.zones:
+        if zone == meta_master_zone:
+            continue
+        zone_meta_checkpoint(zone, meta_master_zone, master_status)
+
+def realm_meta_checkpoint(realm):
+    log.info('meta checkpoint')
+
+    meta_master_zone = realm.meta_master_zone()
+    master_status = meta_master_log_status(meta_master_zone)
+
+    for zonegroup in realm.current_period.zonegroups:
+        zonegroup_meta_checkpoint(zonegroup, meta_master_zone, master_status)
+
+def parse_data_sync_status(data_sync_status_json):
+    log.debug('current data sync status=%s', data_sync_status_json)
+    sync_status = json.loads(data_sync_status_json)
+
+    global_sync_status=sync_status['sync_status']['info']['status']
+    num_shards=sync_status['sync_status']['info']['num_shards']
+
+    sync_markers=sync_status['sync_status']['markers']
+    log.debug('sync_markers=%s', sync_markers)
+    assert(num_shards == len(sync_markers))
+
+    markers={}
+    for i in range(num_shards):
+        markers[i] = sync_markers[i]['val']['marker']
+
+    return (num_shards, markers)
+
+def data_sync_status(target_zone, source_zone):
+    if target_zone == source_zone:
+        return None
+
+    for _ in range(config.checkpoint_retries):
+        cmd = ['data', 'sync', 'status'] + target_zone.zone_args()
+        cmd += ['--source-zone', source_zone.name]
+        data_sync_status_json, retcode = target_zone.cluster.admin(cmd, check_retcode=False, read_only=True)
+        if retcode == 0:
+            return parse_data_sync_status(data_sync_status_json)
+
+        assert(retcode == 2) # ENOENT
+        time.sleep(config.checkpoint_delay)
+
+    assert False, 'failed to read data sync status for target_zone=%s source_zone=%s' % \
+                  (target_zone.name, source_zone.name)
+
+def bucket_sync_status(target_zone, source_zone, bucket_name):
+    if target_zone == source_zone:
+        return None
+
+    cmd = ['bucket', 'sync', 'markers'] + target_zone.zone_args()
+    cmd += ['--source-zone', source_zone.name]
+    cmd += ['--bucket', bucket_name]
+    cmd += ['--tenant', config.tenant, '--uid', user.name] if config.tenant else []
+    while True:
+        bucket_sync_status_json, retcode = target_zone.cluster.admin(cmd, check_retcode=False, read_only=True)
+        if retcode == 0:
+            break
+
+        assert(retcode == 2) # ENOENT
+
+    log.debug('current bucket sync markers=%s', bucket_sync_status_json)
+    sync_status = json.loads(bucket_sync_status_json)
+
+    markers={}
+    for entry in sync_status:
+        val = entry['val']
+        if val['status'] == 'incremental-sync':
+            pos = val['inc_marker']['position'].split('#')[-1] # get rid of shard id; e.g., 6#00000000002.132.3 -> 00000000002.132.3
+        else:
+            pos = ''
+        markers[entry['key']] = pos
+
+    return markers
+
+def data_source_log_status(source_zone):
+    source_cluster = source_zone.cluster
+    cmd = ['datalog', 'status'] + source_zone.zone_args()
+    datalog_status_json, retcode = source_cluster.admin(cmd, read_only=True)
+    datalog_status = json.loads(datalog_status_json)
+
+    markers = {i: s['marker'] for i, s in enumerate(datalog_status)}
+    log.debug('data markers for zone=%s markers=%s', source_zone.name, markers)
+    return markers
+
+def bucket_source_log_status(source_zone, bucket_name):
+    cmd = ['bilog', 'status'] + source_zone.zone_args()
+    cmd += ['--bucket', bucket_name]
+    cmd += ['--tenant', config.tenant, '--uid', user.name] if config.tenant else []
+    source_cluster = source_zone.cluster
+    bilog_status_json, retcode = source_cluster.admin(cmd, read_only=True)
+    bilog_status = json.loads(bilog_status_json)
+
+    m={}
+    markers={}
+    try:
+        m = bilog_status['markers']
+    except:
+        pass
+
+    for s in m:
+        key = s['key']
+        val = s['val']
+        markers[key] = val
+
+    log.debug('bilog markers for zone=%s bucket=%s markers=%s', source_zone.name, bucket_name, markers)
+    return markers
+
+def compare_data_status(target_zone, source_zone, log_status, sync_status):
+    if len(log_status) != len(sync_status):
+        log.error('len(log_status)=%d len(sync_status)=%d', len(log_status), len(sync_status))
+        return False
+
+    msg =  ''
+    for i, l, s in zip(log_status, log_status.values(), sync_status.values()):
+        if l > s:
+            if len(msg):
+                msg += ', '
+            msg += 'shard=' + str(i) + ' master=' + l + ' target=' + s
+
+    if len(msg) > 0:
+        log.warning('data of zone %s behind zone %s: %s', target_zone.name, source_zone.name, msg)
+        return False
+
+    return True
+
+def compare_bucket_status(target_zone, source_zone, bucket_name, log_status, sync_status):
+    if len(log_status) != len(sync_status):
+        log.error('len(log_status)=%d len(sync_status)=%d', len(log_status), len(sync_status))
+        return False
+
+    msg =  ''
+    for i, l, s in zip(log_status, log_status.values(), sync_status.values()):
+        if l > s:
+            if len(msg):
+                msg += ', '
+            msg += 'shard=' + str(i) + ' master=' + l + ' target=' + s
+
+    if len(msg) > 0:
+        log.warning('bucket %s zone %s behind zone %s: %s', bucket_name, target_zone.name, source_zone.name, msg)
+        return False
+
+    return True
+
+def zone_data_checkpoint(target_zone, source_zone):
+    if target_zone == source_zone:
+        return
+
+    log_status = data_source_log_status(source_zone)
+    log.info('starting data checkpoint for target_zone=%s source_zone=%s', target_zone.name, source_zone.name)
+
+    for _ in range(config.checkpoint_retries):
+        num_shards, sync_status = data_sync_status(target_zone, source_zone)
+
+        log.debug('log_status=%s', log_status)
+        log.debug('sync_status=%s', sync_status)
+
+        if compare_data_status(target_zone, source_zone, log_status, sync_status):
+            log.info('finished data checkpoint for target_zone=%s source_zone=%s',
+                     target_zone.name, source_zone.name)
+            return
+        time.sleep(config.checkpoint_delay)
+
+    assert False, 'failed data checkpoint for target_zone=%s source_zone=%s' % \
+                  (target_zone.name, source_zone.name)
+
+def zonegroup_data_checkpoint(zonegroup_conns):
+    for source_conn in zonegroup_conns.rw_zones:
+        for target_conn in zonegroup_conns.zones:
+            if source_conn.zone == target_conn.zone:
+                continue
+            log.debug('data checkpoint: source=%s target=%s', source_conn.zone.name, target_conn.zone.name)
+            zone_data_checkpoint(target_conn.zone, source_conn.zone)
+
+def zone_bucket_checkpoint(target_zone, source_zone, bucket_name):
+    if target_zone == source_zone:
+        return
+
+    log_status = bucket_source_log_status(source_zone, bucket_name)
+    log.info('starting bucket checkpoint for target_zone=%s source_zone=%s bucket=%s', target_zone.name, source_zone.name, bucket_name)
+
+    for _ in range(config.checkpoint_retries):
+        sync_status = bucket_sync_status(target_zone, source_zone, bucket_name)
+
+        log.debug('log_status=%s', log_status)
+        log.debug('sync_status=%s', sync_status)
+
+        if compare_bucket_status(target_zone, source_zone, bucket_name, log_status, sync_status):
+            log.info('finished bucket checkpoint for target_zone=%s source_zone=%s bucket=%s', target_zone.name, source_zone.name, bucket_name)
+            return
+
+        time.sleep(config.checkpoint_delay)
+
+    assert False, 'failed bucket checkpoint for target_zone=%s source_zone=%s bucket=%s' % \
+                  (target_zone.name, source_zone.name, bucket_name)
+
+def zonegroup_bucket_checkpoint(zonegroup_conns, bucket_name):
+    for source_conn in zonegroup_conns.rw_zones:
+        for target_conn in zonegroup_conns.zones:
+            if source_conn.zone == target_conn.zone:
+                continue
+            log.debug('bucket checkpoint: source=%s target=%s bucket=%s', source_conn.zone.name, target_conn.zone.name, bucket_name)
+            zone_bucket_checkpoint(target_conn.zone, source_conn.zone, bucket_name)
+    for source_conn, target_conn in combinations(zonegroup_conns.zones, 2):
+        if target_conn.zone.has_buckets():
+            target_conn.check_bucket_eq(source_conn, bucket_name)
+
+def set_master_zone(zone):
+    zone.modify(zone.cluster, ['--master'])
+    zonegroup = zone.zonegroup
+    zonegroup.period.update(zone, commit=True)
+    zonegroup.master_zone = zone
+    log.info('Set master zone=%s, waiting %ds for reconfiguration..', zone.name, config.reconfigure_delay)
+    time.sleep(config.reconfigure_delay)
+
+def set_sync_from_all(zone, flag):
+    s = 'true' if flag else 'false'
+    zone.modify(zone.cluster, ['--sync-from-all={}'.format(s)])
+    zonegroup = zone.zonegroup
+    zonegroup.period.update(zone, commit=True)
+    log.info('Set sync_from_all flag on zone %s to %s', zone.name, s)
+    time.sleep(config.reconfigure_delay)
+
+def set_redirect_zone(zone, redirect_zone):
+    id_str = redirect_zone.id if redirect_zone else ''
+    zone.modify(zone.cluster, ['--redirect-zone={}'.format(id_str)])
+    zonegroup = zone.zonegroup
+    zonegroup.period.update(zone, commit=True)
+    log.info('Set redirect_zone zone %s to "%s"', zone.name, id_str)
+    time.sleep(config.reconfigure_delay)
+
+def enable_bucket_sync(zone, bucket_name):
+    cmd = ['bucket', 'sync', 'enable', '--bucket', bucket_name] + zone.zone_args()
+    zone.cluster.admin(cmd)
+
+def disable_bucket_sync(zone, bucket_name):
+    cmd = ['bucket', 'sync', 'disable', '--bucket', bucket_name] + zone.zone_args()
+    zone.cluster.admin(cmd)
+
+def check_buckets_sync_status_obj_not_exist(zone, buckets):
+    for _ in range(config.checkpoint_retries):
+        cmd = ['log', 'list'] + zone.zone_arg()
+        log_list, ret = zone.cluster.admin(cmd, check_retcode=False, read_only=True)
+        for bucket in buckets:
+            if log_list.find(':'+bucket+":") >= 0:
+                break
+        else:
+            return
+        time.sleep(config.checkpoint_delay)
+    assert False
+
+def gen_bucket_name():
+    global num_buckets
+
+    num_buckets += 1
+    return run_prefix + '-' + str(num_buckets)
+
+class ZonegroupConns:
+    def __init__(self, zonegroup):
+        self.zonegroup = zonegroup
+        self.zones = []
+        self.ro_zones = []
+        self.rw_zones = []
+        self.master_zone = None
+        for z in zonegroup.zones:
+            zone_conn = z.get_conn(user.credentials)
+            self.zones.append(zone_conn)
+            if z.is_read_only():
+                self.ro_zones.append(zone_conn)
+            else:
+                self.rw_zones.append(zone_conn)
+
+            if z == zonegroup.master_zone:
+                self.master_zone = zone_conn
+
+def check_all_buckets_exist(zone_conn, buckets):
+    if not zone_conn.zone.has_buckets():
+        return True
+
+    for b in buckets:
+        try:
+            zone_conn.get_bucket(b)
+        except:
+            log.critical('zone %s does not contain bucket %s', zone.name, b)
+            return False
+
+    return True
+
+def check_all_buckets_dont_exist(zone_conn, buckets):
+    if not zone_conn.zone.has_buckets():
+        return True
+
+    for b in buckets:
+        try:
+            zone_conn.get_bucket(b)
+        except:
+            continue
+
+        log.critical('zone %s contains bucket %s', zone.zone, b)
+        return False
+
+    return True
+
+def create_bucket_per_zone(zonegroup_conns, buckets_per_zone = 1):
+    buckets = []
+    zone_bucket = []
+    for zone in zonegroup_conns.rw_zones:
+        for i in range(buckets_per_zone):
+            bucket_name = gen_bucket_name()
+            log.info('create bucket zone=%s name=%s', zone.name, bucket_name)
+            bucket = zone.create_bucket(bucket_name)
+            buckets.append(bucket_name)
+            zone_bucket.append((zone, bucket))
+
+    return buckets, zone_bucket
+
+def create_bucket_per_zone_in_realm():
+    buckets = []
+    zone_bucket = []
+    for zonegroup in realm.current_period.zonegroups:
+        zg_conn = ZonegroupConns(zonegroup)
+        b, z = create_bucket_per_zone(zg_conn)
+        buckets.extend(b)
+        zone_bucket.extend(z)
+    return buckets, zone_bucket
+
+def test_bucket_create():
+    zonegroup = realm.master_zonegroup()
+    zonegroup_conns = ZonegroupConns(zonegroup)
+    buckets, _ = create_bucket_per_zone(zonegroup_conns)
+    zonegroup_meta_checkpoint(zonegroup)
+
+    for zone in zonegroup_conns.zones:
+        assert check_all_buckets_exist(zone, buckets)
+
+def test_bucket_recreate():
+    zonegroup = realm.master_zonegroup()
+    zonegroup_conns = ZonegroupConns(zonegroup)
+    buckets, _ = create_bucket_per_zone(zonegroup_conns)
+    zonegroup_meta_checkpoint(zonegroup)
+
+
+    for zone in zonegroup_conns.zones:
+        assert check_all_buckets_exist(zone, buckets)
+
+    # recreate buckets on all zones, make sure they weren't removed
+    for zone in zonegroup_conns.rw_zones:
+        for bucket_name in buckets:
+            bucket = zone.create_bucket(bucket_name)
+
+    for zone in zonegroup_conns.zones:
+        assert check_all_buckets_exist(zone, buckets)
+
+    zonegroup_meta_checkpoint(zonegroup)
+
+    for zone in zonegroup_conns.zones:
+        assert check_all_buckets_exist(zone, buckets)
+
+def test_bucket_remove():
+    zonegroup = realm.master_zonegroup()
+    zonegroup_conns = ZonegroupConns(zonegroup)
+    buckets, zone_bucket = create_bucket_per_zone(zonegroup_conns)
+    zonegroup_meta_checkpoint(zonegroup)
+
+    for zone in zonegroup_conns.zones:
+        assert check_all_buckets_exist(zone, buckets)
+
+    for zone, bucket_name in zone_bucket:
+        zone.conn.delete_bucket(bucket_name)
+
+    zonegroup_meta_checkpoint(zonegroup)
+
+    for zone in zonegroup_conns.zones:
+        assert check_all_buckets_dont_exist(zone, buckets)
+
+def get_bucket(zone, bucket_name):
+    return zone.conn.get_bucket(bucket_name)
+
+def get_key(zone, bucket_name, obj_name):
+    b = get_bucket(zone, bucket_name)
+    return b.get_key(obj_name)
+
+def new_key(zone, bucket_name, obj_name):
+    b = get_bucket(zone, bucket_name)
+    return b.new_key(obj_name)
+
+def check_bucket_eq(zone_conn1, zone_conn2, bucket):
+    if zone_conn2.zone.has_buckets():
+        zone_conn2.check_bucket_eq(zone_conn1, bucket.name)
+
+def test_object_sync():
+    zonegroup = realm.master_zonegroup()
+    zonegroup_conns = ZonegroupConns(zonegroup)
+    buckets, zone_bucket = create_bucket_per_zone(zonegroup_conns)
+
+    objnames = [ 'myobj', '_myobj', ':', '&' ]
+    content = 'asdasd'
+
+    # don't wait for meta sync just yet
+    for zone, bucket_name in zone_bucket:
+        for objname in objnames:
+            k = new_key(zone, bucket_name, objname)
+            k.set_contents_from_string(content)
+
+    zonegroup_meta_checkpoint(zonegroup)
+
+    for source_conn, bucket in zone_bucket:
+        for target_conn in zonegroup_conns.zones:
+            if source_conn.zone == target_conn.zone:
+                continue
+
+            zone_bucket_checkpoint(target_conn.zone, source_conn.zone, bucket.name)
+            check_bucket_eq(source_conn, target_conn, bucket)
+
+def test_object_delete():
+    zonegroup = realm.master_zonegroup()
+    zonegroup_conns = ZonegroupConns(zonegroup)
+    buckets, zone_bucket = create_bucket_per_zone(zonegroup_conns)
+
+    objname = 'myobj'
+    content = 'asdasd'
+
+    # don't wait for meta sync just yet
+    for zone, bucket in zone_bucket:
+        k = new_key(zone, bucket, objname)
+        k.set_contents_from_string(content)
+
+    zonegroup_meta_checkpoint(zonegroup)
+
+    # check object exists
+    for source_conn, bucket in zone_bucket:
+        for target_conn in zonegroup_conns.zones:
+            if source_conn.zone == target_conn.zone:
+                continue
+
+            zone_bucket_checkpoint(target_conn.zone, source_conn.zone, bucket.name)
+            check_bucket_eq(source_conn, target_conn, bucket)
+
+    # check object removal
+    for source_conn, bucket in zone_bucket:
+        k = get_key(source_conn, bucket, objname)
+        k.delete()
+        for target_conn in zonegroup_conns.zones:
+            if source_conn.zone == target_conn.zone:
+                continue
+
+            zone_bucket_checkpoint(target_conn.zone, source_conn.zone, bucket.name)
+            check_bucket_eq(source_conn, target_conn, bucket)
+
+def get_latest_object_version(key):
+    for k in key.bucket.list_versions(key.name):
+        if k.is_latest:
+            return k
+    return None
+
+def test_versioned_object_incremental_sync():
+    zonegroup = realm.master_zonegroup()
+    zonegroup_conns = ZonegroupConns(zonegroup)
+    buckets, zone_bucket = create_bucket_per_zone(zonegroup_conns)
+
+    # enable versioning
+    for _, bucket in zone_bucket:
+        bucket.configure_versioning(True)
+
+    zonegroup_meta_checkpoint(zonegroup)
+
+    # upload a dummy object to each bucket and wait for sync. this forces each
+    # bucket to finish a full sync and switch to incremental
+    for source_conn, bucket in zone_bucket:
+        new_key(source_conn, bucket, 'dummy').set_contents_from_string('')
+        for target_conn in zonegroup_conns.zones:
+            if source_conn.zone == target_conn.zone:
+                continue
+            zone_bucket_checkpoint(target_conn.zone, source_conn.zone, bucket.name)
+
+    for _, bucket in zone_bucket:
+        # create and delete multiple versions of an object from each zone
+        for zone_conn in zonegroup_conns.rw_zones:
+            obj = 'obj-' + zone_conn.name
+            k = new_key(zone_conn, bucket, obj)
+
+            k.set_contents_from_string('version1')
+            log.debug('version1 id=%s', k.version_id)
+            # don't delete version1 - this tests that the initial version
+            # doesn't get squashed into later versions
+
+            # create and delete the following object versions to test that
+            # the operations don't race with each other during sync
+            k.set_contents_from_string('version2')
+            log.debug('version2 id=%s', k.version_id)
+            k.bucket.delete_key(obj, version_id=k.version_id)
+
+            k.set_contents_from_string('version3')
+            log.debug('version3 id=%s', k.version_id)
+            k.bucket.delete_key(obj, version_id=k.version_id)
+
+    for _, bucket in zone_bucket:
+        zonegroup_bucket_checkpoint(zonegroup_conns, bucket.name)
+
+    for _, bucket in zone_bucket:
+        # overwrite the acls to test that metadata-only entries are applied
+        for zone_conn in zonegroup_conns.rw_zones:
+            obj = 'obj-' + zone_conn.name
+            k = new_key(zone_conn, bucket.name, obj)
+            v = get_latest_object_version(k)
+            v.make_public()
+
+    for _, bucket in zone_bucket:
+        zonegroup_bucket_checkpoint(zonegroup_conns, bucket.name)
+
+def test_concurrent_versioned_object_incremental_sync():
+    zonegroup = realm.master_zonegroup()
+    zonegroup_conns = ZonegroupConns(zonegroup)
+    zone = zonegroup_conns.rw_zones[0]
+
+    # create a versioned bucket
+    bucket = zone.create_bucket(gen_bucket_name())
+    log.debug('created bucket=%s', bucket.name)
+    bucket.configure_versioning(True)
+
+    zonegroup_meta_checkpoint(zonegroup)
+
+    # upload a dummy object and wait for sync. this forces each zone to finish
+    # a full sync and switch to incremental
+    new_key(zone, bucket, 'dummy').set_contents_from_string('')
+    zonegroup_bucket_checkpoint(zonegroup_conns, bucket.name)
+
+    # create several concurrent versions on each zone and let them race to sync
+    obj = 'obj'
+    for i in range(10):
+        for zone_conn in zonegroup_conns.rw_zones:
+            k = new_key(zone_conn, bucket, obj)
+            k.set_contents_from_string('version1')
+            log.debug('zone=%s version=%s', zone_conn.zone.name, k.version_id)
+
+    zonegroup_bucket_checkpoint(zonegroup_conns, bucket.name)
+    zonegroup_data_checkpoint(zonegroup_conns)
+
+def test_version_suspended_incremental_sync():
+    zonegroup = realm.master_zonegroup()
+    zonegroup_conns = ZonegroupConns(zonegroup)
+
+    zone = zonegroup_conns.rw_zones[0]
+
+    # create a non-versioned bucket
+    bucket = zone.create_bucket(gen_bucket_name())
+    log.debug('created bucket=%s', bucket.name)
+    zonegroup_meta_checkpoint(zonegroup)
+
+    # upload an initial object
+    key1 = new_key(zone, bucket, 'obj')
+    key1.set_contents_from_string('')
+    log.debug('created initial version id=%s', key1.version_id)
+    zonegroup_bucket_checkpoint(zonegroup_conns, bucket.name)
+
+    # enable versioning
+    bucket.configure_versioning(True)
+    zonegroup_meta_checkpoint(zonegroup)
+
+    # re-upload the object as a new version
+    key2 = new_key(zone, bucket, 'obj')
+    key2.set_contents_from_string('')
+    log.debug('created new version id=%s', key2.version_id)
+    zonegroup_bucket_checkpoint(zonegroup_conns, bucket.name)
+
+    # suspend versioning
+    bucket.configure_versioning(False)
+    zonegroup_meta_checkpoint(zonegroup)
+
+    # re-upload the object as a 'null' version
+    key3 = new_key(zone, bucket, 'obj')
+    key3.set_contents_from_string('')
+    log.debug('created null version id=%s', key3.version_id)
+    zonegroup_bucket_checkpoint(zonegroup_conns, bucket.name)
+
+def test_delete_marker_full_sync():
+    zonegroup = realm.master_zonegroup()
+    zonegroup_conns = ZonegroupConns(zonegroup)
+    buckets, zone_bucket = create_bucket_per_zone(zonegroup_conns)
+
+    # enable versioning
+    for _, bucket in zone_bucket:
+        bucket.configure_versioning(True)
+    zonegroup_meta_checkpoint(zonegroup)
+
+    for zone, bucket in zone_bucket:
+        # upload an initial object
+        key1 = new_key(zone, bucket, 'obj')
+        key1.set_contents_from_string('')
+
+        # create a delete marker
+        key2 = new_key(zone, bucket, 'obj')
+        key2.delete()
+
+    # wait for full sync
+    for _, bucket in zone_bucket:
+        zonegroup_bucket_checkpoint(zonegroup_conns, bucket.name)
+
+def test_suspended_delete_marker_full_sync():
+    zonegroup = realm.master_zonegroup()
+    zonegroup_conns = ZonegroupConns(zonegroup)
+    buckets, zone_bucket = create_bucket_per_zone(zonegroup_conns)
+
+    # enable/suspend versioning
+    for _, bucket in zone_bucket:
+        bucket.configure_versioning(True)
+        bucket.configure_versioning(False)
+    zonegroup_meta_checkpoint(zonegroup)
+
+    for zone, bucket in zone_bucket:
+        # upload an initial object
+        key1 = new_key(zone, bucket, 'obj')
+        key1.set_contents_from_string('')
+
+        # create a delete marker
+        key2 = new_key(zone, bucket, 'obj')
+        key2.delete()
+
+    # wait for full sync
+    for _, bucket in zone_bucket:
+        zonegroup_bucket_checkpoint(zonegroup_conns, bucket.name)
+
+def test_bucket_versioning():
+    buckets, zone_bucket = create_bucket_per_zone_in_realm()
+    for _, bucket in zone_bucket:
+        bucket.configure_versioning(True)
+        res = bucket.get_versioning_status()
+        key = 'Versioning'
+        assert(key in res and res[key] == 'Enabled')
+
+def test_bucket_acl():
+    buckets, zone_bucket = create_bucket_per_zone_in_realm()
+    for _, bucket in zone_bucket:
+        assert(len(bucket.get_acl().acl.grants) == 1) # single grant on owner
+        bucket.set_acl('public-read')
+        assert(len(bucket.get_acl().acl.grants) == 2) # new grant on AllUsers
+
+def test_bucket_cors():
+    buckets, zone_bucket = create_bucket_per_zone_in_realm()
+    for _, bucket in zone_bucket:
+        cors_cfg = CORSConfiguration()
+        cors_cfg.add_rule(['DELETE'], 'https://www.example.com', allowed_header='*', max_age_seconds=3000)
+        bucket.set_cors(cors_cfg)
+        assert(bucket.get_cors().to_xml() == cors_cfg.to_xml())
+
+def test_bucket_delete_notempty():
+    zonegroup = realm.master_zonegroup()
+    zonegroup_conns = ZonegroupConns(zonegroup)
+    buckets, zone_bucket = create_bucket_per_zone(zonegroup_conns)
+    zonegroup_meta_checkpoint(zonegroup)
+
+    for zone_conn, bucket_name in zone_bucket:
+        # upload an object to each bucket on its own zone
+        conn = zone_conn.get_connection()
+        bucket = conn.get_bucket(bucket_name)
+        k = bucket.new_key('foo')
+        k.set_contents_from_string('bar')
+        # attempt to delete the bucket before this object can sync
+        try:
+            conn.delete_bucket(bucket_name)
+        except boto.exception.S3ResponseError as e:
+            assert(e.error_code == 'BucketNotEmpty')
+            continue
+        assert False # expected 409 BucketNotEmpty
+
+    # assert that each bucket still exists on the master
+    c1 = zonegroup_conns.master_zone.conn
+    for _, bucket_name in zone_bucket:
+        assert c1.get_bucket(bucket_name)
+
+def test_multi_period_incremental_sync():
+    zonegroup = realm.master_zonegroup()
+    if len(zonegroup.zones) < 3:
+        raise SkipTest("test_multi_period_incremental_sync skipped. Requires 3 or more zones in master zonegroup.")
+
+    # periods to include in mdlog comparison
+    mdlog_periods = [realm.current_period.id]
+
+    # create a bucket in each zone
+    zonegroup_conns = ZonegroupConns(zonegroup)
+    buckets, zone_bucket = create_bucket_per_zone(zonegroup_conns)
+
+    zonegroup_meta_checkpoint(zonegroup)
+
+    z1, z2, z3 = zonegroup.zones[0:3]
+    assert(z1 == zonegroup.master_zone)
+
+    # kill zone 3 gateways to freeze sync status to incremental in first period
+    z3.stop()
+
+    # change master to zone 2 -> period 2
+    set_master_zone(z2)
+    mdlog_periods += [realm.current_period.id]
+
+    for zone_conn, _ in zone_bucket:
+        if zone_conn.zone == z3:
+            continue
+        bucket_name = gen_bucket_name()
+        log.info('create bucket zone=%s name=%s', zone_conn.name, bucket_name)
+        bucket = zone_conn.conn.create_bucket(bucket_name)
+        buckets.append(bucket_name)
+
+    # wait for zone 1 to sync
+    zone_meta_checkpoint(z1)
+
+    # change master back to zone 1 -> period 3
+    set_master_zone(z1)
+    mdlog_periods += [realm.current_period.id]
+
+    for zone_conn, bucket_name in zone_bucket:
+        if zone_conn.zone == z3:
+            continue
+        bucket_name = gen_bucket_name()
+        log.info('create bucket zone=%s name=%s', zone_conn.name, bucket_name)
+        zone_conn.conn.create_bucket(bucket_name)
+        buckets.append(bucket_name)
+
+    # restart zone 3 gateway and wait for sync
+    z3.start()
+    zonegroup_meta_checkpoint(zonegroup)
+
+    # verify that we end up with the same objects
+    for bucket_name in buckets:
+        for source_conn, _ in zone_bucket:
+            for target_conn in zonegroup_conns.zones:
+                if source_conn.zone == target_conn.zone:
+                    continue
+
+                if target_conn.zone.has_buckets():
+                    target_conn.check_bucket_eq(source_conn, bucket_name)
+
+    # verify that mdlogs are not empty and match for each period
+    for period in mdlog_periods:
+        master_mdlog = mdlog_list(z1, period)
+        assert len(master_mdlog) > 0
+        for zone in zonegroup.zones:
+            if zone == z1:
+                continue
+            mdlog = mdlog_list(zone, period)
+            assert len(mdlog) == len(master_mdlog)
+
+    # autotrim mdlogs for master zone
+    mdlog_autotrim(z1)
+
+    # autotrim mdlogs for peers
+    for zone in zonegroup.zones:
+        if zone == z1:
+            continue
+        mdlog_autotrim(zone)
+
+    # verify that mdlogs are empty for each period
+    for period in mdlog_periods:
+        for zone in zonegroup.zones:
+            mdlog = mdlog_list(zone, period)
+            assert len(mdlog) == 0
+
+def test_datalog_autotrim():
+    zonegroup = realm.master_zonegroup()
+    zonegroup_conns = ZonegroupConns(zonegroup)
+    buckets, zone_bucket = create_bucket_per_zone(zonegroup_conns)
+
+    # upload an object to each zone to generate a datalog entry
+    for zone, bucket in zone_bucket:
+        k = new_key(zone, bucket.name, 'key')
+        k.set_contents_from_string('body')
+
+    # wait for data sync to catch up
+    zonegroup_data_checkpoint(zonegroup_conns)
+
+    # trim each datalog
+    for zone, _ in zone_bucket:
+        datalog_autotrim(zone.zone)
+        datalog = datalog_list(zone.zone)
+        assert len(datalog) == 0
+
+def test_multi_zone_redirect():
+    zonegroup = realm.master_zonegroup()
+    if len(zonegroup.rw_zones) < 2:
+        raise SkipTest("test_multi_period_incremental_sync skipped. Requires 3 or more zones in master zonegroup.")
+
+    zonegroup_conns = ZonegroupConns(zonegroup)
+    (zc1, zc2) = zonegroup_conns.rw_zones[0:2]
+
+    z1, z2 = (zc1.zone, zc2.zone)
+
+    set_sync_from_all(z2, False)
+
+    # create a bucket on the first zone
+    bucket_name = gen_bucket_name()
+    log.info('create bucket zone=%s name=%s', z1.name, bucket_name)
+    bucket = zc1.conn.create_bucket(bucket_name)
+    obj = 'testredirect'
+
+    key = bucket.new_key(obj)
+    data = 'A'*512
+    key.set_contents_from_string(data)
+
+    zonegroup_meta_checkpoint(zonegroup)
+
+    # try to read object from second zone (should fail)
+    bucket2 = get_bucket(zc2, bucket_name)
+    assert_raises(boto.exception.S3ResponseError, bucket2.get_key, obj)
+
+    set_redirect_zone(z2, z1)
+
+    key2 = bucket2.get_key(obj)
+
+    eq(data, key2.get_contents_as_string(encoding='ascii'))
+
+    key = bucket.new_key(obj)
+
+    for x in ['a', 'b', 'c', 'd']:
+        data = x*512
+        key.set_contents_from_string(data)
+        eq(data, key2.get_contents_as_string(encoding='ascii'))
+
+    # revert config changes
+    set_sync_from_all(z2, True)
+    set_redirect_zone(z2, None)
+
+def test_zonegroup_remove():
+    zonegroup = realm.master_zonegroup()
+    zonegroup_conns = ZonegroupConns(zonegroup)
+    if len(zonegroup.zones) < 2:
+        raise SkipTest("test_zonegroup_remove skipped. Requires 2 or more zones in master zonegroup.")
+
+    zonegroup_meta_checkpoint(zonegroup)
+    z1, z2 = zonegroup.zones[0:2]
+    c1, c2 = (z1.cluster, z2.cluster)
+
+    # get admin credentials out of existing zone
+    system_key = z1.data['system_key']
+    admin_creds = Credentials(system_key['access_key'], system_key['secret_key'])
+
+    # create a new zone in zonegroup on c2 and commit
+    zone = Zone('remove', zonegroup, c2)
+    zone.create(c2, admin_creds.credential_args())
+    zonegroup.zones.append(zone)
+    zonegroup.period.update(zone, commit=True)
+
+    zonegroup.remove(c1, zone)
+
+    # another 'zonegroup remove' should fail with ENOENT
+    _, retcode = zonegroup.remove(c1, zone, check_retcode=False)
+    assert(retcode == 2) # ENOENT
+
+    # delete the new zone
+    zone.delete(c2)
+
+    # validate the resulting period
+    zonegroup.period.update(z1, commit=True)
+
+
+def test_zg_master_zone_delete():
+
+    master_zg = realm.master_zonegroup()
+    master_zone = master_zg.master_zone
+
+    assert(len(master_zg.zones) >= 1)
+    master_cluster = master_zg.zones[0].cluster
+
+    rm_zg = ZoneGroup('remove_zg')
+    rm_zg.create(master_cluster)
+
+    rm_zone = Zone('remove', rm_zg, master_cluster)
+    rm_zone.create(master_cluster)
+    master_zg.period.update(master_zone, commit=True)
+
+
+    rm_zone.delete(master_cluster)
+    # Period update: This should now fail as the zone will be the master zone
+    # in that zg
+    _, retcode = master_zg.period.update(master_zone, check_retcode=False)
+    assert(retcode == errno.EINVAL)
+
+    # Proceed to delete the zonegroup as well, previous period now does not
+    # contain a dangling master_zone, this must succeed
+    rm_zg.delete(master_cluster)
+    master_zg.period.update(master_zone, commit=True)
+
+def test_set_bucket_website():
+    buckets, zone_bucket = create_bucket_per_zone_in_realm()
+    for _, bucket in zone_bucket:
+        website_cfg = WebsiteConfiguration(suffix='index.html',error_key='error.html')
+        try:
+            bucket.set_website_configuration(website_cfg)
+        except boto.exception.S3ResponseError as e:
+            if e.error_code == 'MethodNotAllowed':
+                raise SkipTest("test_set_bucket_website skipped. Requires rgw_enable_static_website = 1.")
+        assert(bucket.get_website_configuration_with_xml()[1] == website_cfg.to_xml())
+
+def test_set_bucket_policy():
+    policy = '''{
+  "Version": "2012-10-17",
+  "Statement": [{
+    "Effect": "Allow",
+    "Principal": "*"
+  }]
+}'''
+    buckets, zone_bucket = create_bucket_per_zone_in_realm()
+    for _, bucket in zone_bucket:
+        bucket.set_policy(policy)
+        assert(bucket.get_policy().decode('ascii') == policy)
+
+def test_bucket_sync_disable():
+    zonegroup = realm.master_zonegroup()
+    zonegroup_conns = ZonegroupConns(zonegroup)
+    buckets, zone_bucket = create_bucket_per_zone(zonegroup_conns)
+
+    for bucket_name in buckets:
+        disable_bucket_sync(realm.meta_master_zone(), bucket_name)
+
+    for zone in zonegroup.zones:
+        check_buckets_sync_status_obj_not_exist(zone, buckets)
+
+    zonegroup_data_checkpoint(zonegroup_conns)
+
+def test_bucket_sync_enable_right_after_disable():
+    zonegroup = realm.master_zonegroup()
+    zonegroup_conns = ZonegroupConns(zonegroup)
+    buckets, zone_bucket = create_bucket_per_zone(zonegroup_conns)
+
+    objnames = ['obj1', 'obj2', 'obj3', 'obj4']
+    content = 'asdasd'
+
+    for zone, bucket in zone_bucket:
+        for objname in objnames:
+            k = new_key(zone, bucket.name, objname)
+            k.set_contents_from_string(content)
+
+    for bucket_name in buckets:
+        zonegroup_bucket_checkpoint(zonegroup_conns, bucket_name)
+
+    for bucket_name in buckets:
+        disable_bucket_sync(realm.meta_master_zone(), bucket_name)
+        enable_bucket_sync(realm.meta_master_zone(), bucket_name)
+
+    objnames_2 = ['obj5', 'obj6', 'obj7', 'obj8']
+
+    for zone, bucket in zone_bucket:
+        for objname in objnames_2:
+            k = new_key(zone, bucket.name, objname)
+            k.set_contents_from_string(content)
+
+    for bucket_name in buckets:
+        zonegroup_bucket_checkpoint(zonegroup_conns, bucket_name)
+
+    zonegroup_data_checkpoint(zonegroup_conns)
+
+def test_bucket_sync_disable_enable():
+    zonegroup = realm.master_zonegroup()
+    zonegroup_conns = ZonegroupConns(zonegroup)
+    buckets, zone_bucket = create_bucket_per_zone(zonegroup_conns)
+
+    objnames = [ 'obj1', 'obj2', 'obj3', 'obj4' ]
+    content = 'asdasd'
+
+    for zone, bucket in zone_bucket:
+        for objname in objnames:
+            k = new_key(zone, bucket.name, objname)
+            k.set_contents_from_string(content)
+
+    zonegroup_meta_checkpoint(zonegroup)
+
+    for bucket_name in buckets:
+        zonegroup_bucket_checkpoint(zonegroup_conns, bucket_name)
+
+    for bucket_name in buckets:
+        disable_bucket_sync(realm.meta_master_zone(), bucket_name)
+
+    zonegroup_meta_checkpoint(zonegroup)
+
+    objnames_2 = [ 'obj5', 'obj6', 'obj7', 'obj8' ]
+
+    for zone, bucket in zone_bucket:
+        for objname in objnames_2:
+            k = new_key(zone, bucket.name, objname)
+            k.set_contents_from_string(content)
+
+    for bucket_name in buckets:
+        enable_bucket_sync(realm.meta_master_zone(), bucket_name)
+
+    for bucket_name in buckets:
+        zonegroup_bucket_checkpoint(zonegroup_conns, bucket_name)
+
+    zonegroup_data_checkpoint(zonegroup_conns)
+
+def test_multipart_object_sync():
+    zonegroup = realm.master_zonegroup()
+    zonegroup_conns = ZonegroupConns(zonegroup)
+    buckets, zone_bucket = create_bucket_per_zone(zonegroup_conns)
+
+    _, bucket = zone_bucket[0]
+
+    # initiate a multipart upload
+    upload = bucket.initiate_multipart_upload('MULTIPART')
+    mp = boto.s3.multipart.MultiPartUpload(bucket)
+    mp.key_name = upload.key_name
+    mp.id = upload.id
+    part_size = 5 * 1024 * 1024 # 5M min part size
+    mp.upload_part_from_file(StringIO('a' * part_size), 1)
+    mp.upload_part_from_file(StringIO('b' * part_size), 2)
+    mp.upload_part_from_file(StringIO('c' * part_size), 3)
+    mp.upload_part_from_file(StringIO('d' * part_size), 4)
+    mp.complete_upload()
+
+    zonegroup_bucket_checkpoint(zonegroup_conns, bucket.name)
+
+def test_encrypted_object_sync():
+    zonegroup = realm.master_zonegroup()
+    zonegroup_conns = ZonegroupConns(zonegroup)
+
+    if len(zonegroup.rw_zones) < 2:
+        raise SkipTest("test_zonegroup_remove skipped. Requires 2 or more zones in master zonegroup.")
+
+    (zone1, zone2) = zonegroup_conns.rw_zones[0:2]
+
+    # create a bucket on the first zone
+    bucket_name = gen_bucket_name()
+    log.info('create bucket zone=%s name=%s', zone1.name, bucket_name)
+    bucket = zone1.conn.create_bucket(bucket_name)
+
+    # upload an object with sse-c encryption
+    sse_c_headers = {
+        'x-amz-server-side-encryption-customer-algorithm': 'AES256',
+        'x-amz-server-side-encryption-customer-key': 'pO3upElrwuEXSoFwCfnZPdSsmt/xWeFa0N9KgDijwVs=',
+        'x-amz-server-side-encryption-customer-key-md5': 'DWygnHRtgiJ77HCm+1rvHw=='
+    }
+    key = bucket.new_key('testobj-sse-c')
+    data = 'A'*512
+    key.set_contents_from_string(data, headers=sse_c_headers)
+
+    # upload an object with sse-kms encryption
+    sse_kms_headers = {
+        'x-amz-server-side-encryption': 'aws:kms',
+        # testkey-1 must be present in 'rgw crypt s3 kms encryption keys' (vstart.sh adds this)
+        'x-amz-server-side-encryption-aws-kms-key-id': 'testkey-1',
+    }
+    key = bucket.new_key('testobj-sse-kms')
+    key.set_contents_from_string(data, headers=sse_kms_headers)
+
+    # wait for the bucket metadata and data to sync
+    zonegroup_meta_checkpoint(zonegroup)
+    zone_bucket_checkpoint(zone2.zone, zone1.zone, bucket_name)
+
+    # read the encrypted objects from the second zone
+    bucket2 = get_bucket(zone2, bucket_name)
+    key = bucket2.get_key('testobj-sse-c', headers=sse_c_headers)
+    eq(data, key.get_contents_as_string(headers=sse_c_headers, encoding='ascii'))
+
+    key = bucket2.get_key('testobj-sse-kms')
+    eq(data, key.get_contents_as_string(encoding='ascii'))
+
+def test_bucket_index_log_trim():
+    zonegroup = realm.master_zonegroup()
+    zonegroup_conns = ZonegroupConns(zonegroup)
+
+    zone = zonegroup_conns.rw_zones[0]
+
+    # create a test bucket, upload some objects, and wait for sync
+    def make_test_bucket():
+        name = gen_bucket_name()
+        log.info('create bucket zone=%s name=%s', zone.name, name)
+        bucket = zone.conn.create_bucket(name)
+        for objname in ('a', 'b', 'c', 'd'):
+            k = new_key(zone, name, objname)
+            k.set_contents_from_string('foo')
+        zonegroup_meta_checkpoint(zonegroup)
+        zonegroup_bucket_checkpoint(zonegroup_conns, name)
+        return bucket
+
+    # create a 'cold' bucket
+    cold_bucket = make_test_bucket()
+
+    # trim with max-buckets=0 to clear counters for cold bucket. this should
+    # prevent it from being considered 'active' by the next autotrim
+    bilog_autotrim(zone.zone, [
+        '--rgw-sync-log-trim-max-buckets', '0',
+    ])
+
+    # create an 'active' bucket
+    active_bucket = make_test_bucket()
+
+    # trim with max-buckets=1 min-cold-buckets=0 to trim active bucket only
+    bilog_autotrim(zone.zone, [
+        '--rgw-sync-log-trim-max-buckets', '1',
+        '--rgw-sync-log-trim-min-cold-buckets', '0',
+    ])
+
+    # verify active bucket has empty bilog
+    active_bilog = bilog_list(zone.zone, active_bucket.name)
+    assert(len(active_bilog) == 0)
+
+    # verify cold bucket has nonempty bilog
+    cold_bilog = bilog_list(zone.zone, cold_bucket.name)
+    assert(len(cold_bilog) > 0)
+
+    # trim with min-cold-buckets=999 to trim all buckets
+    bilog_autotrim(zone.zone, [
+        '--rgw-sync-log-trim-max-buckets', '999',
+        '--rgw-sync-log-trim-min-cold-buckets', '999',
+    ])
+
+    # verify cold bucket has empty bilog
+    cold_bilog = bilog_list(zone.zone, cold_bucket.name)
+    assert(len(cold_bilog) == 0)
+
+def test_bucket_creation_time():
+    zonegroup = realm.master_zonegroup()
+    zonegroup_conns = ZonegroupConns(zonegroup)
+
+    zone_buckets = [zone.get_connection().get_all_buckets() for zone in zonegroup_conns.rw_zones]
+    for z1, z2 in combinations(zone_buckets, 2):
+        for a, b in zip(z1, z2):
+            eq(a.name, b.name)
+            eq(a.creation_date, b.creation_date)
diff --git a/src/test/rgw/rgw_multi/tests_es.py b/src/test/rgw/rgw_multi/tests_es.py
new file mode 100644
index 00000000..1854c4cf
--- /dev/null
+++ b/src/test/rgw/rgw_multi/tests_es.py
@@ -0,0 +1,275 @@
+import json
+import logging
+
+import boto
+import boto.s3.connection
+
+import datetime
+import dateutil
+
+from nose.tools import eq_ as eq
+from six.moves import range
+
+from .multisite import *
+from .tests import *
+from .zone_es import *
+
+log = logging.getLogger(__name__)
+
+
+def check_es_configured():
+    realm = get_realm()
+    zonegroup = realm.master_zonegroup()
+
+    es_zones = zonegroup.zones_by_type.get("elasticsearch")
+    if not es_zones:
+        raise SkipTest("Requires at least one ES zone")
+
+def is_es_zone(zone_conn):
+    if not zone_conn:
+        return False
+
+    return zone_conn.zone.tier_type() == "elasticsearch"
+
+def verify_search(bucket_name, src_keys, result_keys, f):
+    check_keys = []
+    for k in src_keys:
+        if bucket_name:
+            if bucket_name != k.bucket.name:
+                continue
+        if f(k):
+            check_keys.append(k)
+    check_keys.sort(key = lambda l: (l.bucket.name, l.name, l.version_id))
+
+    log.debug('check keys:' + dump_json(check_keys))
+    log.debug('result keys:' + dump_json(result_keys))
+
+    for k1, k2 in zip_longest(check_keys, result_keys):
+        assert k1
+        assert k2
+        check_object_eq(k1, k2)
+
+def do_check_mdsearch(conn, bucket, src_keys, req_str, src_filter):
+    if bucket:
+        bucket_name = bucket.name
+    else:
+        bucket_name = ''
+    req = MDSearch(conn, bucket_name, req_str)
+    result_keys = req.search(sort_key = lambda k: (k.bucket.name, k.name, k.version_id))
+    verify_search(bucket_name, src_keys, result_keys, src_filter)
+
+def init_env(create_obj, num_keys = 5, buckets_per_zone = 1, bucket_init_cb = None):
+    check_es_configured()
+
+    realm = get_realm()
+    zonegroup = realm.master_zonegroup()
+    zonegroup_conns = ZonegroupConns(zonegroup)
+    buckets, zone_bucket = create_bucket_per_zone(zonegroup_conns, buckets_per_zone = buckets_per_zone)
+
+    if bucket_init_cb:
+        for zone_conn, bucket in zone_bucket:
+            bucket_init_cb(zone_conn, bucket)
+
+    src_keys = []
+
+    owner = None
+
+    obj_prefix=''.join(random.choice(string.ascii_lowercase) for _ in range(6))
+
+    # don't wait for meta sync just yet
+    for zone, bucket in zone_bucket:
+        for count in range(num_keys):
+            objname = obj_prefix + str(count)
+            k = new_key(zone, bucket.name, objname)
+            # k.set_contents_from_string(content + 'x' * count)
+            if not create_obj:
+                continue
+
+            create_obj(k, count)
+
+            if not owner:
+                for list_key in bucket.list_versions():
+                    owner = list_key.owner
+                    break
+
+            k = bucket.get_key(k.name, version_id = k.version_id)
+            k.owner = owner # owner is not set when doing get_key()
+
+            src_keys.append(k)
+
+    zonegroup_meta_checkpoint(zonegroup)
+
+    sources = []
+    targets = []
+    for target_conn in zonegroup_conns.zones:
+        if not is_es_zone(target_conn):
+            sources.append(target_conn)
+            continue
+
+        targets.append(target_conn)
+
+    buckets = []
+    # make sure all targets are synced
+    for source_conn, bucket in zone_bucket:
+        buckets.append(bucket)
+        for target_conn in targets:
+            zone_bucket_checkpoint(target_conn.zone, source_conn.zone, bucket.name)
+
+    return targets, sources, buckets, src_keys
+
+def test_es_object_search():
+    min_size = 10
+    content = 'a' * min_size
+
+    def create_obj(k, i):
+        k.set_contents_from_string(content + 'x' * i)
+
+    targets, _, buckets, src_keys = init_env(create_obj, num_keys = 5, buckets_per_zone = 2)
+
+    for target_conn in targets:
+
+        # bucket checks
+        for bucket in buckets:
+            # check name
+            do_check_mdsearch(target_conn.conn, None, src_keys , 'bucket == ' + bucket.name, lambda k: k.bucket.name == bucket.name)
+            do_check_mdsearch(target_conn.conn, bucket, src_keys , 'bucket == ' + bucket.name, lambda k: k.bucket.name == bucket.name)
+
+        # check on all buckets
+        for key in src_keys:
+            # limiting to checking specific key name, otherwise could get results from
+            # other runs / tests
+            do_check_mdsearch(target_conn.conn, None, src_keys , 'name == ' + key.name, lambda k: k.name == key.name)
+
+        # check on specific bucket
+        for bucket in buckets:
+            for key in src_keys:
+                do_check_mdsearch(target_conn.conn, bucket, src_keys , 'name < ' + key.name, lambda k: k.name < key.name)
+                do_check_mdsearch(target_conn.conn, bucket, src_keys , 'name <= ' + key.name, lambda k: k.name <= key.name)
+                do_check_mdsearch(target_conn.conn, bucket, src_keys , 'name == ' + key.name, lambda k: k.name == key.name)
+                do_check_mdsearch(target_conn.conn, bucket, src_keys , 'name >= ' + key.name, lambda k: k.name >= key.name)
+                do_check_mdsearch(target_conn.conn, bucket, src_keys , 'name > ' + key.name, lambda k: k.name > key.name)
+
+            do_check_mdsearch(target_conn.conn, bucket, src_keys , 'name == ' + src_keys[0].name + ' or name >= ' + src_keys[2].name,
+                              lambda k: k.name == src_keys[0].name or k.name >= src_keys[2].name)
+
+            # check etag
+            for key in src_keys:
+                do_check_mdsearch(target_conn.conn, bucket, src_keys , 'etag < ' + key.etag[1:-1], lambda k: k.etag < key.etag)
+            for key in src_keys:
+                do_check_mdsearch(target_conn.conn, bucket, src_keys , 'etag == ' + key.etag[1:-1], lambda k: k.etag == key.etag)
+            for key in src_keys:
+                do_check_mdsearch(target_conn.conn, bucket, src_keys , 'etag > ' + key.etag[1:-1], lambda k: k.etag > key.etag)
+
+            # check size
+            for key in src_keys:
+                do_check_mdsearch(target_conn.conn, bucket, src_keys , 'size < ' + str(key.size), lambda k: k.size < key.size)
+            for key in src_keys:
+                do_check_mdsearch(target_conn.conn, bucket, src_keys , 'size <= ' + str(key.size), lambda k: k.size <= key.size)
+            for key in src_keys:
+                do_check_mdsearch(target_conn.conn, bucket, src_keys , 'size == ' + str(key.size), lambda k: k.size == key.size)
+            for key in src_keys:
+                do_check_mdsearch(target_conn.conn, bucket, src_keys , 'size >= ' + str(key.size), lambda k: k.size >= key.size)
+            for key in src_keys:
+                do_check_mdsearch(target_conn.conn, bucket, src_keys , 'size > ' + str(key.size), lambda k: k.size > key.size)
+
+def date_from_str(s):
+    return dateutil.parser.parse(s)
+
+def test_es_object_search_custom():
+    min_size = 10
+    content = 'a' * min_size
+
+    def bucket_init(zone_conn, bucket):
+        req = MDSearchConfig(zone_conn.conn, bucket.name)
+        req.set_config('x-amz-meta-foo-str; string, x-amz-meta-foo-int; int, x-amz-meta-foo-date; date')
+
+    def create_obj(k, i):
+        date = datetime.datetime.now() + datetime.timedelta(seconds=1) * i
+        date_str = date.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z'
+        k.set_contents_from_string(content + 'x' * i, headers = { 'X-Amz-Meta-Foo-Str': str(i * 5),
+                                                                  'X-Amz-Meta-Foo-Int': str(i * 5),
+                                                                  'X-Amz-Meta-Foo-Date': date_str})
+
+    targets, _, buckets, src_keys = init_env(create_obj, num_keys = 5, buckets_per_zone = 1, bucket_init_cb = bucket_init)
+
+
+    for target_conn in targets:
+
+        # bucket checks
+        for bucket in buckets:
+            str_vals = []
+            for key in src_keys:
+                # check string values
+                val = key.get_metadata('foo-str')
+                str_vals.append(val)
+                do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-str < ' + val, lambda k: k.get_metadata('foo-str') < val)
+                do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-str <= ' + val, lambda k: k.get_metadata('foo-str') <= val)
+                do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-str == ' + val, lambda k: k.get_metadata('foo-str') == val)
+                do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-str >= ' + val, lambda k: k.get_metadata('foo-str') >= val)
+                do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-str > ' + val, lambda k: k.get_metadata('foo-str') > val)
+
+                # check int values
+                sval = key.get_metadata('foo-int')
+                val = int(sval)
+                do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-int < ' + sval, lambda k: int(k.get_metadata('foo-int')) < val)
+                do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-int <= ' + sval, lambda k: int(k.get_metadata('foo-int')) <= val)
+                do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-int == ' + sval, lambda k: int(k.get_metadata('foo-int')) == val)
+                do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-int >= ' + sval, lambda k: int(k.get_metadata('foo-int')) >= val)
+                do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-int > ' + sval, lambda k: int(k.get_metadata('foo-int')) > val)
+
+                # check int values
+                sval = key.get_metadata('foo-date')
+                val = date_from_str(sval)
+                do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-date < ' + sval, lambda k: date_from_str(k.get_metadata('foo-date')) < val)
+                do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-date <= ' + sval, lambda k: date_from_str(k.get_metadata('foo-date')) <= val)
+                do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-date == ' + sval, lambda k: date_from_str(k.get_metadata('foo-date')) == val)
+                do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-date >= ' + sval, lambda k: date_from_str(k.get_metadata('foo-date')) >= val)
+                do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-date > ' + sval, lambda k: date_from_str(k.get_metadata('foo-date')) > val)
+
+            # 'or' query
+            for i in range(len(src_keys) // 2):
+                do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-str <= ' + str_vals[i] + ' or x-amz-meta-foo-str >= ' + str_vals[-i],
+                        lambda k: k.get_metadata('foo-str') <= str_vals[i] or k.get_metadata('foo-str') >= str_vals[-i] )
+
+            # 'and' query
+            for i in range(len(src_keys) // 2):
+                do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-str >= ' + str_vals[i] + ' and x-amz-meta-foo-str <= ' + str_vals[i + 1],
+                        lambda k: k.get_metadata('foo-str') >= str_vals[i] and k.get_metadata('foo-str') <= str_vals[i + 1] )
+            # more complicated query
+            for i in range(len(src_keys) // 2):
+                do_check_mdsearch(target_conn.conn, None, src_keys , 'bucket == ' + bucket.name + ' and x-amz-meta-foo-str >= ' + str_vals[i] +
+                                                                     ' and (x-amz-meta-foo-str <= ' + str_vals[i + 1] + ')',
+                        lambda k: k.bucket.name == bucket.name and (k.get_metadata('foo-str') >= str_vals[i] and
+                                                                    k.get_metadata('foo-str') <= str_vals[i + 1]) )
+
+def test_es_bucket_conf():
+    min_size = 0
+
+    def bucket_init(zone_conn, bucket):
+        req = MDSearchConfig(zone_conn.conn, bucket.name)
+        req.set_config('x-amz-meta-foo-str; string, x-amz-meta-foo-int; int, x-amz-meta-foo-date; date')
+
+    targets, sources, buckets, _ = init_env(None, num_keys = 5, buckets_per_zone = 1, bucket_init_cb = bucket_init)
+
+    for source_conn in sources:
+        for bucket in buckets:
+            req = MDSearchConfig(source_conn.conn, bucket.name)
+            conf = req.get_config()
+
+            d = {}
+
+            for entry in conf:
+              d[entry['Key']] = entry['Type']
+
+            eq(len(d), 3)
+            eq(d['x-amz-meta-foo-str'], 'str')
+            eq(d['x-amz-meta-foo-int'], 'int')
+            eq(d['x-amz-meta-foo-date'], 'date')
+
+            req.del_config()
+
+            conf = req.get_config()
+
+            eq(len(conf), 0)
+
+        break # no need to iterate over all zones
diff --git a/src/test/rgw/rgw_multi/tests_ps.py b/src/test/rgw/rgw_multi/tests_ps.py
new file mode 100644
index 00000000..38e786d9
--- /dev/null
+++ b/src/test/rgw/rgw_multi/tests_ps.py
@@ -0,0 +1,3845 @@
+import logging
+import json
+import tempfile
+from six.moves import BaseHTTPServer
+import random
+import threading
+import subprocess
+import socket
+import time
+import os
+from random import randint
+from .tests import get_realm, \
+    ZonegroupConns, \
+    zonegroup_meta_checkpoint, \
+    zone_meta_checkpoint, \
+    zone_bucket_checkpoint, \
+    zone_data_checkpoint, \
+    zonegroup_bucket_checkpoint, \
+    check_bucket_eq, \
+    gen_bucket_name, \
+    get_user, \
+    get_tenant
+from .zone_ps import PSTopic, \
+    PSTopicS3, \
+    PSNotification, \
+    PSSubscription, \
+    PSNotificationS3, \
+    print_connection_info, \
+    delete_all_s3_topics, \
+    put_object_tagging, \
+    get_object_tagging, \
+    delete_all_objects, \
+    delete_all_s3_topics
+from .multisite import User
+from nose import SkipTest
+from nose.tools import assert_not_equal, assert_equal
+import boto.s3.tagging
+
+# configure logging for the tests module
+log = logging.getLogger(__name__)
+
+skip_push_tests = True
+
+####################################
+# utility functions for pubsub tests
+####################################
+
+def set_contents_from_string(key, content):
+    try:
+        key.set_contents_from_string(content)
+    except Exception as e:
+        print('Error: ' + str(e))
+
+
+# HTTP endpoint functions
+# multithreaded streaming server, based on: https://stackoverflow.com/questions/46210672/
+
+class HTTPPostHandler(BaseHTTPServer.BaseHTTPRequestHandler):
+    """HTTP POST hanler class storing the received events in its http server"""
+    def do_POST(self):
+        """implementation of POST handler"""
+        try:
+            content_length = int(self.headers['Content-Length'])
+            body = self.rfile.read(content_length)
+            log.info('HTTP Server (%d) received event: %s', self.server.worker_id, str(body))
+            self.server.append(json.loads(body))
+        except:
+            log.error('HTTP Server received empty event')
+            self.send_response(400)
+        else:
+            self.send_response(100)
+        finally:
+            self.end_headers()
+
+
+class HTTPServerWithEvents(BaseHTTPServer.HTTPServer):
+    """HTTP server used by the handler to store events"""
+    def __init__(self, addr, handler, worker_id):
+        BaseHTTPServer.HTTPServer.__init__(self, addr, handler, False)
+        self.worker_id = worker_id
+        self.events = []
+
+    def append(self, event):
+        self.events.append(event)
+
+
+class HTTPServerThread(threading.Thread):
+    """thread for running the HTTP server. reusing the same socket for all threads"""
+    def __init__(self, i, sock, addr):
+        threading.Thread.__init__(self)
+        self.i = i
+        self.daemon = True
+        self.httpd = HTTPServerWithEvents(addr, HTTPPostHandler, i)
+        self.httpd.socket = sock
+        # prevent the HTTP server from re-binding every handler
+        self.httpd.server_bind = self.server_close = lambda self: None
+        self.start()
+
+    def run(self):
+        try:
+            log.info('HTTP Server (%d) started on: %s', self.i, self.httpd.server_address)
+            self.httpd.serve_forever()
+            log.info('HTTP Server (%d) ended', self.i)
+        except Exception as error:
+            # could happen if the server r/w to a closing socket during shutdown
+            log.info('HTTP Server (%d) ended unexpectedly: %s', self.i, str(error))
+
+    def close(self):
+        self.httpd.shutdown()
+
+    def get_events(self):
+        return self.httpd.events
+
+    def reset_events(self):
+        self.httpd.events = []
+
+
+class StreamingHTTPServer:
+    """multi-threaded http server class also holding list of events received into the handler
+    each thread has its own server, and all servers share the same socket"""
+    def __init__(self, host, port, num_workers=100):
+        addr = (host, port)
+        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        self.sock.bind(addr)
+        self.sock.listen(num_workers)
+        self.workers = [HTTPServerThread(i, self.sock, addr) for i in range(num_workers)]
+
+    def verify_s3_events(self, keys, exact_match=False, deletions=False):
+        """verify stored s3 records agains a list of keys"""
+        events = []
+        for worker in self.workers:
+            events += worker.get_events()
+            worker.reset_events()
+        verify_s3_records_by_elements(events, keys, exact_match=exact_match, deletions=deletions)
+
+    def verify_events(self, keys, exact_match=False, deletions=False):
+        """verify stored events agains a list of keys"""
+        events = []
+        for worker in self.workers:
+            events += worker.get_events()
+            worker.reset_events()
+        verify_events_by_elements(events, keys, exact_match=exact_match, deletions=deletions)
+
+    def get_and_reset_events(self):
+        events = []
+        for worker in self.workers:
+            events += worker.get_events()
+            worker.reset_events()
+        return events
+
+    def close(self):
+        """close all workers in the http server and wait for it to finish"""
+        # make sure that the shared socket is closed
+        # this is needed in case that one of the threads is blocked on the socket
+        self.sock.shutdown(socket.SHUT_RDWR)
+        self.sock.close()
+        # wait for server threads to finish
+        for worker in self.workers:
+            worker.close()
+            worker.join()
+    
+    def get_and_reset_events(self):
+        events = []
+        for worker in self.workers:
+            events += worker.get_events()
+            worker.reset_events()
+        return events
+
+
+# AMQP endpoint functions
+
+rabbitmq_port = 5672
+
+class AMQPReceiver(object):
+    """class for receiving and storing messages on a topic from the AMQP broker"""
+    def __init__(self, exchange, topic):
+        import pika
+        hostname = get_ip()
+        remaining_retries = 10
+        while remaining_retries > 0:
+            try:
+                connection = pika.BlockingConnection(pika.ConnectionParameters(host=hostname, port=rabbitmq_port))
+                break
+            except Exception as error:
+                remaining_retries -= 1
+                print('failed to connect to rabbitmq (remaining retries '
+                    + str(remaining_retries) + '): ' + str(error))
+                time.sleep(0.5)
+
+        if remaining_retries == 0:
+            raise Exception('failed to connect to rabbitmq - no retries left')
+
+        self.channel = connection.channel()
+        self.channel.exchange_declare(exchange=exchange, exchange_type='topic', durable=True)
+        result = self.channel.queue_declare('', exclusive=True)
+        queue_name = result.method.queue
+        self.channel.queue_bind(exchange=exchange, queue=queue_name, routing_key=topic)
+        self.channel.basic_consume(queue=queue_name,
+                                   on_message_callback=self.on_message,
+                                   auto_ack=True)
+        self.events = []
+        self.topic = topic
+
+    def on_message(self, ch, method, properties, body):
+        """callback invoked when a new message arrive on the topic"""
+        log.info('AMQP received event for topic %s:\n %s', self.topic, body)
+        self.events.append(json.loads(body))
+
+    # TODO create a base class for the AMQP and HTTP cases
+    def verify_s3_events(self, keys, exact_match=False, deletions=False):
+        """verify stored s3 records agains a list of keys"""
+        verify_s3_records_by_elements(self.events, keys, exact_match=exact_match, deletions=deletions)
+        self.events = []
+
+    def verify_events(self, keys, exact_match=False, deletions=False):
+        """verify stored events agains a list of keys"""
+        verify_events_by_elements(self.events, keys, exact_match=exact_match, deletions=deletions)
+        self.events = []
+
+    def get_and_reset_events(self):
+        tmp = self.events
+        self.events = []
+        return tmp
+
+
+def amqp_receiver_thread_runner(receiver):
+    """main thread function for the amqp receiver"""
+    try:
+        log.info('AMQP receiver started')
+        receiver.channel.start_consuming()
+        log.info('AMQP receiver ended')
+    except Exception as error:
+        log.info('AMQP receiver ended unexpectedly: %s', str(error))
+
+
+def create_amqp_receiver_thread(exchange, topic):
+    """create amqp receiver and thread"""
+    receiver = AMQPReceiver(exchange, topic)
+    task = threading.Thread(target=amqp_receiver_thread_runner, args=(receiver,))
+    task.daemon = True
+    return task, receiver
+
+
+def stop_amqp_receiver(receiver, task):
+    """stop the receiver thread and wait for it to finis"""
+    try:
+        receiver.channel.stop_consuming()
+        log.info('stopping AMQP receiver')
+    except Exception as error:
+        log.info('failed to gracefuly stop AMQP receiver: %s', str(error))
+    task.join(5)
+
+def check_ps_configured():
+    """check if at least one pubsub zone exist"""
+    realm = get_realm()
+    zonegroup = realm.master_zonegroup()
+
+    ps_zones = zonegroup.zones_by_type.get("pubsub")
+    if not ps_zones:
+        raise SkipTest("Requires at least one PS zone")
+
+
+def is_ps_zone(zone_conn):
+    """check if a specific zone is pubsub zone"""
+    if not zone_conn:
+        return False
+    return zone_conn.zone.tier_type() == "pubsub"
+
+
+def verify_events_by_elements(events, keys, exact_match=False, deletions=False):
+    """ verify there is at least one event per element """
+    err = ''
+    for key in keys:
+        key_found = False
+        if type(events) is list:
+            for event_list in events: 
+                if key_found:
+                    break
+                for event in event_list['events']:
+                    if event['info']['bucket']['name'] == key.bucket.name and \
+                        event['info']['key']['name'] == key.name:
+                        if deletions and event['event'] == 'OBJECT_DELETE':
+                            key_found = True
+                            break
+                        elif not deletions and event['event'] == 'OBJECT_CREATE':
+                            key_found = True
+                            break
+        else:
+            for event in events['events']:
+                if event['info']['bucket']['name'] == key.bucket.name and \
+                    event['info']['key']['name'] == key.name:
+                    if deletions and event['event'] == 'OBJECT_DELETE':
+                        key_found = True
+                        break
+                    elif not deletions and event['event'] == 'OBJECT_CREATE':
+                        key_found = True
+                        break
+
+        if not key_found:
+            err = 'no ' + ('deletion' if deletions else 'creation') + ' event found for key: ' + str(key)
+            log.error(events)
+            assert False, err
+
+    if not len(events) == len(keys):
+        err = 'superfluous events are found'
+        log.debug(err)
+        if exact_match:
+            log.error(events)
+            assert False, err
+
+
+def verify_s3_records_by_elements(records, keys, exact_match=False, deletions=False):
+    """ verify there is at least one record per element """
+    err = ''
+    for key in keys:
+        key_found = False
+        if type(records) is list:
+            for record_list in records:
+                if key_found:
+                    break
+                for record in record_list['Records']:
+                    if record['s3']['bucket']['name'] == key.bucket.name and \
+                        record['s3']['object']['key'] == key.name:
+                        if deletions and 'ObjectRemoved' in record['eventName']:
+                            key_found = True
+                            break
+                        elif not deletions and 'ObjectCreated' in record['eventName']:
+                            key_found = True
+                            break
+        else:
+            for record in records['Records']:
+                if record['s3']['bucket']['name'] == key.bucket.name and \
+                    record['s3']['object']['key'] == key.name:
+                    if deletions and 'ObjectRemoved' in record['eventName']:
+                        key_found = True
+                        break
+                    elif not deletions and 'ObjectCreated' in record['eventName']:
+                        key_found = True
+                        break
+
+        if not key_found:
+            err = 'no ' + ('deletion' if deletions else 'creation') + ' event found for key: ' + str(key)
+            for record_list in records:
+                for record in record_list['Records']:
+                    log.error(str(record['s3']['bucket']['name']) + ',' + str(record['s3']['object']['key']))
+            assert False, err
+
+    if not len(records) == len(keys):
+        err = 'superfluous records are found'
+        log.warning(err)
+        if exact_match:
+            for record_list in records:
+                for record in record_list['Records']:
+                    log.error(str(record['s3']['bucket']['name']) + ',' + str(record['s3']['object']['key']))
+            assert False, err
+
+
+def init_rabbitmq():
+    """ start a rabbitmq broker """
+    hostname = get_ip()
+    #port = str(random.randint(20000, 30000))
+    #data_dir = './' + port + '_data'
+    #log_dir = './' + port + '_log'
+    #print('')
+    #try:
+    #    os.mkdir(data_dir)
+    #    os.mkdir(log_dir)
+    #except:
+    #    print('rabbitmq directories already exists')
+    #env = {'RABBITMQ_NODE_PORT': port,
+    #       'RABBITMQ_NODENAME': 'rabbit'+ port + '@' + hostname,
+    #       'RABBITMQ_USE_LONGNAME': 'true',
+    #       'RABBITMQ_MNESIA_BASE': data_dir,
+    #       'RABBITMQ_LOG_BASE': log_dir}
+    # TODO: support multiple brokers per host using env
+    # make sure we don't collide with the default
+    try:
+        proc = subprocess.Popen('rabbitmq-server')
+    except Exception as error:
+        log.info('failed to execute rabbitmq-server: %s', str(error))
+        print('failed to execute rabbitmq-server: %s' % str(error))
+        return None
+    # TODO add rabbitmq checkpoint instead of sleep
+    time.sleep(5)
+    return proc #, port, data_dir, log_dir
+
+
+def clean_rabbitmq(proc): #, data_dir, log_dir)
+    """ stop the rabbitmq broker """
+    try:
+        subprocess.call(['rabbitmqctl', 'stop'])
+        time.sleep(5)
+        proc.terminate()
+    except:
+        log.info('rabbitmq server already terminated')
+    # TODO: add directory cleanup once multiple brokers are supported
+    #try:
+    #    os.rmdir(data_dir)
+    #    os.rmdir(log_dir)
+    #except:
+    #    log.info('rabbitmq directories already removed')
+
+
+# Kafka endpoint functions
+
+kafka_server = 'localhost'
+
+class KafkaReceiver(object):
+    """class for receiving and storing messages on a topic from the kafka broker"""
+    def __init__(self, topic, security_type):
+        from kafka import KafkaConsumer
+        remaining_retries = 10
+        port = 9092
+        if security_type != 'PLAINTEXT':
+            security_type = 'SSL'
+            port = 9093
+        while remaining_retries > 0:
+            try:
+                self.consumer = KafkaConsumer(topic, bootstrap_servers = kafka_server+':'+str(port), security_protocol=security_type)
+                print('Kafka consumer created on topic: '+topic)
+                break
+            except Exception as error:
+                remaining_retries -= 1
+                print('failed to connect to kafka (remaining retries '
+                    + str(remaining_retries) + '): ' + str(error))
+                time.sleep(1)
+
+        if remaining_retries == 0:
+            raise Exception('failed to connect to kafka - no retries left')
+
+        self.events = []
+        self.topic = topic
+        self.stop = False
+
+    def verify_s3_events(self, keys, exact_match=False, deletions=False):
+        """verify stored s3 records agains a list of keys"""
+        verify_s3_records_by_elements(self.events, keys, exact_match=exact_match, deletions=deletions)
+        self.events = []
+
+
+def kafka_receiver_thread_runner(receiver):
+    """main thread function for the kafka receiver"""
+    try:
+        log.info('Kafka receiver started')
+        print('Kafka receiver started')
+        while not receiver.stop:
+            for msg in receiver.consumer:
+                receiver.events.append(json.loads(msg.value))
+            timer.sleep(0.1)
+        log.info('Kafka receiver ended')
+        print('Kafka receiver ended')
+    except Exception as error:
+        log.info('Kafka receiver ended unexpectedly: %s', str(error))
+        print('Kafka receiver ended unexpectedly: ' + str(error))
+
+
+def create_kafka_receiver_thread(topic, security_type='PLAINTEXT'):
+    """create kafka receiver and thread"""
+    receiver = KafkaReceiver(topic, security_type)
+    task = threading.Thread(target=kafka_receiver_thread_runner, args=(receiver,))
+    task.daemon = True
+    return task, receiver
+
+def stop_kafka_receiver(receiver, task):
+    """stop the receiver thread and wait for it to finis"""
+    receiver.stop = True
+    task.join(1)
+    try:
+        receiver.consumer.close()
+    except Exception as error:
+        log.info('failed to gracefuly stop Kafka receiver: %s', str(error))
+
+
+# follow the instruction here to create and sign a broker certificate:
+# https://github.com/edenhill/librdkafka/wiki/Using-SSL-with-librdkafka
+
+# the generated broker certificate should be stored in the java keystore for the use of the server
+# assuming the jks files were copied to $KAFKA_DIR and broker name is "localhost"
+# following lines must be added to $KAFKA_DIR/config/server.properties
+# listeners=PLAINTEXT://localhost:9092,SSL://localhost:9093,SASL_SSL://localhost:9094
+# sasl.enabled.mechanisms=PLAIN
+# ssl.keystore.location = $KAFKA_DIR/server.keystore.jks
+# ssl.keystore.password = abcdefgh
+# ssl.key.password = abcdefgh
+# ssl.truststore.location = $KAFKA_DIR/server.truststore.jks
+# ssl.truststore.password = abcdefgh
+
+# notes:
+# (1) we dont test client authentication, hence, no need to generate client keys
+# (2) our client is not using the keystore, and the "rootCA.crt" file generated in the process above
+# should be copied to: $KAFKA_DIR
+
+def init_kafka():
+    """ start kafka/zookeeper """
+    try:
+        KAFKA_DIR = os.environ['KAFKA_DIR']
+    except:
+        KAFKA_DIR = ''
+
+    if KAFKA_DIR == '':
+        log.info('KAFKA_DIR must be set to where kafka is installed')
+        print('KAFKA_DIR must be set to where kafka is installed')
+        return None, None, None
+    
+    DEVNULL = open(os.devnull, 'wb')
+
+    print('\nStarting zookeeper...')
+    try:
+        zk_proc = subprocess.Popen([KAFKA_DIR+'bin/zookeeper-server-start.sh', KAFKA_DIR+'config/zookeeper.properties'], stdout=DEVNULL)
+    except Exception as error:
+        log.info('failed to execute zookeeper: %s', str(error))
+        print('failed to execute zookeeper: %s' % str(error))
+        return None, None, None
+
+    time.sleep(5)
+    if zk_proc.poll() is not None:
+        print('zookeeper failed to start')
+        return None, None, None
+    print('Zookeeper started')
+    print('Starting kafka...')
+    kafka_log = open('./kafka.log', 'w')
+    try:
+        kafka_env = os.environ.copy()
+        kafka_env['KAFKA_OPTS']='-Djava.security.auth.login.config='+KAFKA_DIR+'config/kafka_server_jaas.conf'
+        kafka_proc = subprocess.Popen([
+            KAFKA_DIR+'bin/kafka-server-start.sh', 
+            KAFKA_DIR+'config/server.properties'], 
+            stdout=kafka_log,
+            env=kafka_env)
+    except Exception as error:
+        log.info('failed to execute kafka: %s', str(error))
+        print('failed to execute kafka: %s' % str(error))
+        zk_proc.terminate()
+        kafka_log.close()
+        return None, None, None
+
+    # TODO add kafka checkpoint instead of sleep
+    time.sleep(15)
+    if kafka_proc.poll() is not None:
+        zk_proc.terminate()
+        print('kafka failed to start. details in: ./kafka.log')
+        kafka_log.close()
+        return None, None, None
+
+    print('Kafka started')
+    return kafka_proc, zk_proc, kafka_log
+
+
+def clean_kafka(kafka_proc, zk_proc, kafka_log):
+    """ stop kafka/zookeeper """
+    try:
+        kafka_log.close()
+        print('Shutdown Kafka...')
+        kafka_proc.terminate()
+        time.sleep(5)
+        if kafka_proc.poll() is None:
+            print('Failed to shutdown Kafka... killing')
+            kafka_proc.kill()
+        print('Shutdown zookeeper...')
+        zk_proc.terminate()
+        time.sleep(5)
+        if zk_proc.poll() is None:
+            print('Failed to shutdown zookeeper... killing')
+            zk_proc.kill()
+    except:
+        log.info('kafka/zookeeper already terminated')
+
+
+def init_env(require_ps=True):
+    """initialize the environment"""
+    if require_ps:
+        check_ps_configured()
+
+    realm = get_realm()
+    zonegroup = realm.master_zonegroup()
+    zonegroup_conns = ZonegroupConns(zonegroup)
+
+    zonegroup_meta_checkpoint(zonegroup)
+
+    ps_zones = []
+    zones = []
+    for conn in zonegroup_conns.zones:
+        if is_ps_zone(conn):
+            zone_meta_checkpoint(conn.zone)
+            ps_zones.append(conn)
+        elif not conn.zone.is_read_only():
+            zones.append(conn)
+
+    assert_not_equal(len(zones), 0)
+    if require_ps:
+        assert_not_equal(len(ps_zones), 0)
+    return zones, ps_zones
+
+
+def get_ip():
+    """ This method returns the "primary" IP on the local box (the one with a default route)
+    source: https://stackoverflow.com/a/28950776/711085
+    this is needed because on the teuthology machines: socket.getfqdn()/socket.gethostname() return 127.0.0.1 """
+    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    try:
+        # address should not be reachable
+        s.connect(('10.255.255.255', 1))
+        ip = s.getsockname()[0]
+    finally:
+        s.close()
+    return ip
+
+
+TOPIC_SUFFIX = "_topic"
+SUB_SUFFIX = "_sub"
+NOTIFICATION_SUFFIX = "_notif"
+
+##############
+# pubsub tests
+##############
+
+def test_ps_info():
+    """ log information for manual testing """
+    return SkipTest("only used in manual testing")
+    zones, ps_zones = init_env()
+    realm = get_realm()
+    zonegroup = realm.master_zonegroup()
+    bucket_name = gen_bucket_name()
+    # create bucket on the first of the rados zones
+    bucket = zones[0].create_bucket(bucket_name)
+    # create objects in the bucket
+    number_of_objects = 10
+    for i in range(number_of_objects):
+        key = bucket.new_key(str(i))
+        key.set_contents_from_string('bar')
+    print('Zonegroup: ' + zonegroup.name)
+    print('user: ' + get_user())
+    print('tenant: ' + get_tenant())
+    print('Master Zone')
+    print_connection_info(zones[0].conn)
+    print('PubSub Zone')
+    print_connection_info(ps_zones[0].conn)
+    print('Bucket: ' + bucket_name)
+
+
+def test_ps_s3_notification_low_level():
+    """ test low level implementation of s3 notifications """
+    zones, ps_zones = init_env()
+    bucket_name = gen_bucket_name()
+    # create bucket on the first of the rados zones
+    zones[0].create_bucket(bucket_name)
+    # wait for sync
+    zone_meta_checkpoint(ps_zones[0].zone)
+    # create topic
+    topic_name = bucket_name + TOPIC_SUFFIX
+    topic_conf = PSTopic(ps_zones[0].conn, topic_name)
+    result, status = topic_conf.set_config()
+    assert_equal(status/100, 2)
+    parsed_result = json.loads(result)
+    topic_arn = parsed_result['arn']
+    # create s3 notification
+    notification_name = bucket_name + NOTIFICATION_SUFFIX
+    generated_topic_name = notification_name+'_'+topic_name
+    topic_conf_list = [{'Id': notification_name,
+                        'TopicArn': topic_arn,
+                        'Events': ['s3:ObjectCreated:*']
+                       }]
+    s3_notification_conf = PSNotificationS3(ps_zones[0].conn, bucket_name, topic_conf_list)
+    _, status = s3_notification_conf.set_config()
+    assert_equal(status/100, 2)
+    zone_meta_checkpoint(ps_zones[0].zone)
+    # get auto-generated topic
+    generated_topic_conf = PSTopic(ps_zones[0].conn, generated_topic_name)
+    result, status = generated_topic_conf.get_config()
+    parsed_result = json.loads(result)
+    assert_equal(status/100, 2)
+    assert_equal(parsed_result['topic']['name'], generated_topic_name)
+    # get auto-generated notification
+    notification_conf = PSNotification(ps_zones[0].conn, bucket_name,
+                                       generated_topic_name)
+    result, status = notification_conf.get_config()
+    parsed_result = json.loads(result)
+    assert_equal(status/100, 2)
+    assert_equal(len(parsed_result['topics']), 1)
+    # get auto-generated subscription
+    sub_conf = PSSubscription(ps_zones[0].conn, notification_name,
+                              generated_topic_name)
+    result, status = sub_conf.get_config()
+    parsed_result = json.loads(result)
+    assert_equal(status/100, 2)
+    assert_equal(parsed_result['topic'], generated_topic_name)
+    # delete s3 notification
+    _, status = s3_notification_conf.del_config(notification=notification_name)
+    assert_equal(status/100, 2)
+    # delete topic
+    _, status = topic_conf.del_config()
+    assert_equal(status/100, 2)
+
+    # verify low-level cleanup
+    _, status = generated_topic_conf.get_config()
+    assert_equal(status, 404)
+    result, status = notification_conf.get_config()
+    parsed_result = json.loads(result)
+    assert_equal(len(parsed_result['topics']), 0)
+    # TODO should return 404
+    # assert_equal(status, 404)
+    result, status = sub_conf.get_config()
+    parsed_result = json.loads(result)
+    assert_equal(parsed_result['topic'], '')
+    # TODO should return 404
+    # assert_equal(status, 404)
+
+    # cleanup
+    topic_conf.del_config()
+    # delete the bucket
+    zones[0].delete_bucket(bucket_name)
+
+
+def test_ps_s3_notification_records():
+    """ test s3 records fetching """
+    zones, ps_zones = init_env()
+    bucket_name = gen_bucket_name()
+    # create bucket on the first of the rados zones
+    bucket = zones[0].create_bucket(bucket_name)
+    # wait for sync
+    zone_meta_checkpoint(ps_zones[0].zone)
+    # create topic
+    topic_name = bucket_name + TOPIC_SUFFIX
+    topic_conf = PSTopic(ps_zones[0].conn, topic_name)
+    result, status = topic_conf.set_config()
+    assert_equal(status/100, 2)
+    parsed_result = json.loads(result)
+    topic_arn = parsed_result['arn']
+    # create s3 notification
+    notification_name = bucket_name + NOTIFICATION_SUFFIX
+    topic_conf_list = [{'Id': notification_name,
+                        'TopicArn': topic_arn,
+                        'Events': ['s3:ObjectCreated:*']
+                       }]
+    s3_notification_conf = PSNotificationS3(ps_zones[0].conn, bucket_name, topic_conf_list)
+    _, status = s3_notification_conf.set_config()
+    assert_equal(status/100, 2)
+    zone_meta_checkpoint(ps_zones[0].zone)
+    # get auto-generated subscription
+    sub_conf = PSSubscription(ps_zones[0].conn, notification_name,
+                              topic_name)
+    _, status = sub_conf.get_config()
+    assert_equal(status/100, 2)
+    # create objects in the bucket
+    number_of_objects = 10
+    for i in range(number_of_objects):
+        key = bucket.new_key(str(i))
+        key.set_contents_from_string('bar')
+    # wait for sync
+    zone_bucket_checkpoint(ps_zones[0].zone, zones[0].zone, bucket_name)
+
+    # get the events from the subscription
+    result, _ = sub_conf.get_events()
+    records = json.loads(result)
+    for record in records['Records']:
+        log.debug(record)
+    keys = list(bucket.list())
+    # TODO: use exact match
+    verify_s3_records_by_elements(records, keys, exact_match=False)
+
+    # cleanup
+    _, status = s3_notification_conf.del_config()
+    topic_conf.del_config()
+    # delete the keys
+    for key in bucket.list():
+        key.delete()
+    zones[0].delete_bucket(bucket_name)
+
+
+def test_ps_s3_notification():
+    """ test s3 notification set/get/delete """
+    zones, ps_zones = init_env()
+    bucket_name = gen_bucket_name()
+    # create bucket on the first of the rados zones
+    zones[0].create_bucket(bucket_name)
+    # wait for sync
+    zone_meta_checkpoint(ps_zones[0].zone)
+    topic_name = bucket_name + TOPIC_SUFFIX
+    # create topic
+    topic_name = bucket_name + TOPIC_SUFFIX
+    topic_conf = PSTopic(ps_zones[0].conn, topic_name)
+    response, status = topic_conf.set_config()
+    assert_equal(status/100, 2)
+    parsed_result = json.loads(response)
+    topic_arn = parsed_result['arn']
+    # create one s3 notification
+    notification_name1 = bucket_name + NOTIFICATION_SUFFIX + '_1'
+    topic_conf_list = [{'Id': notification_name1,
+                        'TopicArn': topic_arn,
+                        'Events': ['s3:ObjectCreated:*']
+                       }]
+    s3_notification_conf1 = PSNotificationS3(ps_zones[0].conn, bucket_name, topic_conf_list)
+    response, status = s3_notification_conf1.set_config()
+    assert_equal(status/100, 2)
+    # create another s3 notification with the same topic
+    notification_name2 = bucket_name + NOTIFICATION_SUFFIX + '_2'
+    topic_conf_list = [{'Id': notification_name2,
+                        'TopicArn': topic_arn,
+                        'Events': ['s3:ObjectCreated:*', 's3:ObjectRemoved:*']
+                       }]
+    s3_notification_conf2 = PSNotificationS3(ps_zones[0].conn, bucket_name, topic_conf_list)
+    response, status = s3_notification_conf2.set_config()
+    assert_equal(status/100, 2)
+    zone_meta_checkpoint(ps_zones[0].zone)
+
+    # get all notification on a bucket
+    response, status = s3_notification_conf1.get_config()
+    assert_equal(status/100, 2)
+    assert_equal(len(response['TopicConfigurations']), 2)
+    assert_equal(response['TopicConfigurations'][0]['TopicArn'], topic_arn)
+    assert_equal(response['TopicConfigurations'][1]['TopicArn'], topic_arn)
+
+    # get specific notification on a bucket
+    response, status = s3_notification_conf1.get_config(notification=notification_name1)
+    assert_equal(status/100, 2)
+    assert_equal(response['NotificationConfiguration']['TopicConfiguration']['Topic'], topic_arn)
+    assert_equal(response['NotificationConfiguration']['TopicConfiguration']['Id'], notification_name1)
+    response, status = s3_notification_conf2.get_config(notification=notification_name2)
+    assert_equal(status/100, 2)
+    assert_equal(response['NotificationConfiguration']['TopicConfiguration']['Topic'], topic_arn)
+    assert_equal(response['NotificationConfiguration']['TopicConfiguration']['Id'], notification_name2)
+
+    # delete specific notifications
+    _, status = s3_notification_conf1.del_config(notification=notification_name1)
+    assert_equal(status/100, 2)
+    _, status = s3_notification_conf2.del_config(notification=notification_name2)
+    assert_equal(status/100, 2)
+
+    # cleanup
+    topic_conf.del_config()
+    # delete the bucket
+    zones[0].delete_bucket(bucket_name)
+
+
+def test_ps_s3_topic_on_master():
+    """ test s3 topics set/get/delete on master """
+    zones, _  = init_env(require_ps=False)
+    realm = get_realm()
+    zonegroup = realm.master_zonegroup()
+    bucket_name = gen_bucket_name()
+    topic_name = bucket_name + TOPIC_SUFFIX
+
+    # clean all topics
+    delete_all_s3_topics(zones[0], zonegroup.name)
+   
+    # create s3 topics
+    endpoint_address = 'amqp://127.0.0.1:7001/vhost_1'
+    endpoint_args = 'push-endpoint='+endpoint_address+'&amqp-exchange=amqp.direct&amqp-ack-level=none'
+    topic_conf1 = PSTopicS3(zones[0].conn, topic_name+'_1', zonegroup.name, endpoint_args=endpoint_args)
+    topic_arn = topic_conf1.set_config()
+    assert_equal(topic_arn,
+                 'arn:aws:sns:' + zonegroup.name + ':' + get_tenant() + ':' + topic_name + '_1')
+
+    endpoint_address = 'http://127.0.0.1:9001'
+    endpoint_args = 'push-endpoint='+endpoint_address
+    topic_conf2 = PSTopicS3(zones[0].conn, topic_name+'_2', zonegroup.name, endpoint_args=endpoint_args)
+    topic_arn = topic_conf2.set_config()
+    assert_equal(topic_arn,
+                 'arn:aws:sns:' + zonegroup.name + ':' + get_tenant() + ':' + topic_name + '_2')
+    endpoint_address = 'http://127.0.0.1:9002'
+    endpoint_args = 'push-endpoint='+endpoint_address
+    topic_conf3 = PSTopicS3(zones[0].conn, topic_name+'_3', zonegroup.name, endpoint_args=endpoint_args)
+    topic_arn = topic_conf3.set_config()
+    assert_equal(topic_arn,
+                 'arn:aws:sns:' + zonegroup.name + ':' + get_tenant() + ':' + topic_name + '_3')
+
+    # get topic 3
+    result, status = topic_conf3.get_config()
+    assert_equal(status, 200)
+    assert_equal(topic_arn, result['GetTopicResponse']['GetTopicResult']['Topic']['TopicArn'])
+    assert_equal(endpoint_address, result['GetTopicResponse']['GetTopicResult']['Topic']['EndPoint']['EndpointAddress'])
+    # Note that endpoint args may be ordered differently in the result
+
+    # delete topic 1
+    result = topic_conf1.del_config()
+    assert_equal(status, 200)
+
+    # try to get a deleted topic
+    _, status = topic_conf1.get_config()
+    assert_equal(status, 404)
+
+    # get the remaining 2 topics
+    result, status = topic_conf1.get_list()
+    assert_equal(status, 200)
+    assert_equal(len(result['ListTopicsResponse']['ListTopicsResult']['Topics']['member']), 2)
+    
+    # delete topics
+    result = topic_conf2.del_config()
+    # TODO: should be 200OK
+    # assert_equal(status, 200)
+    result = topic_conf3.del_config()
+    # TODO: should be 200OK
+    # assert_equal(status, 200)
+
+    # get topic list, make sure it is empty
+    result, status = topic_conf1.get_list()
+    assert_equal(result['ListTopicsResponse']['ListTopicsResult']['Topics'], None)
+
+
+def test_ps_s3_topic_with_secret_on_master():
+    """ test s3 topics with secret set/get/delete on master """
+    zones, _  = init_env(require_ps=False)
+    if zones[0].secure_conn is None:
+        return SkipTest('secure connection is needed to test topic with secrets')
+
+    realm = get_realm()
+    zonegroup = realm.master_zonegroup()
+    bucket_name = gen_bucket_name()
+    topic_name = bucket_name + TOPIC_SUFFIX
+
+    # clean all topics
+    delete_all_s3_topics(zones[0], zonegroup.name)
+   
+    # create s3 topics
+    endpoint_address = 'amqp://user:password@127.0.0.1:7001'
+    endpoint_args = 'push-endpoint='+endpoint_address+'&amqp-exchange=amqp.direct&amqp-ack-level=none'
+    bad_topic_conf = PSTopicS3(zones[0].conn, topic_name, zonegroup.name, endpoint_args=endpoint_args)
+    try:
+        result = bad_topic_conf.set_config()
+    except Exception as err:
+        print('Error is expected: ' + str(err))
+    else:
+        assert False, 'user password configuration set allowed only over HTTPS'
+    
+    topic_conf = PSTopicS3(zones[0].secure_conn, topic_name, zonegroup.name, endpoint_args=endpoint_args)
+    topic_arn = topic_conf.set_config()
+
+    assert_equal(topic_arn,
+                 'arn:aws:sns:' + zonegroup.name + ':' + get_tenant() + ':' + topic_name)
+
+    _, status = bad_topic_conf.get_config()
+    assert_equal(status/100, 4)
+
+    # get topic
+    result, status = topic_conf.get_config()
+    assert_equal(status, 200)
+    assert_equal(topic_arn, result['GetTopicResponse']['GetTopicResult']['Topic']['TopicArn'])
+    assert_equal(endpoint_address, result['GetTopicResponse']['GetTopicResult']['Topic']['EndPoint']['EndpointAddress'])
+    
+    _, status = bad_topic_conf.get_config()
+    assert_equal(status/100, 4)
+    
+    _, status = topic_conf.get_list()
+    assert_equal(status/100, 2)
+
+    # delete topics
+    result = topic_conf.del_config()
+
+
+def test_ps_s3_notification_on_master():
+    """ test s3 notification set/get/delete on master """
+    zones, _  = init_env(require_ps=False)
+    realm = get_realm()
+    zonegroup = realm.master_zonegroup()
+    bucket_name = gen_bucket_name()
+    # create bucket
+    bucket = zones[0].create_bucket(bucket_name)
+    topic_name = bucket_name + TOPIC_SUFFIX
+    # create s3 topic
+    endpoint_address = 'amqp://127.0.0.1:7001'
+    endpoint_args = 'push-endpoint='+endpoint_address+'&amqp-exchange=amqp.direct&amqp-ack-level=none'
+    topic_conf = PSTopicS3(zones[0].conn, topic_name, zonegroup.name, endpoint_args=endpoint_args)
+    topic_arn = topic_conf.set_config()
+    # create s3 notification
+    notification_name = bucket_name + NOTIFICATION_SUFFIX
+    topic_conf_list = [{'Id': notification_name+'_1',
+                        'TopicArn': topic_arn,
+                        'Events': ['s3:ObjectCreated:*']
+                       },
+                       {'Id': notification_name+'_2',
+                        'TopicArn': topic_arn,
+                        'Events': ['s3:ObjectRemoved:*']
+                       },
+                       {'Id': notification_name+'_3',
+                        'TopicArn': topic_arn,
+                        'Events': []
+                       }]
+    s3_notification_conf = PSNotificationS3(zones[0].conn, bucket_name, topic_conf_list)
+    _, status = s3_notification_conf.set_config()
+    assert_equal(status/100, 2)
+
+    # get notifications on a bucket
+    response, status = s3_notification_conf.get_config(notification=notification_name+'_1')
+    assert_equal(status/100, 2)
+    assert_equal(response['NotificationConfiguration']['TopicConfiguration']['Topic'], topic_arn)
+
+    # delete specific notifications
+    _, status = s3_notification_conf.del_config(notification=notification_name+'_1')
+    assert_equal(status/100, 2)
+
+    # get the remaining 2 notifications on a bucket
+    response, status = s3_notification_conf.get_config()
+    assert_equal(status/100, 2)
+    assert_equal(len(response['TopicConfigurations']), 2)
+    assert_equal(response['TopicConfigurations'][0]['TopicArn'], topic_arn)
+    assert_equal(response['TopicConfigurations'][1]['TopicArn'], topic_arn)
+
+    # delete remaining notifications
+    _, status = s3_notification_conf.del_config()
+    assert_equal(status/100, 2)
+
+    # make sure that the notifications are now deleted
+    _, status = s3_notification_conf.get_config()
+
+    # cleanup
+    topic_conf.del_config()
+    # delete the bucket
+    zones[0].delete_bucket(bucket_name)
+
+
+def ps_s3_notification_filter(on_master):
+    """ test s3 notification filter on master """
+    if skip_push_tests:
+        return SkipTest("PubSub push tests don't run in teuthology")
+    hostname = get_ip()
+    proc = init_rabbitmq()
+    if proc is  None:
+        return SkipTest('end2end amqp tests require rabbitmq-server installed')
+    if on_master:
+        zones, _  = init_env(require_ps=False)
+        ps_zone = zones[0]
+    else:
+        zones, ps_zones  = init_env(require_ps=True)
+        ps_zone = ps_zones[0]
+
+    realm = get_realm()
+    zonegroup = realm.master_zonegroup()
+    
+    # create bucket
+    bucket_name = gen_bucket_name()
+    bucket = zones[0].create_bucket(bucket_name)
+    topic_name = bucket_name + TOPIC_SUFFIX
+
+    # start amqp receivers
+    exchange = 'ex1'
+    task, receiver = create_amqp_receiver_thread(exchange, topic_name)
+    task.start()
+
+    # create s3 topic
+    endpoint_address = 'amqp://' + hostname
+    endpoint_args = 'push-endpoint='+endpoint_address+'&amqp-exchange=' + exchange +'&amqp-ack-level=broker'
+    if on_master:
+        topic_conf = PSTopicS3(ps_zone.conn, topic_name, zonegroup.name, endpoint_args=endpoint_args)
+        topic_arn = topic_conf.set_config()
+    else:
+        topic_conf = PSTopic(ps_zone.conn, topic_name, endpoint=endpoint_address, endpoint_args=endpoint_args)
+        result, _ = topic_conf.set_config()
+        parsed_result = json.loads(result)
+        topic_arn = parsed_result['arn']
+        zone_meta_checkpoint(ps_zone.zone)
+
+    # create s3 notification
+    notification_name = bucket_name + NOTIFICATION_SUFFIX
+    topic_conf_list = [{'Id': notification_name+'_1',
+                        'TopicArn': topic_arn,
+                        'Events': ['s3:ObjectCreated:*'],
+			'Filter': {
+                    	  'Key': {
+                            'FilterRules': [{'Name': 'prefix', 'Value': 'hello'}]
+                    	  }
+                        }
+                       },
+                       {'Id': notification_name+'_2',
+                        'TopicArn': topic_arn,
+                        'Events': ['s3:ObjectCreated:*'],
+			'Filter': {
+                    	  'Key': {
+                            'FilterRules': [{'Name': 'prefix', 'Value': 'world'},
+                                            {'Name': 'suffix', 'Value': 'log'}]
+                    	  }
+                        }
+                       },
+                       {'Id': notification_name+'_3',
+                        'TopicArn': topic_arn,
+                        'Events': [],
+		        'Filter': {
+                          'Key': {
+                            'FilterRules': [{'Name': 'regex', 'Value': '([a-z]+)\\.txt'}]
+                         }
+                        }
+                       }]
+
+    s3_notification_conf = PSNotificationS3(ps_zone.conn, bucket_name, topic_conf_list)
+    result, status = s3_notification_conf.set_config()
+    assert_equal(status/100, 2)
+
+    if on_master:
+        topic_conf_list = [{'Id': notification_name+'_4',
+                            'TopicArn': topic_arn,
+                            'Events': ['s3:ObjectCreated:*', 's3:ObjectRemoved:*'],
+	    	            'Filter': {
+                    	        'Metadata': {
+                                    'FilterRules': [{'Name': 'x-amz-meta-foo', 'Value': 'bar'},
+                                                    {'Name': 'x-amz-meta-hello', 'Value': 'world'}]
+                    	        },
+                                'Key': {
+                                    'FilterRules': [{'Name': 'regex', 'Value': '([a-z]+)'}]
+                                }
+                            }
+                            }]
+
+        try:
+            s3_notification_conf4 = PSNotificationS3(ps_zone.conn, bucket_name, topic_conf_list)
+            _, status = s3_notification_conf4.set_config()
+            assert_equal(status/100, 2)
+            skip_notif4 = False
+        except Exception as error:
+            print('note: metadata filter is not supported by boto3 - skipping test')
+            skip_notif4 = True
+    else:
+        print('filtering by attributes only supported on master zone')
+        skip_notif4 = True
+
+
+    # get all notifications
+    result, status = s3_notification_conf.get_config()
+    assert_equal(status/100, 2)
+    for conf in result['TopicConfigurations']:
+        filter_name = conf['Filter']['Key']['FilterRules'][0]['Name']
+        assert filter_name == 'prefix' or filter_name == 'suffix' or filter_name == 'regex', filter_name
+
+    if not skip_notif4:
+        result, status = s3_notification_conf4.get_config(notification=notification_name+'_4')
+        assert_equal(status/100, 2)
+        filter_name = result['NotificationConfiguration']['TopicConfiguration']['Filter']['S3Metadata']['FilterRule'][0]['Name']
+        assert filter_name == 'x-amz-meta-foo' or filter_name == 'x-amz-meta-hello'
+
+    expected_in1 = ['hello.kaboom', 'hello.txt', 'hello123.txt', 'hello']
+    expected_in2 = ['world1.log', 'world2log', 'world3.log']
+    expected_in3 = ['hello.txt', 'hell.txt', 'worldlog.txt']
+    expected_in4 = ['foo', 'bar', 'hello', 'world']
+    filtered = ['hell.kaboom', 'world.og', 'world.logg', 'he123ll.txt', 'wo', 'log', 'h', 'txt', 'world.log.txt']
+    filtered_with_attr = ['nofoo', 'nobar', 'nohello', 'noworld']
+    # create objects in bucket
+    for key_name in expected_in1:
+        key = bucket.new_key(key_name)
+        key.set_contents_from_string('bar')
+    for key_name in expected_in2:
+        key = bucket.new_key(key_name)
+        key.set_contents_from_string('bar')
+    for key_name in expected_in3:
+        key = bucket.new_key(key_name)
+        key.set_contents_from_string('bar')
+    if not skip_notif4:
+        for key_name in expected_in4:
+            key = bucket.new_key(key_name)
+            key.set_metadata('foo', 'bar')
+            key.set_metadata('hello', 'world')
+            key.set_metadata('goodbye', 'cruel world')
+            key.set_contents_from_string('bar')
+    for key_name in filtered:
+        key = bucket.new_key(key_name)
+        key.set_contents_from_string('bar')
+    for key_name in filtered_with_attr:
+        key.set_metadata('foo', 'nobar')
+        key.set_metadata('hello', 'noworld')
+        key.set_metadata('goodbye', 'cruel world')
+        key = bucket.new_key(key_name)
+        key.set_contents_from_string('bar')
+
+    if on_master:
+        print('wait for 5sec for the messages...')
+        time.sleep(5)
+    else:
+        zone_bucket_checkpoint(ps_zone.zone, zones[0].zone, bucket_name)
+
+    found_in1 = []
+    found_in2 = []
+    found_in3 = []
+    found_in4 = []
+
+    for event in receiver.get_and_reset_events():
+        notif_id = event['Records'][0]['s3']['configurationId']
+        key_name = event['Records'][0]['s3']['object']['key']
+        if notif_id == notification_name+'_1':
+            found_in1.append(key_name)
+        elif notif_id == notification_name+'_2':
+            found_in2.append(key_name)
+        elif notif_id == notification_name+'_3':
+            found_in3.append(key_name)
+        elif not skip_notif4 and notif_id == notification_name+'_4':
+            found_in4.append(key_name)
+        else:
+            assert False, 'invalid notification: ' + notif_id
+
+    assert_equal(set(found_in1), set(expected_in1))
+    assert_equal(set(found_in2), set(expected_in2))
+    assert_equal(set(found_in3), set(expected_in3))
+    if not skip_notif4:
+        assert_equal(set(found_in4), set(expected_in4))
+
+    # cleanup
+    s3_notification_conf.del_config()
+    if not skip_notif4:
+        s3_notification_conf4.del_config()
+    topic_conf.del_config()
+    # delete the bucket
+    for key in bucket.list():
+        key.delete()
+    zones[0].delete_bucket(bucket_name)
+    stop_amqp_receiver(receiver, task)
+    clean_rabbitmq(proc)
+
+
+def test_ps_s3_notification_filter_on_master():
+    ps_s3_notification_filter(on_master=True)
+
+
+def test_ps_s3_notification_filter():
+    ps_s3_notification_filter(on_master=False)
+
+
+def test_ps_s3_notification_errors_on_master():
+    """ test s3 notification set/get/delete on master """
+    zones, _  = init_env(require_ps=False)
+    realm = get_realm()
+    zonegroup = realm.master_zonegroup()
+    bucket_name = gen_bucket_name()
+    # create bucket
+    bucket = zones[0].create_bucket(bucket_name)
+    topic_name = bucket_name + TOPIC_SUFFIX
+    # create s3 topic
+    endpoint_address = 'amqp://127.0.0.1:7001'
+    endpoint_args = 'push-endpoint='+endpoint_address+'&amqp-exchange=amqp.direct&amqp-ack-level=none'
+    topic_conf = PSTopicS3(zones[0].conn, topic_name, zonegroup.name, endpoint_args=endpoint_args)
+    topic_arn = topic_conf.set_config()
+
+    # create s3 notification with invalid event name
+    notification_name = bucket_name + NOTIFICATION_SUFFIX
+    topic_conf_list = [{'Id': notification_name,
+                        'TopicArn': topic_arn,
+                        'Events': ['s3:ObjectCreated:Kaboom']
+                       }]
+    s3_notification_conf = PSNotificationS3(zones[0].conn, bucket_name, topic_conf_list)
+    try:
+      result, status = s3_notification_conf.set_config()
+    except Exception as error:
+      print(str(error) + ' - is expected')
+    else:
+      assert False, 'invalid event name is expected to fail'
+
+    # create s3 notification with missing name
+    topic_conf_list = [{'Id': '',
+                        'TopicArn': topic_arn,
+                        'Events': ['s3:ObjectCreated:Put']
+                       }]
+    s3_notification_conf = PSNotificationS3(zones[0].conn, bucket_name, topic_conf_list)
+    try:
+      _, _ = s3_notification_conf.set_config()
+    except Exception as error:
+      print(str(error) + ' - is expected')
+    else:
+      assert False, 'missing notification name is expected to fail'
+
+    # create s3 notification with invalid topic ARN
+    invalid_topic_arn = 'kaboom'
+    topic_conf_list = [{'Id': notification_name,
+                        'TopicArn': invalid_topic_arn,
+                        'Events': ['s3:ObjectCreated:Put']
+                       }]
+    s3_notification_conf = PSNotificationS3(zones[0].conn, bucket_name, topic_conf_list)
+    try:
+      _, _ = s3_notification_conf.set_config()
+    except Exception as error:
+      print(str(error) + ' - is expected')
+    else:
+      assert False, 'invalid ARN is expected to fail'
+
+    # create s3 notification with unknown topic ARN
+    invalid_topic_arn = 'arn:aws:sns:a::kaboom'
+    topic_conf_list = [{'Id': notification_name,
+                        'TopicArn': invalid_topic_arn ,
+                        'Events': ['s3:ObjectCreated:Put']
+                       }]
+    s3_notification_conf = PSNotificationS3(zones[0].conn, bucket_name, topic_conf_list)
+    try:
+      _, _ = s3_notification_conf.set_config()
+    except Exception as error:
+      print(str(error) + ' - is expected')
+    else:
+      assert False, 'unknown topic is expected to fail'
+
+    # create s3 notification with wrong bucket
+    topic_conf_list = [{'Id': notification_name,
+                        'TopicArn': topic_arn,
+                        'Events': ['s3:ObjectCreated:Put']
+                       }]
+    s3_notification_conf = PSNotificationS3(zones[0].conn, 'kaboom', topic_conf_list)
+    try:
+      _, _ = s3_notification_conf.set_config()
+    except Exception as error:
+      print(str(error) + ' - is expected')
+    else:
+      assert False, 'unknown bucket is expected to fail'
+
+    topic_conf.del_config()
+
+    status = topic_conf.del_config()
+    # deleting an unknown notification is not considered an error
+    assert_equal(status, 200)
+    
+    _, status = topic_conf.get_config()
+    assert_equal(status, 404)
+    
+    # cleanup
+    # delete the bucket
+    zones[0].delete_bucket(bucket_name)
+
+
+def test_objcet_timing():
+    return SkipTest("only used in manual testing")
+    zones, _  = init_env(require_ps=False)
+    
+    # create bucket
+    bucket_name = gen_bucket_name()
+    bucket = zones[0].create_bucket(bucket_name)
+    # create objects in the bucket (async)
+    print('creating objects...')
+    number_of_objects = 1000
+    client_threads = []
+    start_time = time.time()
+    content = str(bytearray(os.urandom(1024*1024)))
+    for i in range(number_of_objects):
+        key = bucket.new_key(str(i))
+        thr = threading.Thread(target = set_contents_from_string, args=(key, content,))
+        thr.start()
+        client_threads.append(thr)
+    [thr.join() for thr in client_threads] 
+
+    time_diff = time.time() - start_time
+    print('average time for object creation: ' + str(time_diff*1000/number_of_objects) + ' milliseconds')
+    
+    print('total number of objects: ' + str(len(list(bucket.list()))))
+
+    print('deleting objects...')
+    client_threads = []
+    start_time = time.time()
+    for key in bucket.list():
+        thr = threading.Thread(target = key.delete, args=())
+        thr.start()
+        client_threads.append(thr)
+    [thr.join() for thr in client_threads] 
+    
+    time_diff = time.time() - start_time
+    print('average time for object deletion: ' + str(time_diff*1000/number_of_objects) + ' milliseconds')
+    
+    # cleanup
+    zones[0].delete_bucket(bucket_name)
+
+
+def test_ps_s3_notification_push_amqp_on_master():
+    """ test pushing amqp s3 notification on master """
+    if skip_push_tests:
+        return SkipTest("PubSub push tests don't run in teuthology")
+    hostname = get_ip()
+    proc = init_rabbitmq()
+    if proc is  None:
+        return SkipTest('end2end amqp tests require rabbitmq-server installed')
+    zones, _  = init_env(require_ps=False)
+    realm = get_realm()
+    zonegroup = realm.master_zonegroup()
+    
+    # create bucket
+    bucket_name = gen_bucket_name()
+    bucket = zones[0].create_bucket(bucket_name)
+    topic_name1 = bucket_name + TOPIC_SUFFIX + '_1'
+    topic_name2 = bucket_name + TOPIC_SUFFIX + '_2'
+
+    # start amqp receivers
+    exchange = 'ex1'
+    task1, receiver1 = create_amqp_receiver_thread(exchange, topic_name1)
+    task2, receiver2 = create_amqp_receiver_thread(exchange, topic_name2)
+    task1.start()
+    task2.start()
+
+    # create two s3 topic
+    endpoint_address = 'amqp://' + hostname
+    # with acks from broker
+    endpoint_args = 'push-endpoint='+endpoint_address+'&amqp-exchange=' + exchange +'&amqp-ack-level=broker'
+    topic_conf1 = PSTopicS3(zones[0].conn, topic_name1, zonegroup.name, endpoint_args=endpoint_args)
+    topic_arn1 = topic_conf1.set_config()
+    # without acks from broker
+    endpoint_args = 'push-endpoint='+endpoint_address+'&amqp-exchange=' + exchange +'&amqp-ack-level=none'
+    topic_conf2 = PSTopicS3(zones[0].conn, topic_name2, zonegroup.name, endpoint_args=endpoint_args)
+    topic_arn2 = topic_conf2.set_config()
+    # create s3 notification
+    notification_name = bucket_name + NOTIFICATION_SUFFIX
+    topic_conf_list = [{'Id': notification_name+'_1', 'TopicArn': topic_arn1,
+                         'Events': []
+                       },
+                       {'Id': notification_name+'_2', 'TopicArn': topic_arn2,
+                         'Events': ['s3:ObjectCreated:*']
+                       }]
+
+    s3_notification_conf = PSNotificationS3(zones[0].conn, bucket_name, topic_conf_list)
+    response, status = s3_notification_conf.set_config()
+    assert_equal(status/100, 2)
+
+    # create objects in the bucket (async)
+    number_of_objects = 100
+    client_threads = []
+    start_time = time.time()
+    for i in range(number_of_objects):
+        key = bucket.new_key(str(i))
+        content = str(os.urandom(1024*1024))
+        thr = threading.Thread(target = set_contents_from_string, args=(key, content,))
+        thr.start()
+        client_threads.append(thr)
+    [thr.join() for thr in client_threads] 
+
+    time_diff = time.time() - start_time
+    print('average time for creation + qmqp notification is: ' + str(time_diff*1000/number_of_objects) + ' milliseconds')
+
+    print('wait for 5sec for the messages...')
+    time.sleep(5)
+
+    # check amqp receiver
+    keys = list(bucket.list())
+    print('total number of objects: ' + str(len(keys)))
+    receiver1.verify_s3_events(keys, exact_match=True)
+    receiver2.verify_s3_events(keys, exact_match=True)
+    
+    # delete objects from the bucket
+    client_threads = []
+    start_time = time.time()
+    for key in bucket.list():
+        thr = threading.Thread(target = key.delete, args=())
+        thr.start()
+        client_threads.append(thr)
+    [thr.join() for thr in client_threads] 
+    
+    time_diff = time.time() - start_time
+    print('average time for creation + http notification is: ' + str(time_diff*1000/number_of_objects) + ' milliseconds')
+
+    print('wait for 5sec for the messages...')
+    time.sleep(5)
+    
+    # check amqp receiver 1 for deletions
+    receiver1.verify_s3_events(keys, exact_match=True, deletions=True)
+    # check amqp receiver 2 has no deletions
+    try:
+        receiver1.verify_s3_events(keys, exact_match=False, deletions=True)
+    except:
+        pass
+    else:
+        err = 'amqp receiver 2 should have no deletions'
+        assert False, err
+
+    # cleanup
+    stop_amqp_receiver(receiver1, task1)
+    stop_amqp_receiver(receiver2, task2)
+    s3_notification_conf.del_config()
+    topic_conf1.del_config()
+    topic_conf2.del_config()
+    # delete the bucket
+    zones[0].delete_bucket(bucket_name)
+    clean_rabbitmq(proc)
+
+
+def test_ps_s3_notification_push_kafka():
+    """ test pushing kafka s3 notification on master """
+    if skip_push_tests:
+        return SkipTest("PubSub push tests don't run in teuthology")
+    kafka_proc, zk_proc, kafka_log = init_kafka()
+    if kafka_proc is  None or zk_proc is None:
+        return SkipTest('end2end kafka tests require kafka/zookeeper installed')
+
+    zones, ps_zones  = init_env(require_ps=True)
+    realm = get_realm()
+    zonegroup = realm.master_zonegroup()
+    
+    # create bucket
+    bucket_name = gen_bucket_name()
+    bucket = zones[0].create_bucket(bucket_name)
+    # wait for sync
+    zone_meta_checkpoint(ps_zones[0].zone)
+    # name is constant for manual testing
+    topic_name = bucket_name+'_topic'
+    # create consumer on the topic
+    task, receiver = create_kafka_receiver_thread(topic_name)
+    task.start()
+
+    # create topic
+    topic_conf = PSTopic(ps_zones[0].conn, topic_name,
+                         endpoint='kafka://' + kafka_server,
+                         endpoint_args='kafka-ack-level=broker')
+    result, status = topic_conf.set_config()
+    assert_equal(status/100, 2)
+    parsed_result = json.loads(result)
+    topic_arn = parsed_result['arn']
+    # create s3 notification
+    notification_name = bucket_name + NOTIFICATION_SUFFIX
+    topic_conf_list = [{'Id': notification_name, 'TopicArn': topic_arn,
+                         'Events': []
+                       }]
+
+    s3_notification_conf = PSNotificationS3(ps_zones[0].conn, bucket_name, topic_conf_list)
+    response, status = s3_notification_conf.set_config()
+    assert_equal(status/100, 2)
+
+    # create objects in the bucket (async)
+    number_of_objects = 10
+    client_threads = []
+    for i in range(number_of_objects):
+        key = bucket.new_key(str(i))
+        content = str(os.urandom(1024*1024))
+        thr = threading.Thread(target = set_contents_from_string, args=(key, content,))
+        thr.start()
+        client_threads.append(thr)
+    [thr.join() for thr in client_threads] 
+
+    # wait for sync
+    zone_bucket_checkpoint(ps_zones[0].zone, zones[0].zone, bucket_name)
+    keys = list(bucket.list())
+    receiver.verify_s3_events(keys, exact_match=True)
+
+    # delete objects from the bucket
+    client_threads = []
+    for key in bucket.list():
+        thr = threading.Thread(target = key.delete, args=())
+        thr.start()
+        client_threads.append(thr)
+    [thr.join() for thr in client_threads] 
+    
+    zone_bucket_checkpoint(ps_zones[0].zone, zones[0].zone, bucket_name)
+    receiver.verify_s3_events(keys, exact_match=True, deletions=True)
+    
+    # cleanup
+    s3_notification_conf.del_config()
+    topic_conf.del_config()
+    # delete the bucket
+    zones[0].delete_bucket(bucket_name)
+    stop_kafka_receiver(receiver, task)
+    clean_kafka(kafka_proc, zk_proc, kafka_log)
+
+
+def test_ps_s3_notification_push_kafka_on_master():
+    """ test pushing kafka s3 notification on master """
+    if skip_push_tests:
+        return SkipTest("PubSub push tests don't run in teuthology")
+    kafka_proc, zk_proc, kafka_log = init_kafka()
+    if kafka_proc is None or zk_proc is None:
+        return SkipTest('end2end kafka tests require kafka/zookeeper installed')
+    zones, _  = init_env(require_ps=False)
+    realm = get_realm()
+    zonegroup = realm.master_zonegroup()
+    
+    # create bucket
+    bucket_name = gen_bucket_name()
+    bucket = zones[0].create_bucket(bucket_name)
+    # name is constant for manual testing
+    topic_name = bucket_name+'_topic'
+    # create consumer on the topic
+    task, receiver = create_kafka_receiver_thread(topic_name+'_1')
+    task.start()
+
+    # create s3 topic
+    endpoint_address = 'kafka://' + kafka_server
+    # without acks from broker
+    endpoint_args = 'push-endpoint='+endpoint_address+'&kafka-ack-level=broker'
+    topic_conf1 = PSTopicS3(zones[0].conn, topic_name+'_1', zonegroup.name, endpoint_args=endpoint_args)
+    topic_arn1 = topic_conf1.set_config()
+    endpoint_args = 'push-endpoint='+endpoint_address+'&kafka-ack-level=none'
+    topic_conf2 = PSTopicS3(zones[0].conn, topic_name+'_2', zonegroup.name, endpoint_args=endpoint_args)
+    topic_arn2 = topic_conf2.set_config()
+    # create s3 notification
+    notification_name = bucket_name + NOTIFICATION_SUFFIX
+    topic_conf_list = [{'Id': notification_name + '_1', 'TopicArn': topic_arn1,
+                         'Events': []
+                       },
+                       {'Id': notification_name + '_2', 'TopicArn': topic_arn2,
+                         'Events': []
+                       }]
+
+    s3_notification_conf = PSNotificationS3(zones[0].conn, bucket_name, topic_conf_list)
+    response, status = s3_notification_conf.set_config()
+    assert_equal(status/100, 2)
+
+    # create objects in the bucket (async)
+    number_of_objects = 10
+    client_threads = []
+    start_time = time.time()
+    for i in range(number_of_objects):
+        key = bucket.new_key(str(i))
+        content = str(os.urandom(1024*1024))
+        thr = threading.Thread(target = set_contents_from_string, args=(key, content,))
+        thr.start()
+        client_threads.append(thr)
+    [thr.join() for thr in client_threads] 
+
+    time_diff = time.time() - start_time
+    print('average time for creation + kafka notification is: ' + str(time_diff*1000/number_of_objects) + ' milliseconds')
+
+    print('wait for 5sec for the messages...')
+    time.sleep(5)
+    keys = list(bucket.list())
+    receiver.verify_s3_events(keys, exact_match=True)
+
+    # delete objects from the bucket
+    client_threads = []
+    start_time = time.time()
+    for key in bucket.list():
+        thr = threading.Thread(target = key.delete, args=())
+        thr.start()
+        client_threads.append(thr)
+    [thr.join() for thr in client_threads] 
+    
+    time_diff = time.time() - start_time
+    print('average time for deletion + kafka notification is: ' + str(time_diff*1000/number_of_objects) + ' milliseconds')
+
+    print('wait for 5sec for the messages...')
+    time.sleep(5)
+    receiver.verify_s3_events(keys, exact_match=True, deletions=True)
+    
+    # cleanup
+    s3_notification_conf.del_config()
+    topic_conf1.del_config()
+    topic_conf2.del_config()
+    # delete the bucket
+    zones[0].delete_bucket(bucket_name)
+    stop_kafka_receiver(receiver, task)
+    clean_kafka(kafka_proc, zk_proc, kafka_log)
+
+
+def kafka_security(security_type):
+    """ test pushing kafka s3 notification on master """
+    if skip_push_tests:
+        return SkipTest("PubSub push tests don't run in teuthology")
+    zones, _  = init_env(require_ps=False)
+    if security_type == 'SSL_SASL' and zones[0].secure_conn is None:
+        return SkipTest("secure connection is needed to test SASL_SSL security")
+    kafka_proc, zk_proc, kafka_log = init_kafka()
+    if kafka_proc is None or zk_proc is None:
+        return SkipTest('end2end kafka tests require kafka/zookeeper installed')
+    realm = get_realm()
+    zonegroup = realm.master_zonegroup()
+    
+    # create bucket
+    bucket_name = gen_bucket_name()
+    bucket = zones[0].create_bucket(bucket_name)
+    # name is constant for manual testing
+    topic_name = bucket_name+'_topic'
+    # create consumer on the topic
+    task, receiver = create_kafka_receiver_thread(topic_name)
+    task.start()
+
+    # create s3 topic
+    if security_type == 'SSL_SASL':
+        endpoint_address = 'kafka://alice:alice-secret@' + kafka_server + ':9094'
+    else:
+        # ssl only
+        endpoint_address = 'kafka://' + kafka_server + ':9093'
+
+    KAFKA_DIR = os.environ['KAFKA_DIR']
+
+    # without acks from broker, with root CA
+    endpoint_args = 'push-endpoint='+endpoint_address+'&kafka-ack-level=none&use-ssl=true&ca-location='+KAFKA_DIR+'rootCA.crt'
+
+    if security_type == 'SSL_SASL':
+        topic_conf = PSTopicS3(zones[0].secure_conn, topic_name, zonegroup.name, endpoint_args=endpoint_args)
+    else:
+        topic_conf = PSTopicS3(zones[0].conn, topic_name, zonegroup.name, endpoint_args=endpoint_args)
+
+    topic_arn = topic_conf.set_config()
+    # create s3 notification
+    notification_name = bucket_name + NOTIFICATION_SUFFIX
+    topic_conf_list = [{'Id': notification_name, 'TopicArn': topic_arn,
+                         'Events': []
+                       }]
+
+    s3_notification_conf = PSNotificationS3(zones[0].conn, bucket_name, topic_conf_list)
+    s3_notification_conf.set_config()
+
+    # create objects in the bucket (async)
+    number_of_objects = 10
+    client_threads = []
+    start_time = time.time()
+    for i in range(number_of_objects):
+        key = bucket.new_key(str(i))
+        content = str(os.urandom(1024*1024))
+        thr = threading.Thread(target = set_contents_from_string, args=(key, content,))
+        thr.start()
+        client_threads.append(thr)
+    [thr.join() for thr in client_threads] 
+
+    time_diff = time.time() - start_time
+    print('average time for creation + kafka notification is: ' + str(time_diff*1000/number_of_objects) + ' milliseconds')
+
+    try:
+        print('wait for 5sec for the messages...')
+        time.sleep(5)
+        keys = list(bucket.list())
+        receiver.verify_s3_events(keys, exact_match=True)
+
+        # delete objects from the bucket
+        client_threads = []
+        start_time = time.time()
+        for key in bucket.list():
+            thr = threading.Thread(target = key.delete, args=())
+            thr.start()
+            client_threads.append(thr)
+        [thr.join() for thr in client_threads] 
+        
+        time_diff = time.time() - start_time
+        print('average time for deletion + kafka notification is: ' + str(time_diff*1000/number_of_objects) + ' milliseconds')
+
+        print('wait for 5sec for the messages...')
+        time.sleep(5)
+        receiver.verify_s3_events(keys, exact_match=True, deletions=True)
+    except Exception as err:
+        assert False, str(err)
+    finally: 
+        # cleanup
+        s3_notification_conf.del_config()
+        topic_conf.del_config()
+        # delete the bucket
+        for key in bucket.list():
+            key.delete()
+        zones[0].delete_bucket(bucket_name)
+        stop_kafka_receiver(receiver, task)
+        clean_kafka(kafka_proc, zk_proc, kafka_log)
+
+
+def test_ps_s3_notification_push_kafka_security_ssl():
+    kafka_security('SSL')
+
+def test_ps_s3_notification_push_kafka_security_ssl_sasl():
+    kafka_security('SSL_SASL')
+
+
+def test_ps_s3_notification_multi_delete_on_master():
+    """ test deletion of multiple keys on master """
+    if skip_push_tests:
+        return SkipTest("PubSub push tests don't run in teuthology")
+    hostname = get_ip()
+    zones, _  = init_env(require_ps=False)
+    realm = get_realm()
+    zonegroup = realm.master_zonegroup()
+    
+    # create random port for the http server
+    host = get_ip()
+    port = random.randint(10000, 20000)
+    # start an http server in a separate thread
+    number_of_objects = 10
+    http_server = StreamingHTTPServer(host, port, num_workers=number_of_objects)
+    
+    # create bucket
+    bucket_name = gen_bucket_name()
+    bucket = zones[0].create_bucket(bucket_name)
+    topic_name = bucket_name + TOPIC_SUFFIX
+
+    # create s3 topic
+    endpoint_address = 'http://'+host+':'+str(port)
+    endpoint_args = 'push-endpoint='+endpoint_address
+    topic_conf = PSTopicS3(zones[0].conn, topic_name, zonegroup.name, endpoint_args=endpoint_args)
+    topic_arn = topic_conf.set_config()
+    # create s3 notification
+    notification_name = bucket_name + NOTIFICATION_SUFFIX
+    topic_conf_list = [{'Id': notification_name,
+                        'TopicArn': topic_arn,
+                        'Events': ['s3:ObjectRemoved:*']
+                       }]
+    s3_notification_conf = PSNotificationS3(zones[0].conn, bucket_name, topic_conf_list)
+    response, status = s3_notification_conf.set_config()
+    assert_equal(status/100, 2)
+
+    # create objects in the bucket
+    client_threads = []
+    for i in range(number_of_objects):
+        obj_size = randint(1, 1024)
+        content = str(os.urandom(obj_size))
+        key = bucket.new_key(str(i))
+        thr = threading.Thread(target = set_contents_from_string, args=(key, content,))
+        thr.start()
+        client_threads.append(thr)
+    [thr.join() for thr in client_threads] 
+    
+    keys = list(bucket.list())
+
+    start_time = time.time()
+    delete_all_objects(zones[0].conn, bucket_name)
+    time_diff = time.time() - start_time
+    print('average time for deletion + http notification is: ' + str(time_diff*1000/number_of_objects) + ' milliseconds')
+
+    print('wait for 5sec for the messages...')
+    time.sleep(5)
+    
+    # check http receiver
+    http_server.verify_s3_events(keys, exact_match=True, deletions=True)
+    
+    # cleanup
+    topic_conf.del_config()
+    s3_notification_conf.del_config(notification=notification_name)
+    # delete the bucket
+    zones[0].delete_bucket(bucket_name)
+    http_server.close()
+
+
+def test_ps_s3_notification_push_http_on_master():
+    """ test pushing http s3 notification on master """
+    if skip_push_tests:
+        return SkipTest("PubSub push tests don't run in teuthology")
+    hostname = get_ip()
+    zones, _  = init_env(require_ps=False)
+    realm = get_realm()
+    zonegroup = realm.master_zonegroup()
+    
+    # create random port for the http server
+    host = get_ip()
+    port = random.randint(10000, 20000)
+    # start an http server in a separate thread
+    number_of_objects = 100
+    http_server = StreamingHTTPServer(host, port, num_workers=number_of_objects)
+    
+    # create bucket
+    bucket_name = gen_bucket_name()
+    bucket = zones[0].create_bucket(bucket_name)
+    topic_name = bucket_name + TOPIC_SUFFIX
+
+    # create s3 topic
+    endpoint_address = 'http://'+host+':'+str(port)
+    endpoint_args = 'push-endpoint='+endpoint_address
+    topic_conf = PSTopicS3(zones[0].conn, topic_name, zonegroup.name, endpoint_args=endpoint_args)
+    topic_arn = topic_conf.set_config()
+    # create s3 notification
+    notification_name = bucket_name + NOTIFICATION_SUFFIX
+    topic_conf_list = [{'Id': notification_name,
+                        'TopicArn': topic_arn,
+                        'Events': []
+                       }]
+    s3_notification_conf = PSNotificationS3(zones[0].conn, bucket_name, topic_conf_list)
+    response, status = s3_notification_conf.set_config()
+    assert_equal(status/100, 2)
+
+    # create objects in the bucket
+    client_threads = []
+    start_time = time.time()
+    content = 'bar'
+    for i in range(number_of_objects):
+        key = bucket.new_key(str(i))
+        thr = threading.Thread(target = set_contents_from_string, args=(key, content,))
+        thr.start()
+        client_threads.append(thr)
+    [thr.join() for thr in client_threads] 
+
+    time_diff = time.time() - start_time
+    print('average time for creation + http notification is: ' + str(time_diff*1000/number_of_objects) + ' milliseconds')
+
+    print('wait for 5sec for the messages...')
+    time.sleep(5)
+    
+    # check http receiver
+    keys = list(bucket.list())
+    print('total number of objects: ' + str(len(keys)))
+    http_server.verify_s3_events(keys, exact_match=True)
+    
+    # delete objects from the bucket
+    client_threads = []
+    start_time = time.time()
+    for key in bucket.list():
+        thr = threading.Thread(target = key.delete, args=())
+        thr.start()
+        client_threads.append(thr)
+    [thr.join() for thr in client_threads] 
+    
+    time_diff = time.time() - start_time
+    print('average time for creation + http notification is: ' + str(time_diff*1000/number_of_objects) + ' milliseconds')
+
+    print('wait for 5sec for the messages...')
+    time.sleep(5)
+    
+    # check http receiver
+    http_server.verify_s3_events(keys, exact_match=True, deletions=True)
+    
+    # cleanup
+    topic_conf.del_config()
+    s3_notification_conf.del_config(notification=notification_name)
+    # delete the bucket
+    zones[0].delete_bucket(bucket_name)
+    http_server.close()
+
+
+def test_ps_s3_opaque_data():
+    """ test that opaque id set in topic, is sent in notification """
+    if skip_push_tests:
+        return SkipTest("PubSub push tests don't run in teuthology")
+    hostname = get_ip()
+    zones, ps_zones  = init_env(require_ps=True)
+    realm = get_realm()
+    zonegroup = realm.master_zonegroup()
+    
+    # create random port for the http server
+    host = get_ip()
+    port = random.randint(10000, 20000)
+    # start an http server in a separate thread
+    number_of_objects = 10
+    http_server = StreamingHTTPServer(host, port, num_workers=number_of_objects)
+    
+    # create bucket
+    bucket_name = gen_bucket_name()
+    bucket = zones[0].create_bucket(bucket_name)
+    topic_name = bucket_name + TOPIC_SUFFIX
+    # wait for sync
+    zone_meta_checkpoint(ps_zones[0].zone)
+
+    # create s3 topic
+    endpoint_address = 'http://'+host+':'+str(port)
+    opaque_data = 'http://1.2.3.4:8888'
+    endpoint_args = 'push-endpoint='+endpoint_address+'&OpaqueData='+opaque_data
+    topic_conf = PSTopic(ps_zones[0].conn, topic_name, endpoint=endpoint_address, endpoint_args=endpoint_args)
+    result, status = topic_conf.set_config()
+    assert_equal(status/100, 2)
+    parsed_result = json.loads(result)
+    topic_arn = parsed_result['arn']
+    # create s3 notification
+    notification_name = bucket_name + NOTIFICATION_SUFFIX
+    topic_conf_list = [{'Id': notification_name,
+                        'TopicArn': topic_arn,
+                        'Events': []
+                       }]
+    s3_notification_conf = PSNotificationS3(ps_zones[0].conn, bucket_name, topic_conf_list)
+    response, status = s3_notification_conf.set_config()
+    assert_equal(status/100, 2)
+
+    # create objects in the bucket
+    client_threads = []
+    content = 'bar'
+    for i in range(number_of_objects):
+        key = bucket.new_key(str(i))
+        thr = threading.Thread(target = set_contents_from_string, args=(key, content,))
+        thr.start()
+        client_threads.append(thr)
+    [thr.join() for thr in client_threads] 
+
+    # wait for sync
+    zone_bucket_checkpoint(ps_zones[0].zone, zones[0].zone, bucket_name)
+
+    # check http receiver
+    keys = list(bucket.list())
+    print('total number of objects: ' + str(len(keys)))
+    events = http_server.get_and_reset_events()
+    for event in events:
+        assert_equal(event['Records'][0]['opaqueData'], opaque_data)
+    
+    # cleanup
+    for key in keys:
+        key.delete()
+    [thr.join() for thr in client_threads] 
+    topic_conf.del_config()
+    s3_notification_conf.del_config(notification=notification_name)
+    # delete the bucket
+    zones[0].delete_bucket(bucket_name)
+    http_server.close()
+
+
+def test_ps_s3_opaque_data_on_master():
+    """ test that opaque id set in topic, is sent in notification on master """
+    if skip_push_tests:
+        return SkipTest("PubSub push tests don't run in teuthology")
+    hostname = get_ip()
+    zones, _  = init_env(require_ps=False)
+    realm = get_realm()
+    zonegroup = realm.master_zonegroup()
+    
+    # create random port for the http server
+    host = get_ip()
+    port = random.randint(10000, 20000)
+    # start an http server in a separate thread
+    number_of_objects = 10
+    http_server = StreamingHTTPServer(host, port, num_workers=number_of_objects)
+    
+    # create bucket
+    bucket_name = gen_bucket_name()
+    bucket = zones[0].create_bucket(bucket_name)
+    topic_name = bucket_name + TOPIC_SUFFIX
+
+    # create s3 topic
+    endpoint_address = 'http://'+host+':'+str(port)
+    endpoint_args = 'push-endpoint='+endpoint_address
+    opaque_data = 'http://1.2.3.4:8888'
+    topic_conf = PSTopicS3(zones[0].conn, topic_name, zonegroup.name, endpoint_args=endpoint_args, opaque_data=opaque_data)
+    topic_arn = topic_conf.set_config()
+    # create s3 notification
+    notification_name = bucket_name + NOTIFICATION_SUFFIX
+    topic_conf_list = [{'Id': notification_name,
+                        'TopicArn': topic_arn,
+                        'Events': []
+                       }]
+    s3_notification_conf = PSNotificationS3(zones[0].conn, bucket_name, topic_conf_list)
+    response, status = s3_notification_conf.set_config()
+    assert_equal(status/100, 2)
+
+    # create objects in the bucket
+    client_threads = []
+    start_time = time.time()
+    content = 'bar'
+    for i in range(number_of_objects):
+        key = bucket.new_key(str(i))
+        thr = threading.Thread(target = set_contents_from_string, args=(key, content,))
+        thr.start()
+        client_threads.append(thr)
+    [thr.join() for thr in client_threads] 
+
+    time_diff = time.time() - start_time
+    print('average time for creation + http notification is: ' + str(time_diff*1000/number_of_objects) + ' milliseconds')
+
+    print('wait for 5sec for the messages...')
+    time.sleep(5)
+    
+    # check http receiver
+    keys = list(bucket.list())
+    print('total number of objects: ' + str(len(keys)))
+    events = http_server.get_and_reset_events()
+    for event in events:
+        assert_equal(event['Records'][0]['opaqueData'], opaque_data)
+    
+    # cleanup
+    for key in keys:
+        key.delete()
+    [thr.join() for thr in client_threads] 
+    topic_conf.del_config()
+    s3_notification_conf.del_config(notification=notification_name)
+    # delete the bucket
+    zones[0].delete_bucket(bucket_name)
+    http_server.close()
+
+def test_ps_topic():
+    """ test set/get/delete of topic """
+    _, ps_zones = init_env()
+    realm = get_realm()
+    zonegroup = realm.master_zonegroup()
+    bucket_name = gen_bucket_name()
+    topic_name = bucket_name+TOPIC_SUFFIX
+
+    # create topic
+    topic_conf = PSTopic(ps_zones[0].conn, topic_name)
+    _, status = topic_conf.set_config()
+    assert_equal(status/100, 2)
+    # get topic
+    result, _ = topic_conf.get_config()
+    # verify topic content
+    parsed_result = json.loads(result)
+    assert_equal(parsed_result['topic']['name'], topic_name)
+    assert_equal(len(parsed_result['subs']), 0)
+    assert_equal(parsed_result['topic']['arn'],
+                 'arn:aws:sns:' + zonegroup.name + ':' + get_tenant() + ':' + topic_name)
+    # delete topic
+    _, status = topic_conf.del_config()
+    assert_equal(status/100, 2)
+    # verift topic is deleted
+    result, status = topic_conf.get_config()
+    assert_equal(status, 404)
+    parsed_result = json.loads(result)
+    assert_equal(parsed_result['Code'], 'NoSuchKey')
+
+
+def test_ps_topic_with_endpoint():
+    """ test set topic with endpoint"""
+    _, ps_zones = init_env()
+    bucket_name = gen_bucket_name()
+    topic_name = bucket_name+TOPIC_SUFFIX
+
+    # create topic
+    dest_endpoint = 'amqp://localhost:7001'
+    dest_args = 'amqp-exchange=amqp.direct&amqp-ack-level=none'
+    topic_conf = PSTopic(ps_zones[0].conn, topic_name,
+                         endpoint=dest_endpoint,
+                         endpoint_args=dest_args)
+    _, status = topic_conf.set_config()
+    assert_equal(status/100, 2)
+    # get topic
+    result, _ = topic_conf.get_config()
+    # verify topic content
+    parsed_result = json.loads(result)
+    assert_equal(parsed_result['topic']['name'], topic_name)
+    assert_equal(parsed_result['topic']['dest']['push_endpoint'], dest_endpoint)
+    # cleanup
+    topic_conf.del_config()
+
+
+def test_ps_notification():
+    """ test set/get/delete of notification """
+    zones, ps_zones = init_env()
+    bucket_name = gen_bucket_name()
+    topic_name = bucket_name+TOPIC_SUFFIX
+
+    # create topic
+    topic_conf = PSTopic(ps_zones[0].conn, topic_name)
+    topic_conf.set_config()
+    # create bucket on the first of the rados zones
+    zones[0].create_bucket(bucket_name)
+    # wait for sync
+    zone_meta_checkpoint(ps_zones[0].zone)
+    # create notifications
+    notification_conf = PSNotification(ps_zones[0].conn, bucket_name,
+                                       topic_name)
+    _, status = notification_conf.set_config()
+    assert_equal(status/100, 2)
+    # get notification
+    result, _ = notification_conf.get_config()
+    parsed_result = json.loads(result)
+    assert_equal(len(parsed_result['topics']), 1)
+    assert_equal(parsed_result['topics'][0]['topic']['name'],
+                 topic_name)
+    # delete notification
+    _, status = notification_conf.del_config()
+    assert_equal(status/100, 2)
+    result, status = notification_conf.get_config()
+    parsed_result = json.loads(result)
+    assert_equal(len(parsed_result['topics']), 0)
+    # TODO should return 404
+    # assert_equal(status, 404)
+
+    # cleanup
+    topic_conf.del_config()
+    zones[0].delete_bucket(bucket_name)
+
+
+def test_ps_notification_events():
+    """ test set/get/delete of notification on specific events"""
+    zones, ps_zones = init_env()
+    bucket_name = gen_bucket_name()
+    topic_name = bucket_name+TOPIC_SUFFIX
+
+    # create topic
+    topic_conf = PSTopic(ps_zones[0].conn, topic_name)
+    topic_conf.set_config()
+    # create bucket on the first of the rados zones
+    zones[0].create_bucket(bucket_name)
+    # wait for sync
+    zone_meta_checkpoint(ps_zones[0].zone)
+    # create notifications
+    events = "OBJECT_CREATE,OBJECT_DELETE"
+    notification_conf = PSNotification(ps_zones[0].conn, bucket_name,
+                                       topic_name,
+                                       events)
+    _, status = notification_conf.set_config()
+    assert_equal(status/100, 2)
+    # get notification
+    result, _ = notification_conf.get_config()
+    parsed_result = json.loads(result)
+    assert_equal(len(parsed_result['topics']), 1)
+    assert_equal(parsed_result['topics'][0]['topic']['name'],
+                 topic_name)
+    assert_not_equal(len(parsed_result['topics'][0]['events']), 0)
+    # TODO add test for invalid event name
+
+    # cleanup
+    notification_conf.del_config()
+    topic_conf.del_config()
+    zones[0].delete_bucket(bucket_name)
+
+
+def test_ps_subscription():
+    """ test set/get/delete of subscription """
+    zones, ps_zones = init_env()
+    bucket_name = gen_bucket_name()
+    topic_name = bucket_name+TOPIC_SUFFIX
+
+    # create topic
+    topic_conf = PSTopic(ps_zones[0].conn, topic_name)
+    topic_conf.set_config()
+    # create bucket on the first of the rados zones
+    bucket = zones[0].create_bucket(bucket_name)
+    # wait for sync
+    zone_meta_checkpoint(ps_zones[0].zone)
+    # create notifications
+    notification_conf = PSNotification(ps_zones[0].conn, bucket_name,
+                                       topic_name)
+    _, status = notification_conf.set_config()
+    assert_equal(status/100, 2)
+    # create subscription
+    sub_conf = PSSubscription(ps_zones[0].conn, bucket_name+SUB_SUFFIX,
+                              topic_name)
+    _, status = sub_conf.set_config()
+    assert_equal(status/100, 2)
+    # get the subscription
+    result, _ = sub_conf.get_config()
+    parsed_result = json.loads(result)
+    assert_equal(parsed_result['topic'], topic_name)
+    # create objects in the bucket
+    number_of_objects = 10
+    for i in range(number_of_objects):
+        key = bucket.new_key(str(i))
+        key.set_contents_from_string('bar')
+    # wait for sync
+    zone_bucket_checkpoint(ps_zones[0].zone, zones[0].zone, bucket_name)
+
+    # get the create events from the subscription
+    result, _ = sub_conf.get_events()
+    events = json.loads(result)
+    for event in events['events']:
+        log.debug('Event: objname: "' + str(event['info']['key']['name']) + '" type: "' + str(event['event']) + '"')
+    keys = list(bucket.list())
+    # TODO: use exact match
+    verify_events_by_elements(events, keys, exact_match=False)
+    # delete objects from the bucket
+    for key in bucket.list():
+        key.delete()
+    # wait for sync
+    zone_meta_checkpoint(ps_zones[0].zone)
+    zone_bucket_checkpoint(ps_zones[0].zone, zones[0].zone, bucket_name)
+
+    # get the delete events from the subscriptions
+    result, _ = sub_conf.get_events()
+    for event in events['events']:
+        log.debug('Event: objname: "' + str(event['info']['key']['name']) + '" type: "' + str(event['event']) + '"')
+    # TODO: check deletions
+    # TODO: use exact match
+    # verify_events_by_elements(events, keys, exact_match=False, deletions=True)
+    # we should see the creations as well as the deletions
+    # delete subscription
+    _, status = sub_conf.del_config()
+    assert_equal(status/100, 2)
+    result, status = sub_conf.get_config()
+    parsed_result = json.loads(result)
+    assert_equal(parsed_result['topic'], '')
+    # TODO should return 404
+    # assert_equal(status, 404)
+
+    # cleanup
+    notification_conf.del_config()
+    topic_conf.del_config()
+    zones[0].delete_bucket(bucket_name)
+
+
+def test_ps_event_type_subscription():
+    """ test subscriptions for different events """
+    zones, ps_zones = init_env()
+    bucket_name = gen_bucket_name()
+
+    # create topic for objects creation
+    topic_create_name = bucket_name+TOPIC_SUFFIX+'_create'
+    topic_create_conf = PSTopic(ps_zones[0].conn, topic_create_name)
+    topic_create_conf.set_config()
+    # create topic for objects deletion
+    topic_delete_name = bucket_name+TOPIC_SUFFIX+'_delete'
+    topic_delete_conf = PSTopic(ps_zones[0].conn, topic_delete_name)
+    topic_delete_conf.set_config()
+    # create topic for all events
+    topic_name = bucket_name+TOPIC_SUFFIX+'_all'
+    topic_conf = PSTopic(ps_zones[0].conn, topic_name)
+    topic_conf.set_config()
+    # create bucket on the first of the rados zones
+    bucket = zones[0].create_bucket(bucket_name)
+    # wait for sync
+    zone_meta_checkpoint(ps_zones[0].zone)
+    zone_bucket_checkpoint(ps_zones[0].zone, zones[0].zone, bucket_name)
+    # create notifications for objects creation
+    notification_create_conf = PSNotification(ps_zones[0].conn, bucket_name,
+                                              topic_create_name, "OBJECT_CREATE")
+    _, status = notification_create_conf.set_config()
+    assert_equal(status/100, 2)
+    # create notifications for objects deletion
+    notification_delete_conf = PSNotification(ps_zones[0].conn, bucket_name,
+                                              topic_delete_name, "OBJECT_DELETE")
+    _, status = notification_delete_conf.set_config()
+    assert_equal(status/100, 2)
+    # create notifications for all events
+    notification_conf = PSNotification(ps_zones[0].conn, bucket_name,
+                                       topic_name, "OBJECT_DELETE,OBJECT_CREATE")
+    _, status = notification_conf.set_config()
+    assert_equal(status/100, 2)
+    # create subscription for objects creation
+    sub_create_conf = PSSubscription(ps_zones[0].conn, bucket_name+SUB_SUFFIX+'_create',
+                                     topic_create_name)
+    _, status = sub_create_conf.set_config()
+    assert_equal(status/100, 2)
+    # create subscription for objects deletion
+    sub_delete_conf = PSSubscription(ps_zones[0].conn, bucket_name+SUB_SUFFIX+'_delete',
+                                     topic_delete_name)
+    _, status = sub_delete_conf.set_config()
+    assert_equal(status/100, 2)
+    # create subscription for all events
+    sub_conf = PSSubscription(ps_zones[0].conn, bucket_name+SUB_SUFFIX+'_all',
+                              topic_name)
+    _, status = sub_conf.set_config()
+    assert_equal(status/100, 2)
+    # create objects in the bucket
+    number_of_objects = 10
+    for i in range(number_of_objects):
+        key = bucket.new_key(str(i))
+        key.set_contents_from_string('bar')
+    # wait for sync
+    zone_bucket_checkpoint(ps_zones[0].zone, zones[0].zone, bucket_name)
+
+    # get the events from the creation subscription
+    result, _ = sub_create_conf.get_events()
+    events = json.loads(result)
+    for event in events['events']:
+        log.debug('Event (OBJECT_CREATE): objname: "' + str(event['info']['key']['name']) +
+                  '" type: "' + str(event['event']) + '"')
+    keys = list(bucket.list())
+    # TODO: use exact match
+    verify_events_by_elements(events, keys, exact_match=False)
+    # get the events from the deletions subscription
+    result, _ = sub_delete_conf.get_events()
+    events = json.loads(result)
+    for event in events['events']:
+        log.debug('Event (OBJECT_DELETE): objname: "' + str(event['info']['key']['name']) +
+                  '" type: "' + str(event['event']) + '"')
+    assert_equal(len(events['events']), 0)
+    # get the events from the all events subscription
+    result, _ = sub_conf.get_events()
+    events = json.loads(result)
+    for event in events['events']:
+        log.debug('Event (OBJECT_CREATE,OBJECT_DELETE): objname: "' +
+                  str(event['info']['key']['name']) + '" type: "' + str(event['event']) + '"')
+    # TODO: use exact match
+    verify_events_by_elements(events, keys, exact_match=False)
+    # delete objects from the bucket
+    for key in bucket.list():
+        key.delete()
+    # wait for sync
+    zone_bucket_checkpoint(ps_zones[0].zone, zones[0].zone, bucket_name)
+    log.debug("Event (OBJECT_DELETE) synced")
+
+    # get the events from the creations subscription
+    result, _ = sub_create_conf.get_events()
+    events = json.loads(result)
+    for event in events['events']:
+        log.debug('Event (OBJECT_CREATE): objname: "' + str(event['info']['key']['name']) +
+                  '" type: "' + str(event['event']) + '"')
+    # deletions should not change the creation events
+    # TODO: use exact match
+    verify_events_by_elements(events, keys, exact_match=False)
+    # get the events from the deletions subscription
+    result, _ = sub_delete_conf.get_events()
+    events = json.loads(result)
+    for event in events['events']:
+        log.debug('Event (OBJECT_DELETE): objname: "' + str(event['info']['key']['name']) +
+                  '" type: "' + str(event['event']) + '"')
+    # only deletions should be listed here
+    # TODO: use exact match
+    verify_events_by_elements(events, keys, exact_match=False, deletions=True)
+    # get the events from the all events subscription
+    result, _ = sub_create_conf.get_events()
+    events = json.loads(result)
+    for event in events['events']:
+        log.debug('Event (OBJECT_CREATE,OBJECT_DELETE): objname: "' + str(event['info']['key']['name']) +
+                  '" type: "' + str(event['event']) + '"')
+    # both deletions and creations should be here
+    # TODO: use exact match
+    verify_events_by_elements(events, keys, exact_match=False, deletions=False)
+    # verify_events_by_elements(events, keys, exact_match=False, deletions=True)
+    # TODO: (1) test deletions (2) test overall number of events
+
+    # test subscription deletion when topic is specified
+    _, status = sub_create_conf.del_config(topic=True)
+    assert_equal(status/100, 2)
+    _, status = sub_delete_conf.del_config(topic=True)
+    assert_equal(status/100, 2)
+    _, status = sub_conf.del_config(topic=True)
+    assert_equal(status/100, 2)
+
+    # cleanup
+    notification_create_conf.del_config()
+    notification_delete_conf.del_config()
+    notification_conf.del_config()
+    topic_create_conf.del_config()
+    topic_delete_conf.del_config()
+    topic_conf.del_config()
+    zones[0].delete_bucket(bucket_name)
+
+
+def test_ps_event_fetching():
+    """ test incremental fetching of events from a subscription """
+    zones, ps_zones = init_env()
+    bucket_name = gen_bucket_name()
+    topic_name = bucket_name+TOPIC_SUFFIX
+
+    # create topic
+    topic_conf = PSTopic(ps_zones[0].conn, topic_name)
+    topic_conf.set_config()
+    # create bucket on the first of the rados zones
+    bucket = zones[0].create_bucket(bucket_name)
+    # wait for sync
+    zone_meta_checkpoint(ps_zones[0].zone)
+    # create notifications
+    notification_conf = PSNotification(ps_zones[0].conn, bucket_name,
+                                       topic_name)
+    _, status = notification_conf.set_config()
+    assert_equal(status/100, 2)
+    # create subscription
+    sub_conf = PSSubscription(ps_zones[0].conn, bucket_name+SUB_SUFFIX,
+                              topic_name)
+    _, status = sub_conf.set_config()
+    assert_equal(status/100, 2)
+    # create objects in the bucket
+    number_of_objects = 100
+    for i in range(number_of_objects):
+        key = bucket.new_key(str(i))
+        key.set_contents_from_string('bar')
+    # wait for sync
+    zone_bucket_checkpoint(ps_zones[0].zone, zones[0].zone, bucket_name)
+    max_events = 15
+    total_events_count = 0
+    next_marker = None
+    all_events = []
+    while True:
+        # get the events from the subscription
+        result, _ = sub_conf.get_events(max_events, next_marker)
+        events = json.loads(result)
+        total_events_count += len(events['events'])
+        all_events.extend(events['events'])
+        next_marker = events['next_marker']
+        for event in events['events']:
+            log.debug('Event: objname: "' + str(event['info']['key']['name']) + '" type: "' + str(event['event']) + '"')
+        if next_marker == '':
+            break
+    keys = list(bucket.list())
+    # TODO: use exact match
+    verify_events_by_elements({'events': all_events}, keys, exact_match=False)
+
+    # cleanup
+    sub_conf.del_config()
+    notification_conf.del_config()
+    topic_conf.del_config()
+    for key in bucket.list():
+        key.delete()
+    zones[0].delete_bucket(bucket_name)
+
+
+def test_ps_event_acking():
+    """ test acking of some events in a subscription """
+    zones, ps_zones = init_env()
+    bucket_name = gen_bucket_name()
+    topic_name = bucket_name+TOPIC_SUFFIX
+
+    # create topic
+    topic_conf = PSTopic(ps_zones[0].conn, topic_name)
+    topic_conf.set_config()
+    # create bucket on the first of the rados zones
+    bucket = zones[0].create_bucket(bucket_name)
+    # wait for sync
+    zone_meta_checkpoint(ps_zones[0].zone)
+    # create notifications
+    notification_conf = PSNotification(ps_zones[0].conn, bucket_name,
+                                       topic_name)
+    _, status = notification_conf.set_config()
+    assert_equal(status/100, 2)
+    # create subscription
+    sub_conf = PSSubscription(ps_zones[0].conn, bucket_name+SUB_SUFFIX,
+                              topic_name)
+    _, status = sub_conf.set_config()
+    assert_equal(status/100, 2)
+    # create objects in the bucket
+    number_of_objects = 10
+    for i in range(number_of_objects):
+        key = bucket.new_key(str(i))
+        key.set_contents_from_string('bar')
+    # wait for sync
+    zone_bucket_checkpoint(ps_zones[0].zone, zones[0].zone, bucket_name)
+
+    # get the create events from the subscription
+    result, _ = sub_conf.get_events()
+    events = json.loads(result)
+    original_number_of_events = len(events)
+    for event in events['events']:
+        log.debug('Event (before ack)  id: "' + str(event['id']) + '"')
+    keys = list(bucket.list())
+    # TODO: use exact match
+    verify_events_by_elements(events, keys, exact_match=False)
+    # ack half of the  events
+    events_to_ack = number_of_objects/2
+    for event in events['events']:
+        if events_to_ack == 0:
+            break
+        _, status = sub_conf.ack_events(event['id'])
+        assert_equal(status/100, 2)
+        events_to_ack -= 1
+
+    # verify that acked events are gone
+    result, _ = sub_conf.get_events()
+    events = json.loads(result)
+    for event in events['events']:
+        log.debug('Event (after ack) id: "' + str(event['id']) + '"')
+    assert len(events) >= (original_number_of_events - number_of_objects/2)
+
+    # cleanup
+    sub_conf.del_config()
+    notification_conf.del_config()
+    topic_conf.del_config()
+    for key in bucket.list():
+        key.delete()
+    zones[0].delete_bucket(bucket_name)
+
+
+def test_ps_creation_triggers():
+    """ test object creation notifications in using put/copy/post """
+    zones, ps_zones = init_env()
+    bucket_name = gen_bucket_name()
+    topic_name = bucket_name+TOPIC_SUFFIX
+
+    # create topic
+    topic_conf = PSTopic(ps_zones[0].conn, topic_name)
+    topic_conf.set_config()
+    # create bucket on the first of the rados zones
+    bucket = zones[0].create_bucket(bucket_name)
+    # wait for sync
+    zone_meta_checkpoint(ps_zones[0].zone)
+    # create notifications
+    notification_conf = PSNotification(ps_zones[0].conn, bucket_name,
+                                       topic_name)
+    _, status = notification_conf.set_config()
+    assert_equal(status/100, 2)
+    # create subscription
+    sub_conf = PSSubscription(ps_zones[0].conn, bucket_name+SUB_SUFFIX,
+                              topic_name)
+    _, status = sub_conf.set_config()
+    assert_equal(status/100, 2)
+    # create objects in the bucket using PUT
+    key = bucket.new_key('put')
+    key.set_contents_from_string('bar')
+    # create objects in the bucket using COPY
+    bucket.copy_key('copy', bucket.name, key.name)
+    # create objects in the bucket using multi-part upload
+    fp = tempfile.TemporaryFile(mode='w')
+    fp.write('bar')
+    fp.close()
+    uploader = bucket.initiate_multipart_upload('multipart')
+    fp = tempfile.NamedTemporaryFile(mode='r')
+    uploader.upload_part_from_file(fp, 1)
+    uploader.complete_upload()
+    fp.close()
+    # wait for sync
+    zone_bucket_checkpoint(ps_zones[0].zone, zones[0].zone, bucket_name)
+
+    # get the create events from the subscription
+    result, _ = sub_conf.get_events()
+    events = json.loads(result)
+    for event in events['events']:
+        log.debug('Event key: "' + str(event['info']['key']['name']) + '" type: "' + str(event['event']) + '"')
+
+    # TODO: verify the specific 3 keys: 'put', 'copy' and 'multipart'
+    assert len(events['events']) >= 3
+    # cleanup
+    sub_conf.del_config()
+    notification_conf.del_config()
+    topic_conf.del_config()
+    for key in bucket.list():
+        key.delete()
+    zones[0].delete_bucket(bucket_name)
+
+
+def test_ps_s3_creation_triggers_on_master():
+    """ test object creation s3 notifications in using put/copy/post on master"""
+    if skip_push_tests:
+        return SkipTest("PubSub push tests don't run in teuthology")
+    hostname = get_ip()
+    proc = init_rabbitmq()
+    if proc is  None:
+        return SkipTest('end2end amqp tests require rabbitmq-server installed')
+    zones, _  = init_env(require_ps=False)
+    realm = get_realm()
+    zonegroup = realm.master_zonegroup()
+    
+    # create bucket
+    bucket_name = gen_bucket_name()
+    bucket = zones[0].create_bucket(bucket_name)
+    topic_name = bucket_name + TOPIC_SUFFIX
+
+    # start amqp receiver
+    exchange = 'ex1'
+    task, receiver = create_amqp_receiver_thread(exchange, topic_name)
+    task.start()
+
+    # create s3 topic
+    endpoint_address = 'amqp://' + hostname
+    endpoint_args = 'push-endpoint='+endpoint_address+'&amqp-exchange=' + exchange +'&amqp-ack-level=broker'
+    topic_conf = PSTopicS3(zones[0].conn, topic_name, zonegroup.name, endpoint_args=endpoint_args)
+    topic_arn = topic_conf.set_config()
+    # create s3 notification
+    notification_name = bucket_name + NOTIFICATION_SUFFIX
+    topic_conf_list = [{'Id': notification_name,'TopicArn': topic_arn,
+                        'Events': ['s3:ObjectCreated:Put', 's3:ObjectCreated:Copy']
+                       }]
+
+    s3_notification_conf = PSNotificationS3(zones[0].conn, bucket_name, topic_conf_list)
+    response, status = s3_notification_conf.set_config()
+    assert_equal(status/100, 2)
+
+    # create objects in the bucket using PUT
+    key = bucket.new_key('put')
+    key.set_contents_from_string('bar')
+    # create objects in the bucket using COPY
+    bucket.copy_key('copy', bucket.name, key.name)
+    # create objects in the bucket using multi-part upload
+    fp = tempfile.TemporaryFile(mode='w')
+    fp.write('bar')
+    fp.close()
+    uploader = bucket.initiate_multipart_upload('multipart')
+    fp = tempfile.NamedTemporaryFile(mode='r')
+    uploader.upload_part_from_file(fp, 1)
+    uploader.complete_upload()
+    fp.close()
+
+    print('wait for 5sec for the messages...')
+    time.sleep(5)
+
+    # check amqp receiver
+    keys = list(bucket.list())
+    receiver.verify_s3_events(keys, exact_match=True)
+
+    # cleanup
+    stop_amqp_receiver(receiver, task)
+    s3_notification_conf.del_config()
+    topic_conf.del_config()
+    for key in bucket.list():
+        key.delete()
+    # delete the bucket
+    zones[0].delete_bucket(bucket_name)
+    clean_rabbitmq(proc)
+
+
+def test_ps_s3_multipart_on_master():
+    """ test multipart object upload on master"""
+    if skip_push_tests:
+        return SkipTest("PubSub push tests don't run in teuthology")
+    hostname = get_ip()
+    proc = init_rabbitmq()
+    if proc is  None:
+        return SkipTest('end2end amqp tests require rabbitmq-server installed')
+    zones, _  = init_env(require_ps=False)
+    realm = get_realm()
+    zonegroup = realm.master_zonegroup()
+    
+    # create bucket
+    bucket_name = gen_bucket_name()
+    bucket = zones[0].create_bucket(bucket_name)
+    topic_name = bucket_name + TOPIC_SUFFIX
+
+    # start amqp receivers
+    exchange = 'ex1'
+    task1, receiver1 = create_amqp_receiver_thread(exchange, topic_name+'_1')
+    task1.start()
+    task2, receiver2 = create_amqp_receiver_thread(exchange, topic_name+'_2')
+    task2.start()
+    task3, receiver3 = create_amqp_receiver_thread(exchange, topic_name+'_3')
+    task3.start()
+
+    # create s3 topics
+    endpoint_address = 'amqp://' + hostname
+    endpoint_args = 'push-endpoint=' + endpoint_address + '&amqp-exchange=' + exchange + '&amqp-ack-level=broker'
+    topic_conf1 = PSTopicS3(zones[0].conn, topic_name+'_1', zonegroup.name, endpoint_args=endpoint_args)
+    topic_arn1 = topic_conf1.set_config()
+    topic_conf2 = PSTopicS3(zones[0].conn, topic_name+'_2', zonegroup.name, endpoint_args=endpoint_args)
+    topic_arn2 = topic_conf2.set_config()
+    topic_conf3 = PSTopicS3(zones[0].conn, topic_name+'_3', zonegroup.name, endpoint_args=endpoint_args)
+    topic_arn3 = topic_conf3.set_config()
+
+    # create s3 notifications
+    notification_name = bucket_name + NOTIFICATION_SUFFIX
+    topic_conf_list = [{'Id': notification_name+'_1', 'TopicArn': topic_arn1,
+                        'Events': ['s3:ObjectCreated:*']
+                       },
+                       {'Id': notification_name+'_2', 'TopicArn': topic_arn2,
+                        'Events': ['s3:ObjectCreated:Post']
+                       },
+                       {'Id': notification_name+'_3', 'TopicArn': topic_arn3,
+                        'Events': ['s3:ObjectCreated:CompleteMultipartUpload']
+                       }]
+    s3_notification_conf = PSNotificationS3(zones[0].conn, bucket_name, topic_conf_list)
+    response, status = s3_notification_conf.set_config()
+    assert_equal(status/100, 2)
+
+    # create objects in the bucket using multi-part upload
+    fp = tempfile.NamedTemporaryFile(mode='w+b')
+    content = bytearray(os.urandom(1024*1024))
+    fp.write(content)
+    fp.flush()
+    fp.seek(0)
+    uploader = bucket.initiate_multipart_upload('multipart')
+    uploader.upload_part_from_file(fp, 1)
+    uploader.complete_upload()
+    fp.close()
+
+    print('wait for 5sec for the messages...')
+    time.sleep(5)
+
+    # check amqp receiver
+    events = receiver1.get_and_reset_events()
+    assert_equal(len(events), 3)
+
+    events = receiver2.get_and_reset_events()
+    assert_equal(len(events), 1)
+    assert_equal(events[0]['Records'][0]['eventName'], 's3:ObjectCreated:Post')
+    assert_equal(events[0]['Records'][0]['s3']['configurationId'], notification_name+'_2')
+
+    events = receiver3.get_and_reset_events()
+    assert_equal(len(events), 1)
+    assert_equal(events[0]['Records'][0]['eventName'], 's3:ObjectCreated:CompleteMultipartUpload')
+    assert_equal(events[0]['Records'][0]['s3']['configurationId'], notification_name+'_3')
+
+    # cleanup
+    stop_amqp_receiver(receiver1, task1)
+    stop_amqp_receiver(receiver2, task2)
+    stop_amqp_receiver(receiver3, task3)
+    s3_notification_conf.del_config()
+    topic_conf1.del_config()
+    topic_conf2.del_config()
+    topic_conf3.del_config()
+    for key in bucket.list():
+        key.delete()
+    # delete the bucket
+    zones[0].delete_bucket(bucket_name)
+    clean_rabbitmq(proc)
+
+
+def test_ps_versioned_deletion():
+    """ test notification of deletion markers """
+    zones, ps_zones = init_env()
+    bucket_name = gen_bucket_name()
+    topic_name = bucket_name+TOPIC_SUFFIX
+
+    # create topics
+    topic_conf1 = PSTopic(ps_zones[0].conn, topic_name+'_1')
+    _, status = topic_conf1.set_config()
+    assert_equal(status/100, 2)
+    topic_conf2 = PSTopic(ps_zones[0].conn, topic_name+'_2')
+    _, status = topic_conf2.set_config()
+    assert_equal(status/100, 2)
+    
+    # create bucket on the first of the rados zones
+    bucket = zones[0].create_bucket(bucket_name)
+    bucket.configure_versioning(True)
+    
+    # wait for sync
+    zone_meta_checkpoint(ps_zones[0].zone)
+    
+    # create notifications
+    event_type1 = 'OBJECT_DELETE'
+    notification_conf1 = PSNotification(ps_zones[0].conn, bucket_name,
+                                        topic_name+'_1',
+                                        event_type1)
+    _, status = notification_conf1.set_config()
+    assert_equal(status/100, 2)
+    event_type2 = 'DELETE_MARKER_CREATE'
+    notification_conf2 = PSNotification(ps_zones[0].conn, bucket_name,
+                                        topic_name+'_2',
+                                        event_type2)
+    _, status = notification_conf2.set_config()
+    assert_equal(status/100, 2)
+    
+    # create subscriptions
+    sub_conf1 = PSSubscription(ps_zones[0].conn, bucket_name+SUB_SUFFIX+'_1',
+                               topic_name+'_1')
+    _, status = sub_conf1.set_config()
+    assert_equal(status/100, 2)
+    sub_conf2 = PSSubscription(ps_zones[0].conn, bucket_name+SUB_SUFFIX+'_2',
+                               topic_name+'_2')
+    _, status = sub_conf2.set_config()
+    assert_equal(status/100, 2)
+    
+    # create objects in the bucket
+    key = bucket.new_key('foo')
+    key.set_contents_from_string('bar')
+    v1 = key.version_id
+    key.set_contents_from_string('kaboom')
+    v2 = key.version_id
+    # create deletion marker
+    delete_marker_key = bucket.delete_key(key.name)
+    
+    # wait for sync
+    zone_bucket_checkpoint(ps_zones[0].zone, zones[0].zone, bucket_name)
+
+    # delete the deletion marker
+    delete_marker_key.delete()
+    # delete versions
+    bucket.delete_key(key.name, version_id=v2)
+    bucket.delete_key(key.name, version_id=v1)
+    
+    # wait for sync
+    zone_bucket_checkpoint(ps_zones[0].zone, zones[0].zone, bucket_name)
+
+    # get the delete events from the subscription
+    result, _ = sub_conf1.get_events()
+    events = json.loads(result)
+    for event in events['events']:
+        log.debug('Event key: "' + str(event['info']['key']['name']) + '" type: "' + str(event['event']) + '"')
+        assert_equal(str(event['event']), event_type1)
+
+    result, _ = sub_conf2.get_events()
+    events = json.loads(result)
+    for event in events['events']:
+        log.debug('Event key: "' + str(event['info']['key']['name']) + '" type: "' + str(event['event']) + '"')
+        assert_equal(str(event['event']), event_type2)
+
+    # cleanup
+    # follwing is needed for the cleanup in the case of 3-zones
+    # see: http://tracker.ceph.com/issues/39142
+    realm = get_realm()
+    zonegroup = realm.master_zonegroup()
+    zonegroup_conns = ZonegroupConns(zonegroup)
+    try:
+        zonegroup_bucket_checkpoint(zonegroup_conns, bucket_name)
+        zones[0].delete_bucket(bucket_name)
+    except:
+        log.debug('zonegroup_bucket_checkpoint failed, cannot delete bucket')
+    sub_conf1.del_config()
+    sub_conf2.del_config()
+    notification_conf1.del_config()
+    notification_conf2.del_config()
+    topic_conf1.del_config()
+    topic_conf2.del_config()
+
+
+def test_ps_s3_metadata_on_master():
+    """ test s3 notification of metadata on master """
+    if skip_push_tests:
+        return SkipTest("PubSub push tests don't run in teuthology")
+    hostname = get_ip()
+    proc = init_rabbitmq()
+    if proc is  None:
+        return SkipTest('end2end amqp tests require rabbitmq-server installed')
+    zones, _  = init_env(require_ps=False)
+    realm = get_realm()
+    zonegroup = realm.master_zonegroup()
+    
+    # create bucket
+    bucket_name = gen_bucket_name()
+    bucket = zones[0].create_bucket(bucket_name)
+    topic_name = bucket_name + TOPIC_SUFFIX
+
+    # start amqp receiver
+    exchange = 'ex1'
+    task, receiver = create_amqp_receiver_thread(exchange, topic_name)
+    task.start()
+
+    # create s3 topic
+    endpoint_address = 'amqp://' + hostname
+    endpoint_args = 'push-endpoint='+endpoint_address+'&amqp-exchange=' + exchange +'&amqp-ack-level=broker'
+    topic_conf = PSTopicS3(zones[0].conn, topic_name, zonegroup.name, endpoint_args=endpoint_args)
+    topic_arn = topic_conf.set_config()
+    # create s3 notification
+    notification_name = bucket_name + NOTIFICATION_SUFFIX
+    meta_key = 'meta1'
+    meta_value = 'This is my metadata value'
+    meta_prefix = 'x-amz-meta-'
+    topic_conf_list = [{'Id': notification_name,'TopicArn': topic_arn,
+        'Events': ['s3:ObjectCreated:*', 's3:ObjectRemoved:*'],
+        'Filter': {
+            'Metadata': {
+                'FilterRules': [{'Name': meta_prefix+meta_key, 'Value': meta_value}]
+            }
+        }
+    }]
+
+    s3_notification_conf = PSNotificationS3(zones[0].conn, bucket_name, topic_conf_list)
+    response, status = s3_notification_conf.set_config()
+    assert_equal(status/100, 2)
+
+    # create objects in the bucket
+    key_name = 'foo'
+    key = bucket.new_key(key_name)
+    key.set_metadata(meta_key, meta_value)
+    key.set_contents_from_string('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')
+    
+    # create objects in the bucket using COPY
+    bucket.copy_key('copy_of_foo', bucket.name, key.name)
+    # create objects in the bucket using multi-part upload
+    fp = tempfile.TemporaryFile(mode='w')
+    fp.write('bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb')
+    fp.close()
+    uploader = bucket.initiate_multipart_upload('multipart_foo', 
+            metadata={meta_key: meta_value})
+    fp = tempfile.TemporaryFile(mode='r')
+    uploader.upload_part_from_file(fp, 1)
+    uploader.complete_upload()
+    fp.close()
+    print('wait for 5sec for the messages...')
+    time.sleep(5)
+    # check amqp receiver
+    event_count = 0
+    for event in receiver.get_and_reset_events():
+        s3_event = event['Records'][0]['s3']
+        assert_equal(s3_event['object']['metadata'][0]['key'], meta_prefix+meta_key)
+        assert_equal(s3_event['object']['metadata'][0]['val'], meta_value)
+        event_count +=1
+
+    # only PUT and POST has the metadata value
+    assert_equal(event_count, 2)
+
+    # delete objects
+    for key in bucket.list():
+        key.delete()
+    print('wait for 5sec for the messages...')
+    time.sleep(5)
+    # check amqp receiver
+    event_count = 0
+    for event in receiver.get_and_reset_events():
+        s3_event = event['Records'][0]['s3']
+        assert_equal(s3_event['object']['metadata'][0]['key'], meta_prefix+meta_key)
+        assert_equal(s3_event['object']['metadata'][0]['val'], meta_value)
+        event_count +=1
+
+    # all 3 object has metadata when deleted
+    assert_equal(event_count, 3)
+
+    # cleanup
+    stop_amqp_receiver(receiver, task)
+    s3_notification_conf.del_config()
+    topic_conf.del_config()
+    # delete the bucket
+    zones[0].delete_bucket(bucket_name)
+    clean_rabbitmq(proc)
+
+
+def test_ps_s3_tags_on_master():
+    """ test s3 notification of tags on master """
+    if skip_push_tests:
+        return SkipTest("PubSub push tests don't run in teuthology")
+    hostname = get_ip()
+    proc = init_rabbitmq()
+    if proc is  None:
+        return SkipTest('end2end amqp tests require rabbitmq-server installed')
+    zones, _  = init_env(require_ps=False)
+    realm = get_realm()
+    zonegroup = realm.master_zonegroup()
+    
+    # create bucket
+    bucket_name = gen_bucket_name()
+    bucket = zones[0].create_bucket(bucket_name)
+    topic_name = bucket_name + TOPIC_SUFFIX
+
+    # start amqp receiver
+    exchange = 'ex1'
+    task, receiver = create_amqp_receiver_thread(exchange, topic_name)
+    task.start()
+
+    # create s3 topic
+    endpoint_address = 'amqp://' + hostname
+    endpoint_args = 'push-endpoint='+endpoint_address+'&amqp-exchange=' + exchange +'&amqp-ack-level=broker'
+    topic_conf = PSTopicS3(zones[0].conn, topic_name, zonegroup.name, endpoint_args=endpoint_args)
+    topic_arn = topic_conf.set_config()
+    # create s3 notification
+    notification_name = bucket_name + NOTIFICATION_SUFFIX
+    topic_conf_list = [{'Id': notification_name,'TopicArn': topic_arn,
+        'Events': ['s3:ObjectCreated:*', 's3:ObjectRemoved:*'],
+        'Filter': {
+            'Tags': {
+                'FilterRules': [{'Name': 'hello', 'Value': 'world'}]
+            }
+        }
+    }]
+
+    s3_notification_conf = PSNotificationS3(zones[0].conn, bucket_name, topic_conf_list)
+    response, status = s3_notification_conf.set_config()
+    assert_equal(status/100, 2)
+
+    # create objects in the bucket with tags
+    tags = 'hello=world&ka=boom'
+    key_name1 = 'key1'
+    put_object_tagging(zones[0].conn, bucket_name, key_name1, tags)
+    tags = 'foo=bar&ka=boom'
+    key_name2 = 'key2'
+    put_object_tagging(zones[0].conn, bucket_name, key_name2, tags)
+    key_name3 = 'key3'
+    key = bucket.new_key(key_name3)
+    key.set_contents_from_string('bar')
+    # create objects in the bucket using COPY
+    bucket.copy_key('copy_of_'+key_name1, bucket.name, key_name1)
+    print('wait for 5sec for the messages...')
+    time.sleep(5)
+    expected_tags = [{'val': 'world', 'key': 'hello'}, {'val': 'boom', 'key': 'ka'}]
+    # check amqp receiver
+    for event in receiver.get_and_reset_events():
+        obj_tags =  event['Records'][0]['s3']['object']['tags']
+        assert_equal(obj_tags[0], expected_tags[0])
+
+    # delete the objects
+    for key in bucket.list():
+        key.delete()
+    print('wait for 5sec for the messages...')
+    time.sleep(5)
+    # check amqp receiver
+    for event in receiver.get_and_reset_events():
+        obj_tags =  event['Records'][0]['s3']['object']['tags']
+        assert_equal(obj_tags[0], expected_tags[0])
+
+    # cleanup
+    stop_amqp_receiver(receiver, task)
+    s3_notification_conf.del_config()
+    topic_conf.del_config()
+    # delete the bucket
+    zones[0].delete_bucket(bucket_name)
+    clean_rabbitmq(proc)
+
+
+def test_ps_s3_versioning_on_master():
+    """ test s3 notification of object versions """
+    if skip_push_tests:
+        return SkipTest("PubSub push tests don't run in teuthology")
+    hostname = get_ip()
+    proc = init_rabbitmq()
+    if proc is None:
+        return SkipTest('end2end amqp tests require rabbitmq-server installed')
+    master_zone, _ = init_env(require_ps=False)
+    realm = get_realm()
+    zonegroup = realm.master_zonegroup()
+
+    # create bucket
+    bucket_name = gen_bucket_name()
+    bucket = master_zone.create_bucket(bucket_name)
+    bucket.configure_versioning(True)
+    topic_name = bucket_name + TOPIC_SUFFIX
+
+    # start amqp receiver
+    exchange = 'ex1'
+    task, receiver = create_amqp_receiver_thread(exchange, topic_name)
+    task.start()
+
+    # create s3 topic
+    endpoint_address = 'amqp://' + hostname
+    endpoint_args = 'push-endpoint='+endpoint_address+'&amqp-exchange=' + exchange +'&amqp-ack-level=broker'
+    topic_conf = PSTopicS3(master_zone.conn, topic_name, zonegroup.name, endpoint_args=endpoint_args)
+    topic_arn = topic_conf.set_config()
+    # create notification
+    notification_name = bucket_name + NOTIFICATION_SUFFIX
+    topic_conf_list = [{'Id': notification_name, 'TopicArn': topic_arn,
+                        'Events': []
+                       }]
+    s3_notification_conf = PSNotificationS3(master_zone.conn, bucket_name, topic_conf_list)
+    _, status = s3_notification_conf.set_config()
+    assert_equal(status/100, 2)
+
+    # create objects in the bucket
+    key_value = 'foo'
+    key = bucket.new_key(key_value)
+    key.set_contents_from_string('hello')
+    ver1 = key.version_id
+    key.set_contents_from_string('world')
+    ver2 = key.version_id
+
+    print('wait for 5sec for the messages...')
+    time.sleep(5)
+
+    # check amqp receiver
+    events = receiver.get_and_reset_events()
+    num_of_versions = 0
+    for event_list in events:
+        for event in event_list['Records']:
+            assert_equal(event['s3']['object']['key'], key_value)
+            version = event['s3']['object']['versionId']
+            num_of_versions += 1
+            if version not in (ver1, ver2):
+                print('version mismatch: '+version+' not in: ('+ver1+', '+ver2+')')
+                assert_equal(1, 0)
+            else:
+                print('version ok: '+version+' in: ('+ver1+', '+ver2+')')
+
+    assert_equal(num_of_versions, 2)
+
+    # cleanup
+    stop_amqp_receiver(receiver, task)
+    s3_notification_conf.del_config()
+    topic_conf.del_config()
+    # delete the bucket
+    bucket.delete_key(key.name, version_id=ver2)
+    bucket.delete_key(key.name, version_id=ver1)
+    master_zone.delete_bucket(bucket_name)
+    clean_rabbitmq(proc)
+
+
+def test_ps_s3_versioned_deletion_on_master():
+    """ test s3 notification of deletion markers on master """
+    if skip_push_tests:
+        return SkipTest("PubSub push tests don't run in teuthology")
+    hostname = get_ip()
+    proc = init_rabbitmq()
+    if proc is  None:
+        return SkipTest('end2end amqp tests require rabbitmq-server installed')
+    zones, _  = init_env(require_ps=False)
+    realm = get_realm()
+    zonegroup = realm.master_zonegroup()
+    
+    # create bucket
+    bucket_name = gen_bucket_name()
+    bucket = zones[0].create_bucket(bucket_name)
+    bucket.configure_versioning(True)
+    topic_name = bucket_name + TOPIC_SUFFIX
+
+    # start amqp receiver
+    exchange = 'ex1'
+    task, receiver = create_amqp_receiver_thread(exchange, topic_name)
+    task.start()
+
+    # create s3 topic
+    endpoint_address = 'amqp://' + hostname
+    endpoint_args = 'push-endpoint='+endpoint_address+'&amqp-exchange=' + exchange +'&amqp-ack-level=broker'
+    topic_conf = PSTopicS3(zones[0].conn, topic_name, zonegroup.name, endpoint_args=endpoint_args)
+    topic_arn = topic_conf.set_config()
+    # create s3 notification
+    notification_name = bucket_name + NOTIFICATION_SUFFIX
+    topic_conf_list = [{'Id': notification_name+'_1', 'TopicArn': topic_arn,
+                        'Events': ['s3:ObjectRemoved:*']
+                       },
+                       {'Id': notification_name+'_2', 'TopicArn': topic_arn,
+                        'Events': ['s3:ObjectRemoved:DeleteMarkerCreated']
+                       },
+                       {'Id': notification_name+'_3', 'TopicArn': topic_arn,
+                         'Events': ['s3:ObjectRemoved:Delete']
+                       }]
+    s3_notification_conf = PSNotificationS3(zones[0].conn, bucket_name, topic_conf_list)
+    response, status = s3_notification_conf.set_config()
+    assert_equal(status/100, 2)
+
+    # create objects in the bucket
+    key = bucket.new_key('foo')
+    key.set_contents_from_string('bar')
+    v1 = key.version_id
+    key.set_contents_from_string('kaboom')
+    v2 = key.version_id
+    # create delete marker (non versioned deletion)
+    delete_marker_key = bucket.delete_key(key.name)
+    
+    time.sleep(1)
+    
+    # versioned deletion
+    bucket.delete_key(key.name, version_id=v2)
+    bucket.delete_key(key.name, version_id=v1)
+    delete_marker_key.delete()
+
+    print('wait for 5sec for the messages...')
+    time.sleep(5)
+
+    # check amqp receiver
+    events = receiver.get_and_reset_events()
+    delete_events = 0
+    delete_marker_create_events = 0
+    for event_list in events:
+        for event in event_list['Records']:
+            if event['eventName'] == 's3:ObjectRemoved:Delete':
+                delete_events += 1
+                assert event['s3']['configurationId'] in [notification_name+'_1', notification_name+'_3']
+            if event['eventName'] == 's3:ObjectRemoved:DeleteMarkerCreated':
+                delete_marker_create_events += 1
+                assert event['s3']['configurationId'] in [notification_name+'_1', notification_name+'_2']
+   
+    # 3 key versions were deleted (v1, v2 and the deletion marker)
+    # notified over the same topic via 2 notifications (1,3)
+    assert_equal(delete_events, 3*2)
+    # 1 deletion marker was created
+    # notified over the same topic over 2 notifications (1,2)
+    assert_equal(delete_marker_create_events, 1*2)
+
+    # cleanup
+    stop_amqp_receiver(receiver, task)
+    s3_notification_conf.del_config()
+    topic_conf.del_config()
+    # delete the bucket
+    zones[0].delete_bucket(bucket_name)
+    clean_rabbitmq(proc)
+
+
+def test_ps_push_http():
+    """ test pushing to http endpoint """
+    if skip_push_tests:
+        return SkipTest("PubSub push tests don't run in teuthology")
+    zones, ps_zones = init_env()
+    bucket_name = gen_bucket_name()
+    topic_name = bucket_name+TOPIC_SUFFIX
+
+    # create random port for the http server
+    host = get_ip()
+    port = random.randint(10000, 20000)
+    # start an http server in a separate thread
+    http_server = StreamingHTTPServer(host, port)
+
+    # create topic
+    topic_conf = PSTopic(ps_zones[0].conn, topic_name)
+    _, status = topic_conf.set_config()
+    assert_equal(status/100, 2)
+    # create bucket on the first of the rados zones
+    bucket = zones[0].create_bucket(bucket_name)
+    # wait for sync
+    zone_meta_checkpoint(ps_zones[0].zone)
+    # create notifications
+    notification_conf = PSNotification(ps_zones[0].conn, bucket_name,
+                                       topic_name)
+    _, status = notification_conf.set_config()
+    assert_equal(status/100, 2)
+    # create subscription
+    sub_conf = PSSubscription(ps_zones[0].conn, bucket_name+SUB_SUFFIX,
+                              topic_name, endpoint='http://'+host+':'+str(port))
+    _, status = sub_conf.set_config()
+    assert_equal(status/100, 2)
+    # create objects in the bucket
+    number_of_objects = 10
+    for i in range(number_of_objects):
+        key = bucket.new_key(str(i))
+        key.set_contents_from_string('bar')
+    # wait for sync
+    zone_bucket_checkpoint(ps_zones[0].zone, zones[0].zone, bucket_name)
+    # check http server
+    keys = list(bucket.list())
+    # TODO: use exact match
+    http_server.verify_events(keys, exact_match=False)
+
+    # delete objects from the bucket
+    for key in bucket.list():
+        key.delete()
+    # wait for sync
+    zone_meta_checkpoint(ps_zones[0].zone)
+    zone_bucket_checkpoint(ps_zones[0].zone, zones[0].zone, bucket_name)
+    # check http server
+    # TODO: use exact match
+    http_server.verify_events(keys, deletions=True, exact_match=False)
+
+    # cleanup
+    sub_conf.del_config()
+    notification_conf.del_config()
+    topic_conf.del_config()
+    zones[0].delete_bucket(bucket_name)
+    http_server.close()
+
+
+def test_ps_s3_push_http():
+    """ test pushing to http endpoint s3 record format"""
+    if skip_push_tests:
+        return SkipTest("PubSub push tests don't run in teuthology")
+    zones, ps_zones = init_env()
+    bucket_name = gen_bucket_name()
+    topic_name = bucket_name+TOPIC_SUFFIX
+
+    # create random port for the http server
+    host = get_ip()
+    port = random.randint(10000, 20000)
+    # start an http server in a separate thread
+    http_server = StreamingHTTPServer(host, port)
+
+    # create topic
+    topic_conf = PSTopic(ps_zones[0].conn, topic_name,
+                         endpoint='http://'+host+':'+str(port))
+    result, status = topic_conf.set_config()
+    assert_equal(status/100, 2)
+    parsed_result = json.loads(result)
+    topic_arn = parsed_result['arn']
+    # create bucket on the first of the rados zones
+    bucket = zones[0].create_bucket(bucket_name)
+    # wait for sync
+    zone_meta_checkpoint(ps_zones[0].zone)
+    # create s3 notification
+    notification_name = bucket_name + NOTIFICATION_SUFFIX
+    topic_conf_list = [{'Id': notification_name,
+                        'TopicArn': topic_arn,
+                        'Events': ['s3:ObjectCreated:*', 's3:ObjectRemoved:*']
+                       }]
+    s3_notification_conf = PSNotificationS3(ps_zones[0].conn, bucket_name, topic_conf_list)
+    _, status = s3_notification_conf.set_config()
+    assert_equal(status/100, 2)
+    # create objects in the bucket
+    number_of_objects = 10
+    for i in range(number_of_objects):
+        key = bucket.new_key(str(i))
+        key.set_contents_from_string('bar')
+    # wait for sync
+    zone_bucket_checkpoint(ps_zones[0].zone, zones[0].zone, bucket_name)
+    # check http server
+    keys = list(bucket.list())
+    # TODO: use exact match
+    http_server.verify_s3_events(keys, exact_match=False)
+
+    # delete objects from the bucket
+    for key in bucket.list():
+        key.delete()
+    # wait for sync
+    zone_meta_checkpoint(ps_zones[0].zone)
+    zone_bucket_checkpoint(ps_zones[0].zone, zones[0].zone, bucket_name)
+    # check http server
+    # TODO: use exact match
+    http_server.verify_s3_events(keys, deletions=True, exact_match=False)
+
+    # cleanup
+    s3_notification_conf.del_config()
+    topic_conf.del_config()
+    zones[0].delete_bucket(bucket_name)
+    http_server.close()
+
+
+def test_ps_push_amqp():
+    """ test pushing to amqp endpoint """
+    if skip_push_tests:
+        return SkipTest("PubSub push tests don't run in teuthology")
+    hostname = get_ip()
+    proc = init_rabbitmq()
+    if proc is  None:
+        return SkipTest('end2end amqp tests require rabbitmq-server installed')
+    zones, ps_zones = init_env()
+    bucket_name = gen_bucket_name()
+    topic_name = bucket_name+TOPIC_SUFFIX
+
+    # create topic
+    exchange = 'ex1'
+    task, receiver = create_amqp_receiver_thread(exchange, topic_name)
+    task.start()
+    topic_conf = PSTopic(ps_zones[0].conn, topic_name)
+    _, status = topic_conf.set_config()
+    assert_equal(status/100, 2)
+    # create bucket on the first of the rados zones
+    bucket = zones[0].create_bucket(bucket_name)
+    # wait for sync
+    zone_meta_checkpoint(ps_zones[0].zone)
+    # create notifications
+    notification_conf = PSNotification(ps_zones[0].conn, bucket_name,
+                                       topic_name)
+    _, status = notification_conf.set_config()
+    assert_equal(status/100, 2)
+    # create subscription
+    sub_conf = PSSubscription(ps_zones[0].conn, bucket_name+SUB_SUFFIX,
+                              topic_name, endpoint='amqp://'+hostname,
+                              endpoint_args='amqp-exchange='+exchange+'&amqp-ack-level=broker')
+    _, status = sub_conf.set_config()
+    assert_equal(status/100, 2)
+    # create objects in the bucket
+    number_of_objects = 10
+    for i in range(number_of_objects):
+        key = bucket.new_key(str(i))
+        key.set_contents_from_string('bar')
+    # wait for sync
+    zone_bucket_checkpoint(ps_zones[0].zone, zones[0].zone, bucket_name)
+    # check amqp receiver
+    keys = list(bucket.list())
+    # TODO: use exact match
+    receiver.verify_events(keys, exact_match=False)
+
+    # delete objects from the bucket
+    for key in bucket.list():
+        key.delete()
+    # wait for sync
+    zone_meta_checkpoint(ps_zones[0].zone)
+    zone_bucket_checkpoint(ps_zones[0].zone, zones[0].zone, bucket_name)
+    # check amqp receiver
+    # TODO: use exact match
+    receiver.verify_events(keys, deletions=True, exact_match=False)
+
+    # cleanup
+    stop_amqp_receiver(receiver, task)
+    sub_conf.del_config()
+    notification_conf.del_config()
+    topic_conf.del_config()
+    zones[0].delete_bucket(bucket_name)
+    clean_rabbitmq(proc)
+
+
+def test_ps_s3_push_amqp():
+    """ test pushing to amqp endpoint s3 record format"""
+    if skip_push_tests:
+        return SkipTest("PubSub push tests don't run in teuthology")
+    hostname = get_ip()
+    proc = init_rabbitmq()
+    if proc is  None:
+        return SkipTest('end2end amqp tests require rabbitmq-server installed')
+    zones, ps_zones = init_env()
+    bucket_name = gen_bucket_name()
+    topic_name = bucket_name+TOPIC_SUFFIX
+
+    # create topic
+    exchange = 'ex1'
+    task, receiver = create_amqp_receiver_thread(exchange, topic_name)
+    task.start()
+    topic_conf = PSTopic(ps_zones[0].conn, topic_name,
+                         endpoint='amqp://' + hostname,
+                         endpoint_args='amqp-exchange=' + exchange + '&amqp-ack-level=none')
+    result, status = topic_conf.set_config()
+    assert_equal(status/100, 2)
+    parsed_result = json.loads(result)
+    topic_arn = parsed_result['arn']
+    # create bucket on the first of the rados zones
+    bucket = zones[0].create_bucket(bucket_name)
+    # wait for sync
+    zone_meta_checkpoint(ps_zones[0].zone)
+    # create s3 notification
+    notification_name = bucket_name + NOTIFICATION_SUFFIX
+    topic_conf_list = [{'Id': notification_name,
+                        'TopicArn': topic_arn,
+                        'Events': ['s3:ObjectCreated:*', 's3:ObjectRemoved:*']
+                       }]
+    s3_notification_conf = PSNotificationS3(ps_zones[0].conn, bucket_name, topic_conf_list)
+    _, status = s3_notification_conf.set_config()
+    assert_equal(status/100, 2)
+    # create objects in the bucket
+    number_of_objects = 10
+    for i in range(number_of_objects):
+        key = bucket.new_key(str(i))
+        key.set_contents_from_string('bar')
+    # wait for sync
+    zone_bucket_checkpoint(ps_zones[0].zone, zones[0].zone, bucket_name)
+    # check amqp receiver
+    keys = list(bucket.list())
+    # TODO: use exact match
+    receiver.verify_s3_events(keys, exact_match=False)
+
+    # delete objects from the bucket
+    for key in bucket.list():
+        key.delete()
+    # wait for sync
+    zone_meta_checkpoint(ps_zones[0].zone)
+    zone_bucket_checkpoint(ps_zones[0].zone, zones[0].zone, bucket_name)
+    # check amqp receiver
+    # TODO: use exact match
+    receiver.verify_s3_events(keys, deletions=True, exact_match=False)
+
+    # cleanup
+    stop_amqp_receiver(receiver, task)
+    s3_notification_conf.del_config()
+    topic_conf.del_config()
+    zones[0].delete_bucket(bucket_name)
+    clean_rabbitmq(proc)
+
+
+def test_ps_delete_bucket():
+    """ test notification status upon bucket deletion """
+    zones, ps_zones = init_env()
+    bucket_name = gen_bucket_name()
+    # create bucket on the first of the rados zones
+    bucket = zones[0].create_bucket(bucket_name)
+    # wait for sync
+    zone_meta_checkpoint(ps_zones[0].zone)
+    topic_name = bucket_name + TOPIC_SUFFIX
+    # create topic
+    topic_name = bucket_name + TOPIC_SUFFIX
+    topic_conf = PSTopic(ps_zones[0].conn, topic_name)
+    response, status = topic_conf.set_config()
+    assert_equal(status/100, 2)
+    parsed_result = json.loads(response)
+    topic_arn = parsed_result['arn']
+    # create one s3 notification
+    notification_name = bucket_name + NOTIFICATION_SUFFIX
+    topic_conf_list = [{'Id': notification_name,
+                        'TopicArn': topic_arn,
+                        'Events': ['s3:ObjectCreated:*']
+                       }]
+    s3_notification_conf = PSNotificationS3(ps_zones[0].conn, bucket_name, topic_conf_list)
+    response, status = s3_notification_conf.set_config()
+    assert_equal(status/100, 2)
+
+    # create non-s3 notification
+    notification_conf = PSNotification(ps_zones[0].conn, bucket_name,
+                                       topic_name)
+    _, status = notification_conf.set_config()
+    assert_equal(status/100, 2)
+
+    # create objects in the bucket
+    number_of_objects = 10
+    for i in range(number_of_objects):
+        key = bucket.new_key(str(i))
+        key.set_contents_from_string('bar')
+    # wait for bucket sync
+    zone_bucket_checkpoint(ps_zones[0].zone, zones[0].zone, bucket_name)
+    keys = list(bucket.list())
+    # delete objects from the bucket
+    for key in bucket.list():
+        key.delete()
+    # wait for bucket sync
+    zone_bucket_checkpoint(ps_zones[0].zone, zones[0].zone, bucket_name)
+    # delete the bucket
+    zones[0].delete_bucket(bucket_name)
+    # wait for meta sync
+    zone_meta_checkpoint(ps_zones[0].zone)
+
+    # get the events from the auto-generated subscription
+    sub_conf = PSSubscription(ps_zones[0].conn, notification_name,
+                              topic_name)
+    result, _ = sub_conf.get_events()
+    records = json.loads(result)
+    # TODO: use exact match
+    verify_s3_records_by_elements(records, keys, exact_match=False)
+
+    # s3 notification is deleted with bucket
+    _, status = s3_notification_conf.get_config(notification=notification_name)
+    assert_equal(status, 404)
+    # non-s3 notification is deleted with bucket
+    _, status = notification_conf.get_config()
+    assert_equal(status, 404)
+    # cleanup
+    sub_conf.del_config()
+    topic_conf.del_config()
+
+
+def test_ps_missing_topic():
+    """ test creating a subscription when no topic info exists"""
+    zones, ps_zones = init_env()
+    bucket_name = gen_bucket_name()
+    topic_name = bucket_name+TOPIC_SUFFIX
+
+    # create bucket on the first of the rados zones
+    zones[0].create_bucket(bucket_name)
+    # wait for sync
+    zone_meta_checkpoint(ps_zones[0].zone)
+    # create s3 notification
+    notification_name = bucket_name + NOTIFICATION_SUFFIX
+    topic_arn = 'arn:aws:sns:::' + topic_name
+    topic_conf_list = [{'Id': notification_name,
+                        'TopicArn': topic_arn,
+                        'Events': ['s3:ObjectCreated:*']
+                       }]
+    s3_notification_conf = PSNotificationS3(ps_zones[0].conn, bucket_name, topic_conf_list)
+    try:
+        s3_notification_conf.set_config()
+    except:
+        log.info('missing topic is expected')
+    else:
+        assert 'missing topic is expected'
+
+    # cleanup
+    zones[0].delete_bucket(bucket_name)
+
+
+def test_ps_s3_topic_update():
+    """ test updating topic associated with a notification"""
+    if skip_push_tests:
+        return SkipTest("PubSub push tests don't run in teuthology")
+    rabbit_proc = init_rabbitmq()
+    if rabbit_proc is  None:
+        return SkipTest('end2end amqp tests require rabbitmq-server installed')
+    zones, ps_zones = init_env()
+    bucket_name = gen_bucket_name()
+    topic_name = bucket_name+TOPIC_SUFFIX
+
+    # create amqp topic
+    hostname = get_ip()
+    exchange = 'ex1'
+    amqp_task, receiver = create_amqp_receiver_thread(exchange, topic_name)
+    amqp_task.start()
+    topic_conf = PSTopic(ps_zones[0].conn, topic_name,
+                          endpoint='amqp://' + hostname,
+                          endpoint_args='amqp-exchange=' + exchange + '&amqp-ack-level=none')
+    result, status = topic_conf.set_config()
+    assert_equal(status/100, 2)
+    parsed_result = json.loads(result)
+    topic_arn = parsed_result['arn']
+    # get topic
+    result, _ = topic_conf.get_config()
+    # verify topic content
+    parsed_result = json.loads(result)
+    assert_equal(parsed_result['topic']['name'], topic_name)
+    assert_equal(parsed_result['topic']['dest']['push_endpoint'], topic_conf.parameters['push-endpoint'])
+    
+    # create http server
+    port = random.randint(10000, 20000)
+    # start an http server in a separate thread
+    http_server = StreamingHTTPServer(hostname, port)
+
+    # create bucket on the first of the rados zones
+    bucket = zones[0].create_bucket(bucket_name)
+    # wait for sync
+    zone_meta_checkpoint(ps_zones[0].zone)
+    # create s3 notification
+    notification_name = bucket_name + NOTIFICATION_SUFFIX
+    topic_conf_list = [{'Id': notification_name,
+                        'TopicArn': topic_arn,
+                        'Events': ['s3:ObjectCreated:*']
+                       }]
+    s3_notification_conf = PSNotificationS3(ps_zones[0].conn, bucket_name, topic_conf_list)
+    _, status = s3_notification_conf.set_config()
+    assert_equal(status/100, 2)
+    # create objects in the bucket
+    number_of_objects = 10
+    for i in range(number_of_objects):
+        key = bucket.new_key(str(i))
+        key.set_contents_from_string('bar')
+    # wait for sync
+    zone_bucket_checkpoint(ps_zones[0].zone, zones[0].zone, bucket_name)
+
+    keys = list(bucket.list())
+    # TODO: use exact match
+    receiver.verify_s3_events(keys, exact_match=False)
+
+    # update the same topic with new endpoint
+    topic_conf = PSTopic(ps_zones[0].conn, topic_name,
+                         endpoint='http://'+ hostname + ':' + str(port))
+    _, status = topic_conf.set_config()
+    assert_equal(status/100, 2)
+    # get topic
+    result, _ = topic_conf.get_config()
+    # verify topic content
+    parsed_result = json.loads(result)
+    assert_equal(parsed_result['topic']['name'], topic_name)
+    assert_equal(parsed_result['topic']['dest']['push_endpoint'], topic_conf.parameters['push-endpoint'])
+
+    # delete current objects and create new objects in the bucket
+    for key in bucket.list():
+        key.delete()
+    for i in range(number_of_objects):
+        key = bucket.new_key(str(i+100))
+        key.set_contents_from_string('bar')
+    # wait for sync
+    zone_meta_checkpoint(ps_zones[0].zone)
+    zone_bucket_checkpoint(ps_zones[0].zone, zones[0].zone, bucket_name)
+
+    keys = list(bucket.list())
+    # verify that notifications are still sent to amqp
+    # TODO: use exact match
+    receiver.verify_s3_events(keys, exact_match=False)
+
+    # update notification to update the endpoint from the topic
+    topic_conf_list = [{'Id': notification_name,
+                        'TopicArn': topic_arn,
+                        'Events': ['s3:ObjectCreated:*']
+                       }]
+    s3_notification_conf = PSNotificationS3(ps_zones[0].conn, bucket_name, topic_conf_list)
+    _, status = s3_notification_conf.set_config()
+    assert_equal(status/100, 2)
+    
+    # delete current objects and create new objects in the bucket
+    for key in bucket.list():
+        key.delete()
+    for i in range(number_of_objects):
+        key = bucket.new_key(str(i+200))
+        key.set_contents_from_string('bar')
+    # wait for sync
+    zone_meta_checkpoint(ps_zones[0].zone)
+    zone_bucket_checkpoint(ps_zones[0].zone, zones[0].zone, bucket_name)
+
+    keys = list(bucket.list())
+    # check that updates switched to http
+    # TODO: use exact match
+    http_server.verify_s3_events(keys, exact_match=False)
+
+    # cleanup
+    # delete objects from the bucket
+    stop_amqp_receiver(receiver, amqp_task)
+    for key in bucket.list():
+        key.delete()
+    s3_notification_conf.del_config()
+    topic_conf.del_config()
+    zones[0].delete_bucket(bucket_name)
+    http_server.close()
+    clean_rabbitmq(rabbit_proc)
+
+
+def test_ps_s3_notification_update():
+    """ test updating the topic of a notification"""
+    if skip_push_tests:
+        return SkipTest("PubSub push tests don't run in teuthology")
+    hostname = get_ip()
+    rabbit_proc = init_rabbitmq()
+    if rabbit_proc is  None:
+        return SkipTest('end2end amqp tests require rabbitmq-server installed')
+
+    zones, ps_zones = init_env()
+    bucket_name = gen_bucket_name()
+    topic_name1 = bucket_name+'amqp'+TOPIC_SUFFIX
+    topic_name2 = bucket_name+'http'+TOPIC_SUFFIX
+
+    # create topics
+    # start amqp receiver in a separate thread
+    exchange = 'ex1'
+    amqp_task, receiver = create_amqp_receiver_thread(exchange, topic_name1)
+    amqp_task.start()
+    # create random port for the http server
+    http_port = random.randint(10000, 20000)
+    # start an http server in a separate thread
+    http_server = StreamingHTTPServer(hostname, http_port)
+
+    topic_conf1 = PSTopic(ps_zones[0].conn, topic_name1,
+                          endpoint='amqp://' + hostname,
+                          endpoint_args='amqp-exchange=' + exchange + '&amqp-ack-level=none')
+    result, status = topic_conf1.set_config()
+    parsed_result = json.loads(result)
+    topic_arn1 = parsed_result['arn']
+    assert_equal(status/100, 2)
+    topic_conf2 = PSTopic(ps_zones[0].conn, topic_name2,
+                          endpoint='http://'+hostname+':'+str(http_port))
+    result, status = topic_conf2.set_config()
+    parsed_result = json.loads(result)
+    topic_arn2 = parsed_result['arn']
+    assert_equal(status/100, 2)
+
+    # create bucket on the first of the rados zones
+    bucket = zones[0].create_bucket(bucket_name)
+    # wait for sync
+    zone_meta_checkpoint(ps_zones[0].zone)
+    # create s3 notification with topic1
+    notification_name = bucket_name + NOTIFICATION_SUFFIX
+    topic_conf_list = [{'Id': notification_name,
+                        'TopicArn': topic_arn1,
+                        'Events': ['s3:ObjectCreated:*']
+                       }]
+    s3_notification_conf = PSNotificationS3(ps_zones[0].conn, bucket_name, topic_conf_list)
+    _, status = s3_notification_conf.set_config()
+    assert_equal(status/100, 2)
+    # create objects in the bucket
+    number_of_objects = 10
+    for i in range(number_of_objects):
+        key = bucket.new_key(str(i))
+        key.set_contents_from_string('bar')
+    # wait for sync
+    zone_bucket_checkpoint(ps_zones[0].zone, zones[0].zone, bucket_name)
+
+    keys = list(bucket.list())
+    # TODO: use exact match
+    receiver.verify_s3_events(keys, exact_match=False);
+
+    # update notification to use topic2
+    topic_conf_list = [{'Id': notification_name,
+                        'TopicArn': topic_arn2,
+                        'Events': ['s3:ObjectCreated:*']
+                       }]
+    s3_notification_conf = PSNotificationS3(ps_zones[0].conn, bucket_name, topic_conf_list)
+    _, status = s3_notification_conf.set_config()
+    assert_equal(status/100, 2)
+
+    # delete current objects and create new objects in the bucket
+    for key in bucket.list():
+        key.delete()
+    for i in range(number_of_objects):
+        key = bucket.new_key(str(i+100))
+        key.set_contents_from_string('bar')
+    # wait for sync
+    zone_meta_checkpoint(ps_zones[0].zone)
+    zone_bucket_checkpoint(ps_zones[0].zone, zones[0].zone, bucket_name)
+
+    keys = list(bucket.list())
+    # check that updates switched to http
+    # TODO: use exact match
+    http_server.verify_s3_events(keys, exact_match=False)
+
+    # cleanup
+    # delete objects from the bucket
+    stop_amqp_receiver(receiver, amqp_task)
+    for key in bucket.list():
+        key.delete()
+    s3_notification_conf.del_config()
+    topic_conf1.del_config()
+    topic_conf2.del_config()
+    zones[0].delete_bucket(bucket_name)
+    http_server.close()
+    clean_rabbitmq(rabbit_proc)
+
+
+def test_ps_s3_multiple_topics_notification():
+    """ test notification creation with multiple topics"""
+    if skip_push_tests:
+        return SkipTest("PubSub push tests don't run in teuthology")
+    hostname = get_ip()
+    rabbit_proc = init_rabbitmq()
+    if rabbit_proc is  None:
+        return SkipTest('end2end amqp tests require rabbitmq-server installed')
+
+    zones, ps_zones = init_env()
+    bucket_name = gen_bucket_name()
+    topic_name1 = bucket_name+'amqp'+TOPIC_SUFFIX
+    topic_name2 = bucket_name+'http'+TOPIC_SUFFIX
+
+    # create topics
+    # start amqp receiver in a separate thread
+    exchange = 'ex1'
+    amqp_task, receiver = create_amqp_receiver_thread(exchange, topic_name1)
+    amqp_task.start()
+    # create random port for the http server
+    http_port = random.randint(10000, 20000)
+    # start an http server in a separate thread
+    http_server = StreamingHTTPServer(hostname, http_port)
+
+    topic_conf1 = PSTopic(ps_zones[0].conn, topic_name1,
+                          endpoint='amqp://' + hostname,
+                          endpoint_args='amqp-exchange=' + exchange + '&amqp-ack-level=none')
+    result, status = topic_conf1.set_config()
+    parsed_result = json.loads(result)
+    topic_arn1 = parsed_result['arn']
+    assert_equal(status/100, 2)
+    topic_conf2 = PSTopic(ps_zones[0].conn, topic_name2,
+                          endpoint='http://'+hostname+':'+str(http_port))
+    result, status = topic_conf2.set_config()
+    parsed_result = json.loads(result)
+    topic_arn2 = parsed_result['arn']
+    assert_equal(status/100, 2)
+
+    # create bucket on the first of the rados zones
+    bucket = zones[0].create_bucket(bucket_name)
+    # wait for sync
+    zone_meta_checkpoint(ps_zones[0].zone)
+    # create s3 notification
+    notification_name1 = bucket_name + NOTIFICATION_SUFFIX + '_1'
+    notification_name2 = bucket_name + NOTIFICATION_SUFFIX + '_2'
+    topic_conf_list = [
+        {
+            'Id': notification_name1,
+            'TopicArn': topic_arn1,
+            'Events': ['s3:ObjectCreated:*']
+        },
+        {
+            'Id': notification_name2,
+            'TopicArn': topic_arn2,
+            'Events': ['s3:ObjectCreated:*']
+        }]
+    s3_notification_conf = PSNotificationS3(ps_zones[0].conn, bucket_name, topic_conf_list)
+    _, status = s3_notification_conf.set_config()
+    assert_equal(status/100, 2)
+    result, _ = s3_notification_conf.get_config()
+    assert_equal(len(result['TopicConfigurations']), 2)
+    assert_equal(result['TopicConfigurations'][0]['Id'], notification_name1)
+    assert_equal(result['TopicConfigurations'][1]['Id'], notification_name2)
+
+    # get auto-generated subscriptions
+    sub_conf1 = PSSubscription(ps_zones[0].conn, notification_name1,
+                               topic_name1)
+    _, status = sub_conf1.get_config()
+    assert_equal(status/100, 2)
+    sub_conf2 = PSSubscription(ps_zones[0].conn, notification_name2,
+                               topic_name2)
+    _, status = sub_conf2.get_config()
+    assert_equal(status/100, 2)
+
+    # create objects in the bucket
+    number_of_objects = 10
+    for i in range(number_of_objects):
+        key = bucket.new_key(str(i))
+        key.set_contents_from_string('bar')
+    # wait for sync
+    zone_bucket_checkpoint(ps_zones[0].zone, zones[0].zone, bucket_name)
+
+    # get the events from both of the subscription
+    result, _ = sub_conf1.get_events()
+    records = json.loads(result)
+    for record in records['Records']:
+        log.debug(record)
+    keys = list(bucket.list())
+    # TODO: use exact match
+    verify_s3_records_by_elements(records, keys, exact_match=False)
+    receiver.verify_s3_events(keys, exact_match=False)
+
+    result, _ = sub_conf2.get_events()
+    parsed_result = json.loads(result)
+    for record in parsed_result['Records']:
+        log.debug(record)
+    keys = list(bucket.list())
+    # TODO: use exact match
+    verify_s3_records_by_elements(records, keys, exact_match=False)
+    http_server.verify_s3_events(keys, exact_match=False)
+
+    # cleanup
+    stop_amqp_receiver(receiver, amqp_task)
+    s3_notification_conf.del_config()
+    topic_conf1.del_config()
+    topic_conf2.del_config()
+    # delete objects from the bucket
+    for key in bucket.list():
+        key.delete()
+    zones[0].delete_bucket(bucket_name)
+    http_server.close()
+    clean_rabbitmq(rabbit_proc)
diff --git a/src/test/rgw/rgw_multi/tools.py b/src/test/rgw/rgw_multi/tools.py
new file mode 100644
index 00000000..dd7f91ad
--- /dev/null
+++ b/src/test/rgw/rgw_multi/tools.py
@@ -0,0 +1,97 @@
+import json
+import boto
+
+def append_attr_value(d, attr, attrv):
+    if attrv and len(str(attrv)) > 0:
+        d[attr] = attrv
+
+def append_attr(d, k, attr):
+    try:
+        attrv = getattr(k, attr)
+    except:
+        return
+    append_attr_value(d, attr, attrv)
+
+def get_attrs(k, attrs):
+    d = {}
+    for a in attrs:
+        append_attr(d, k, a)
+
+    return d
+
+def append_query_arg(s, n, v):
+    if not v:
+        return s
+    nv = '{n}={v}'.format(n=n, v=v)
+    if not s:
+        return nv
+    return '{s}&{nv}'.format(s=s, nv=nv)
+
+class KeyJSONEncoder(boto.s3.key.Key):
+    @staticmethod
+    def default(k, versioned=False):
+        attrs = ['bucket', 'name', 'size', 'last_modified', 'metadata', 'cache_control',
+                 'content_type', 'content_disposition', 'content_language',
+                 'owner', 'storage_class', 'md5', 'version_id', 'encrypted',
+                 'delete_marker', 'expiry_date', 'VersionedEpoch', 'RgwxTag']
+        d = get_attrs(k, attrs)
+        d['etag'] = k.etag[1:-1]
+        if versioned:
+            d['is_latest'] = k.is_latest
+        return d
+
+class DeleteMarkerJSONEncoder(boto.s3.key.Key):
+    @staticmethod
+    def default(k):
+        attrs = ['name', 'version_id', 'last_modified', 'owner']
+        d = get_attrs(k, attrs)
+        d['delete_marker'] = True
+        d['is_latest'] = k.is_latest
+        return d
+
+class UserJSONEncoder(boto.s3.user.User):
+    @staticmethod
+    def default(k):
+        attrs = ['id', 'display_name']
+        return get_attrs(k, attrs)
+
+class BucketJSONEncoder(boto.s3.bucket.Bucket):
+    @staticmethod
+    def default(k):
+        attrs = ['name', 'creation_date']
+        return get_attrs(k, attrs)
+
+class BotoJSONEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, boto.s3.key.Key):
+            return KeyJSONEncoder.default(obj)
+        if isinstance(obj, boto.s3.deletemarker.DeleteMarker):
+            return DeleteMarkerJSONEncoder.default(obj)
+        if isinstance(obj, boto.s3.user.User):
+            return UserJSONEncoder.default(obj)
+        if isinstance(obj, boto.s3.prefix.Prefix):
+            return (lambda x: {'prefix': x.name})(obj)
+        if isinstance(obj, boto.s3.bucket.Bucket):
+            return BucketJSONEncoder.default(obj)
+        return json.JSONEncoder.default(self, obj)
+
+
+def dump_json(o, cls=BotoJSONEncoder):
+    return json.dumps(o, cls=cls, indent=4)
+
+def assert_raises(excClass, callableObj, *args, **kwargs):
+    """
+    Like unittest.TestCase.assertRaises, but returns the exception.
+    """
+    try:
+        callableObj(*args, **kwargs)
+    except excClass as e:
+        return e
+    else:
+        if hasattr(excClass, '__name__'):
+            excName = excClass.__name__
+        else:
+            excName = str(excClass)
+            raise AssertionError("%s not raised" % excName)
+
+
diff --git a/src/test/rgw/rgw_multi/zone_cloud.py b/src/test/rgw/rgw_multi/zone_cloud.py
new file mode 100644
index 00000000..43b1f76e
--- /dev/null
+++ b/src/test/rgw/rgw_multi/zone_cloud.py
@@ -0,0 +1,332 @@
+import json
+import requests.compat
+import logging
+
+import boto
+import boto.s3.connection
+
+import dateutil.parser
+import datetime
+
+import re
+
+from nose.tools import eq_ as eq
+try:
+    from itertools import izip_longest as zip_longest
+except ImportError:
+    from itertools import zip_longest
+
+from six.moves.urllib.parse import urlparse
+
+from .multisite import *
+from .tools import *
+
+log = logging.getLogger(__name__)
+
+def get_key_ver(k):
+    if not k.version_id:
+        return 'null'
+    return k.version_id
+
+def unquote(s):
+    if s[0] == '"' and s[-1] == '"':
+        return s[1:-1]
+    return s
+
+def check_object_eq(k1, k2, check_extra = True):
+    assert k1
+    assert k2
+    log.debug('comparing key name=%s', k1.name)
+    eq(k1.name, k2.name)
+    eq(k1.metadata, k2.metadata)
+    # eq(k1.cache_control, k2.cache_control)
+    eq(k1.content_type, k2.content_type)
+    eq(k1.content_encoding, k2.content_encoding)
+    eq(k1.content_disposition, k2.content_disposition)
+    eq(k1.content_language, k2.content_language)
+
+    eq(unquote(k1.etag), unquote(k2.etag))
+
+    mtime1 = dateutil.parser.parse(k1.last_modified)
+    mtime2 = dateutil.parser.parse(k2.last_modified)
+    log.debug('k1.last_modified=%s k2.last_modified=%s', k1.last_modified, k2.last_modified)
+    assert abs((mtime1 - mtime2).total_seconds()) < 1 # handle different time resolution
+    # if check_extra:
+        # eq(k1.owner.id, k2.owner.id)
+        # eq(k1.owner.display_name, k2.owner.display_name)
+    # eq(k1.storage_class, k2.storage_class)
+    eq(k1.size, k2.size)
+    eq(get_key_ver(k1), get_key_ver(k2))
+    # eq(k1.encrypted, k2.encrypted)
+
+def make_request(conn, method, bucket, key, query_args, headers):
+    result = conn.make_request(method, bucket=bucket, key=key, query_args=query_args, headers=headers)
+    if result.status // 100 != 2:
+        raise boto.exception.S3ResponseError(result.status, result.reason, result.read())
+    return result
+
+class CloudKey:
+    def __init__(self, zone_bucket, k):
+        self.zone_bucket = zone_bucket
+
+        # we need two keys: when listing buckets, we get keys that only contain partial data
+        # but we need to have the full data so that we could use all the meta-rgwx- headers
+        # that are needed in order to create a correct representation of the object
+        self.key = k
+        self.rgwx_key = k # assuming k has all the meta info on, if not then we'll update it in update()
+        self.update()
+
+    def update(self):
+        k = self.key
+        rk = self.rgwx_key
+
+        self.size = rk.size
+        orig_name = rk.metadata.get('rgwx-source-key')
+        if not orig_name:
+            self.rgwx_key = self.zone_bucket.bucket.get_key(k.name, version_id = k.version_id)
+            rk = self.rgwx_key
+            orig_name = rk.metadata.get('rgwx-source-key')
+
+        self.name = orig_name
+        self.version_id = rk.metadata.get('rgwx-source-version-id')
+
+        ve = rk.metadata.get('rgwx-versioned-epoch')
+        if ve:
+            self.versioned_epoch = int(ve)
+        else:
+            self.versioned_epoch = 0
+
+        mt = rk.metadata.get('rgwx-source-mtime')
+        if mt:
+            self.last_modified = datetime.datetime.utcfromtimestamp(float(mt)).strftime('%a, %d %b %Y %H:%M:%S GMT')
+        else:
+            self.last_modified = k.last_modified
+
+        et = rk.metadata.get('rgwx-source-etag')
+        if rk.etag.find('-') >= 0 or et.find('-') >= 0:
+            # in this case we will use the source etag as it was uploaded via multipart upload
+            # in one of the zones, so there's no way to make sure etags are calculated the same
+            # way. In the other case we'd just want to keep the etag that was generated in the
+            # regular upload mechanism, which should be consistent in both ends
+            self.etag = et
+        else:
+            self.etag = rk.etag
+
+        if k.etag[0] == '"' and self.etag[0] != '"': # inconsistent etag quoting when listing bucket vs object get
+            self.etag = '"' + self.etag + '"'
+
+        new_meta = {}
+        for meta_key, meta_val in k.metadata.items():
+            if not meta_key.startswith('rgwx-'):
+                new_meta[meta_key] = meta_val
+
+        self.metadata = new_meta
+
+        self.cache_control = k.cache_control
+        self.content_type = k.content_type
+        self.content_encoding = k.content_encoding
+        self.content_disposition = k.content_disposition
+        self.content_language = k.content_language
+
+
+    def get_contents_as_string(self, encoding=None):
+        r = self.key.get_contents_as_string(encoding=encoding)
+
+        # the previous call changed the status of the source object, as it loaded
+        # its metadata
+
+        self.rgwx_key = self.key
+        self.update()
+
+        return r
+
+def append_query_arg(s, n, v):
+    if not v:
+        return s
+    nv = '{n}={v}'.format(n=n, v=v)
+    if not s:
+        return nv
+    return '{s}&{nv}'.format(s=s, nv=nv)
+
+
+class CloudZoneBucket:
+    def __init__(self, zone_conn, target_path, name):
+        self.zone_conn = zone_conn
+        self.name = name
+        self.cloud_conn = zone_conn.zone.cloud_conn
+
+        target_path = target_path[:]
+        if target_path[-1] != '/':
+            target_path += '/'
+        target_path = target_path.replace('${bucket}', name)
+
+        tp = target_path.split('/', 1)
+
+        if len(tp) == 1:
+            self.target_bucket = target_path
+            self.target_prefix = ''
+        else:
+            self.target_bucket = tp[0]
+            self.target_prefix = tp[1]
+
+        log.debug('target_path=%s target_bucket=%s target_prefix=%s', target_path, self.target_bucket, self.target_prefix)
+        self.bucket = self.cloud_conn.get_bucket(self.target_bucket)
+
+    def get_all_versions(self):
+        l = []
+
+        for k in self.bucket.get_all_keys(prefix=self.target_prefix):
+            new_key = CloudKey(self, k)
+
+            log.debug('appending o=[\'%s\', \'%s\', \'%d\']', new_key.name, new_key.version_id, new_key.versioned_epoch)
+            l.append(new_key)
+
+
+        sort_key = lambda k: (k.name, -k.versioned_epoch)
+        l.sort(key = sort_key)
+
+        for new_key in l:
+            yield new_key
+
+    def get_key(self, name, version_id=None):
+        return CloudKey(self, self.bucket.get_key(name, version_id=version_id))
+
+
+def parse_endpoint(endpoint):
+    o = urlparse(endpoint)
+
+    netloc = o.netloc.split(':')
+    
+    host = netloc[0]
+
+    if len(netloc) > 1:
+        port = int(netloc[1])
+    else:
+        port = o.port
+
+    is_secure = False
+
+    if o.scheme == 'https':
+        is_secure = True
+
+    if not port:
+        if is_secure:
+            port = 443
+        else:
+            port = 80
+
+    return host, port, is_secure
+
+
+class CloudZone(Zone):
+    def __init__(self, name, cloud_endpoint, credentials, source_bucket, target_path,
+            zonegroup = None, cluster = None, data = None, zone_id = None, gateways = None):
+        self.cloud_endpoint = cloud_endpoint
+        self.credentials = credentials
+        self.source_bucket = source_bucket
+        self.target_path = target_path
+
+        self.target_path = self.target_path.replace('${zone}', name)
+        # self.target_path = self.target_path.replace('${zone_id}', zone_id)
+        self.target_path = self.target_path.replace('${zonegroup}', zonegroup.name)
+        self.target_path = self.target_path.replace('${zonegroup_id}', zonegroup.id)
+
+        log.debug('target_path=%s', self.target_path)
+
+        host, port, is_secure = parse_endpoint(cloud_endpoint)
+
+        self.cloud_conn = boto.connect_s3(
+                aws_access_key_id = credentials.access_key,
+                aws_secret_access_key = credentials.secret,
+                host = host,
+                port = port,
+                is_secure = is_secure,
+                calling_format = boto.s3.connection.OrdinaryCallingFormat())
+        super(CloudZone, self).__init__(name, zonegroup, cluster, data, zone_id, gateways)
+
+
+    def is_read_only(self):
+        return True
+
+    def tier_type(self):
+        return "cloud"
+
+    def create(self, cluster, args = None, check_retcode = True):
+        """ create the object with the given arguments """
+
+        if args is None:
+            args = ''
+
+        tier_config = ','.join([ 'connection.endpoint=' + self.cloud_endpoint,
+                                 'connection.access_key=' + self.credentials.access_key,
+                                 'connection.secret=' + self.credentials.secret,
+                                 'target_path=' + re.escape(self.target_path)])
+
+        args += [ '--tier-type', self.tier_type(), '--tier-config', tier_config ] 
+
+        return self.json_command(cluster, 'create', args, check_retcode=check_retcode)
+
+    def has_buckets(self):
+        return False
+
+    class Conn(ZoneConn):
+        def __init__(self, zone, credentials):
+            super(CloudZone.Conn, self).__init__(zone, credentials)
+
+        def get_bucket(self, bucket_name):
+            return CloudZoneBucket(self, self.zone.target_path, bucket_name)
+
+        def create_bucket(self, name):
+            # should not be here, a bug in the test suite
+            log.critical('Conn.create_bucket() should not be called in cloud zone')
+            assert False
+
+        def check_bucket_eq(self, zone_conn, bucket_name):
+            assert(zone_conn.zone.tier_type() == "rados")
+
+            log.info('comparing bucket=%s zones={%s, %s}', bucket_name, self.name, self.name)
+            b1 = self.get_bucket(bucket_name)
+            b2 = zone_conn.get_bucket(bucket_name)
+
+            log.debug('bucket1 objects:')
+            for o in b1.get_all_versions():
+                log.debug('o=%s', o.name)
+            log.debug('bucket2 objects:')
+            for o in b2.get_all_versions():
+                log.debug('o=%s', o.name)
+
+            for k1, k2 in zip_longest(b1.get_all_versions(), b2.get_all_versions()):
+                if k1 is None:
+                    log.critical('key=%s is missing from zone=%s', k2.name, self.name)
+                    assert False
+                if k2 is None:
+                    log.critical('key=%s is missing from zone=%s', k1.name, zone_conn.name)
+                    assert False
+
+                check_object_eq(k1, k2)
+
+
+            log.info('success, bucket identical: bucket=%s zones={%s, %s}', bucket_name, self.name, zone_conn.name)
+
+            return True
+
+    def get_conn(self, credentials):
+        return self.Conn(self, credentials)
+
+
+class CloudZoneConfig:
+    def __init__(self, cfg, section):
+        self.endpoint = cfg.get(section, 'endpoint')
+        access_key = cfg.get(section, 'access_key')
+        secret = cfg.get(section, 'secret')
+        self.credentials = Credentials(access_key, secret)
+        try:
+            self.target_path = cfg.get(section, 'target_path')
+        except:
+            self.target_path = 'rgw-${zonegroup_id}/${bucket}'
+
+        try:
+            self.source_bucket = cfg.get(section, 'source_bucket')
+        except:
+            self.source_bucket = '*'
+
diff --git a/src/test/rgw/rgw_multi/zone_es.py b/src/test/rgw/rgw_multi/zone_es.py
new file mode 100644
index 00000000..60d93b8a
--- /dev/null
+++ b/src/test/rgw/rgw_multi/zone_es.py
@@ -0,0 +1,260 @@
+import json
+import requests.compat
+import logging
+
+import boto
+import boto.s3.connection
+
+import dateutil.parser
+
+from nose.tools import eq_ as eq
+try:
+    from itertools import izip_longest as zip_longest
+except ImportError:
+    from itertools import zip_longest
+
+from .multisite import *
+from .tools import *
+
+log = logging.getLogger(__name__)
+
+def get_key_ver(k):
+    if not k.version_id:
+        return 'null'
+    return k.version_id
+
+def check_object_eq(k1, k2, check_extra = True):
+    assert k1
+    assert k2
+    log.debug('comparing key name=%s', k1.name)
+    eq(k1.name, k2.name)
+    eq(k1.metadata, k2.metadata)
+    # eq(k1.cache_control, k2.cache_control)
+    eq(k1.content_type, k2.content_type)
+    # eq(k1.content_encoding, k2.content_encoding)
+    # eq(k1.content_disposition, k2.content_disposition)
+    # eq(k1.content_language, k2.content_language)
+    eq(k1.etag, k2.etag)
+    mtime1 = dateutil.parser.parse(k1.last_modified)
+    mtime2 = dateutil.parser.parse(k2.last_modified)
+    assert abs((mtime1 - mtime2).total_seconds()) < 1 # handle different time resolution
+    if check_extra:
+        eq(k1.owner.id, k2.owner.id)
+        eq(k1.owner.display_name, k2.owner.display_name)
+    # eq(k1.storage_class, k2.storage_class)
+    eq(k1.size, k2.size)
+    eq(get_key_ver(k1), get_key_ver(k2))
+    # eq(k1.encrypted, k2.encrypted)
+
+def make_request(conn, method, bucket, key, query_args, headers):
+    result = conn.make_request(method, bucket=bucket, key=key, query_args=query_args, headers=headers)
+    if result.status // 100 != 2:
+        raise boto.exception.S3ResponseError(result.status, result.reason, result.read())
+    return result
+
+def append_query_arg(s, n, v):
+    if not v:
+        return s
+    nv = '{n}={v}'.format(n=n, v=v)
+    if not s:
+        return nv
+    return '{s}&{nv}'.format(s=s, nv=nv)
+
+class MDSearch:
+    def __init__(self, conn, bucket_name, query, query_args = None, marker = None):
+        self.conn = conn
+        self.bucket_name = bucket_name or ''
+        if bucket_name:
+            self.bucket = boto.s3.bucket.Bucket(name=bucket_name)
+        else:
+            self.bucket = None
+        self.query = query
+        self.query_args = query_args
+        self.max_keys = None
+        self.marker = marker
+
+    def raw_search(self):
+        q = self.query or ''
+        query_args = append_query_arg(self.query_args, 'query', requests.compat.quote_plus(q))
+        if self.max_keys is not None:
+            query_args = append_query_arg(query_args, 'max-keys', self.max_keys)
+        if self.marker:
+            query_args = append_query_arg(query_args, 'marker', self.marker)
+
+        query_args = append_query_arg(query_args, 'format', 'json')
+
+        headers = {}
+
+        result = make_request(self.conn, "GET", bucket=self.bucket_name, key='', query_args=query_args, headers=headers)
+
+        l = []
+
+        result_dict = json.loads(result.read())
+
+        for entry in result_dict['Objects']:
+            bucket = self.conn.get_bucket(entry['Bucket'], validate = False)
+            k = boto.s3.key.Key(bucket, entry['Key'])
+
+            k.version_id = entry['Instance']
+            k.etag = entry['ETag']
+            k.owner = boto.s3.user.User(id=entry['Owner']['ID'], display_name=entry['Owner']['DisplayName'])
+            k.last_modified = entry['LastModified']
+            k.size = entry['Size']
+            k.content_type = entry['ContentType']
+            k.versioned_epoch = entry['VersionedEpoch']
+
+            k.metadata = {}
+            for e in entry['CustomMetadata']:
+                k.metadata[e['Name']] = str(e['Value']) # int values will return as int, cast to string for compatibility with object meta response
+
+            l.append(k)
+
+        return result_dict, l
+
+    def search(self, drain = True, sort = True, sort_key = None):
+        l = []
+
+        is_done = False
+
+        while not is_done:
+            result, result_keys = self.raw_search()
+
+            l = l + result_keys
+
+            is_done = not (drain and (result['IsTruncated'] == "true"))
+            marker = result['Marker']
+
+        if sort:
+            if not sort_key:
+                sort_key = lambda k: (k.name, -k.versioned_epoch)
+            l.sort(key = sort_key)
+
+        return l
+
+
+class MDSearchConfig:
+    def __init__(self, conn, bucket_name):
+        self.conn = conn
+        self.bucket_name = bucket_name or ''
+        if bucket_name:
+            self.bucket = boto.s3.bucket.Bucket(name=bucket_name)
+        else:
+            self.bucket = None
+
+    def send_request(self, conf, method):
+        query_args = 'mdsearch'
+        headers = None
+        if conf:
+            headers = { 'X-Amz-Meta-Search': conf }
+
+        query_args = append_query_arg(query_args, 'format', 'json')
+
+        return make_request(self.conn, method, bucket=self.bucket_name, key='', query_args=query_args, headers=headers)
+
+    def get_config(self):
+        result = self.send_request(None, 'GET')
+        return json.loads(result.read())
+
+    def set_config(self, conf):
+        self.send_request(conf, 'POST')
+
+    def del_config(self):
+        self.send_request(None, 'DELETE')
+
+
+class ESZoneBucket:
+    def __init__(self, zone_conn, name, conn):
+        self.zone_conn = zone_conn
+        self.name = name
+        self.conn = conn
+
+        self.bucket = boto.s3.bucket.Bucket(name=name)
+
+    def get_all_versions(self):
+
+        marker = None
+        is_done = False
+
+        req = MDSearch(self.conn, self.name, 'bucket == ' + self.name, marker=marker)
+
+        for k in req.search():
+            yield k
+
+
+
+
+class ESZone(Zone):
+    def __init__(self, name, es_endpoint, zonegroup = None, cluster = None, data = None, zone_id = None, gateways = None):
+        self.es_endpoint = es_endpoint
+        super(ESZone, self).__init__(name, zonegroup, cluster, data, zone_id, gateways)
+
+    def is_read_only(self):
+        return True
+
+    def tier_type(self):
+        return "elasticsearch"
+
+    def create(self, cluster, args = None, check_retcode = True):
+        """ create the object with the given arguments """
+
+        if args is None:
+            args = ''
+
+        tier_config = ','.join([ 'endpoint=' + self.es_endpoint, 'explicit_custom_meta=false' ])
+
+        args += [ '--tier-type', self.tier_type(), '--tier-config', tier_config ] 
+
+        return self.json_command(cluster, 'create', args, check_retcode=check_retcode)
+
+    def has_buckets(self):
+        return False
+
+    class Conn(ZoneConn):
+        def __init__(self, zone, credentials):
+            super(ESZone.Conn, self).__init__(zone, credentials)
+
+        def get_bucket(self, bucket_name):
+            return ESZoneBucket(self, bucket_name, self.conn)
+
+        def create_bucket(self, name):
+            # should not be here, a bug in the test suite
+            log.critical('Conn.create_bucket() should not be called in ES zone')
+            assert False
+
+        def check_bucket_eq(self, zone_conn, bucket_name):
+            assert(zone_conn.zone.tier_type() == "rados")
+
+            log.info('comparing bucket=%s zones={%s, %s}', bucket_name, self.name, self.name)
+            b1 = self.get_bucket(bucket_name)
+            b2 = zone_conn.get_bucket(bucket_name)
+
+            log.debug('bucket1 objects:')
+            for o in b1.get_all_versions():
+                log.debug('o=%s', o.name)
+            log.debug('bucket2 objects:')
+            for o in b2.get_all_versions():
+                log.debug('o=%s', o.name)
+
+            for k1, k2 in zip_longest(b1.get_all_versions(), b2.get_all_versions()):
+                if k1 is None:
+                    log.critical('key=%s is missing from zone=%s', k2.name, self.name)
+                    assert False
+                if k2 is None:
+                    log.critical('key=%s is missing from zone=%s', k1.name, zone_conn.name)
+                    assert False
+
+                check_object_eq(k1, k2)
+
+
+            log.info('success, bucket identical: bucket=%s zones={%s, %s}', bucket_name, self.name, zone_conn.name)
+
+            return True
+
+    def get_conn(self, credentials):
+        return self.Conn(self, credentials)
+
+
+class ESZoneConfig:
+    def __init__(self, cfg, section):
+        self.endpoint = cfg.get(section, 'endpoint')
+
diff --git a/src/test/rgw/rgw_multi/zone_ps.py b/src/test/rgw/rgw_multi/zone_ps.py
new file mode 100644
index 00000000..b030969f
--- /dev/null
+++ b/src/test/rgw/rgw_multi/zone_ps.py
@@ -0,0 +1,432 @@
+import logging
+import ssl
+import urllib
+import hmac
+import hashlib
+import base64
+import xmltodict
+from six.moves import http_client
+from six.moves.urllib import parse as urlparse
+from time import gmtime, strftime
+from .multisite import Zone
+import boto3
+from botocore.client import Config
+
+log = logging.getLogger('rgw_multi.tests')
+
+def put_object_tagging(conn, bucket_name, key, tags):
+    client = boto3.client('s3',
+            endpoint_url='http://'+conn.host+':'+str(conn.port),
+            aws_access_key_id=conn.aws_access_key_id,
+            aws_secret_access_key=conn.aws_secret_access_key,
+            config=Config(signature_version='s3'))
+    return client.put_object(Body='aaaaaaaaaaa', Bucket=bucket_name, Key=key, Tagging=tags)
+
+
+def get_object_tagging(conn, bucket, object_key):
+    client = boto3.client('s3',
+            endpoint_url='http://'+conn.host+':'+str(conn.port),
+            aws_access_key_id=conn.aws_access_key_id,
+            aws_secret_access_key=conn.aws_secret_access_key,
+            config=Config(signature_version='s3'))
+    return client.get_object_tagging(
+                Bucket=bucket, 
+                Key=object_key
+            )
+
+
+class PSZone(Zone):  # pylint: disable=too-many-ancestors
+    """ PubSub zone class """
+    def __init__(self, name, zonegroup=None, cluster=None, data=None, zone_id=None, gateways=None, full_sync='false', retention_days ='7'):
+        self.full_sync = full_sync
+        self.retention_days = retention_days
+        self.master_zone = zonegroup.master_zone
+        super(PSZone, self).__init__(name, zonegroup, cluster, data, zone_id, gateways)
+
+    def is_read_only(self):
+        return True
+
+    def tier_type(self):
+        return "pubsub"
+
+    def create(self, cluster, args=None, **kwargs):
+        if args is None:
+            args = ''
+        tier_config = ','.join(['start_with_full_sync=' + self.full_sync, 'event_retention_days=' + self.retention_days])
+        args += ['--tier-type', self.tier_type(), '--sync-from-all=0', '--sync-from', self.master_zone.name, '--tier-config', tier_config] 
+        return self.json_command(cluster, 'create', args)
+
+    def has_buckets(self):
+        return False
+
+
+NO_HTTP_BODY = ''
+
+
+def print_connection_info(conn):
+    """print connection details"""
+    print('Endpoint: ' + conn.host + ':' + str(conn.port))
+    print('AWS Access Key:: ' + conn.aws_access_key_id)
+    print('AWS Secret Key:: ' + conn.aws_secret_access_key)
+
+
+def make_request(conn, method, resource, parameters=None, sign_parameters=False, extra_parameters=None):
+    """generic request sending to pubsub radogw
+    should cover: topics, notificatios and subscriptions
+    """
+    url_params = ''
+    if parameters is not None:
+        url_params = urlparse.urlencode(parameters)
+        # remove 'None' from keys with no values
+        url_params = url_params.replace('=None', '')
+        url_params = '?' + url_params
+    if extra_parameters is not None:
+        url_params = url_params + '&' + extra_parameters
+    string_date = strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime())
+    string_to_sign = method + '\n\n\n' + string_date + '\n' + resource
+    if sign_parameters:
+        string_to_sign += url_params
+    signature = base64.b64encode(hmac.new(conn.aws_secret_access_key.encode('utf-8'),
+                                          string_to_sign.encode('utf-8'),
+                                          hashlib.sha1).digest()).decode('ascii')
+    headers = {'Authorization': 'AWS '+conn.aws_access_key_id+':'+signature,
+               'Date': string_date,
+               'Host': conn.host+':'+str(conn.port)}
+    http_conn = http_client.HTTPConnection(conn.host, conn.port)
+    if log.getEffectiveLevel() <= 10:
+        http_conn.set_debuglevel(5)
+    http_conn.request(method, resource+url_params, NO_HTTP_BODY, headers)
+    response = http_conn.getresponse()
+    data = response.read()
+    status = response.status
+    http_conn.close()
+    return data.decode('utf-8'), status
+
+
+def print_connection_info(conn):
+    """print info of connection"""
+    print("Host: " + conn.host+':'+str(conn.port))
+    print("AWS Secret Key: " + conn.aws_secret_access_key)
+    print("AWS Access Key: " + conn.aws_access_key_id)
+
+
+class PSTopic:
+    """class to set/get/delete a topic
+    PUT /topics/<topic name>[?push-endpoint=<endpoint>&[<arg1>=<value1>...]]
+    GET /topics/<topic name>
+    DELETE /topics/<topic name>
+    """
+    def __init__(self, conn, topic_name, endpoint=None, endpoint_args=None):
+        self.conn = conn
+        assert topic_name.strip()
+        self.resource = '/topics/'+topic_name
+        if endpoint is not None:
+            self.parameters = {'push-endpoint': endpoint}
+            self.extra_parameters = endpoint_args
+        else:
+            self.parameters = None
+            self.extra_parameters = None
+
+    def send_request(self, method, get_list=False, parameters=None, extra_parameters=None):
+        """send request to radosgw"""
+        if get_list:
+            return make_request(self.conn, method, '/topics')
+        return make_request(self.conn, method, self.resource, 
+                            parameters=parameters, extra_parameters=extra_parameters)
+
+    def get_config(self):
+        """get topic info"""
+        return self.send_request('GET')
+
+    def set_config(self):
+        """set topic"""
+        return self.send_request('PUT', parameters=self.parameters, extra_parameters=self.extra_parameters)
+
+    def del_config(self):
+        """delete topic"""
+        return self.send_request('DELETE')
+    
+    def get_list(self):
+        """list all topics"""
+        return self.send_request('GET', get_list=True)
+
+
+def delete_all_s3_topics(zone, region):
+    try:
+        conn = zone.secure_conn if zone.secure_conn is not None else zone.conn
+        protocol = 'https' if conn.is_secure else 'http'
+        client = boto3.client('sns',
+                endpoint_url=protocol+'://'+conn.host+':'+str(conn.port),
+                aws_access_key_id=conn.aws_access_key_id,
+                aws_secret_access_key=conn.aws_secret_access_key,
+                region_name=region,
+                verify='./cert.pem',
+                config=Config(signature_version='s3'))
+
+        topics = client.list_topics()['Topics']
+        for topic in topics:
+            print('topic cleanup, deleting: ' + topic['TopicArn'])
+            assert client.delete_topic(TopicArn=topic['TopicArn'])['ResponseMetadata']['HTTPStatusCode'] == 200
+    except Exception as err:
+        print('failed to do topic cleanup: ' + str(err))
+    
+
+def delete_all_objects(conn, bucket_name):
+    client = boto3.client('s3',
+                      endpoint_url='http://'+conn.host+':'+str(conn.port),
+                      aws_access_key_id=conn.aws_access_key_id,
+                      aws_secret_access_key=conn.aws_secret_access_key)
+
+    objects = []
+    for key in client.list_objects(Bucket=bucket_name)['Contents']:
+        objects.append({'Key': key['Key']})
+    # delete objects from the bucket
+    response = client.delete_objects(Bucket=bucket_name,
+            Delete={'Objects': objects})
+    return response
+
+
+class PSTopicS3:
+    """class to set/list/get/delete a topic
+    POST ?Action=CreateTopic&Name=<topic name>[&OpaqueData=<data>[&push-endpoint=<endpoint>&[<arg1>=<value1>...]]]
+    POST ?Action=ListTopics
+    POST ?Action=GetTopic&TopicArn=<topic-arn>
+    POST ?Action=DeleteTopic&TopicArn=<topic-arn>
+    """
+    def __init__(self, conn, topic_name, region, endpoint_args=None, opaque_data=None):
+        self.conn = conn
+        self.topic_name = topic_name.strip()
+        assert self.topic_name
+        self.topic_arn = ''
+        self.attributes = {}
+        if endpoint_args is not None:
+            self.attributes = {nvp[0] : nvp[1] for nvp in urlparse.parse_qsl(endpoint_args, keep_blank_values=True)}
+        if opaque_data is not None:
+            self.attributes['OpaqueData'] = opaque_data
+        protocol = 'https' if conn.is_secure else 'http'
+        self.client = boto3.client('sns',
+                           endpoint_url=protocol+'://'+conn.host+':'+str(conn.port),
+                           aws_access_key_id=conn.aws_access_key_id,
+                           aws_secret_access_key=conn.aws_secret_access_key,
+                           region_name=region,
+                           verify='./cert.pem',
+                           config=Config(signature_version='s3'))
+
+
+    def get_config(self):
+        """get topic info"""
+        parameters = {'Action': 'GetTopic', 'TopicArn': self.topic_arn}
+        body = urlparse.urlencode(parameters)
+        string_date = strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime())
+        content_type = 'application/x-www-form-urlencoded; charset=utf-8'
+        resource = '/'
+        method = 'POST'
+        string_to_sign = method + '\n\n' + content_type + '\n' + string_date + '\n' + resource
+        log.debug('StringTosign: %s', string_to_sign) 
+        signature = base64.b64encode(hmac.new(self.conn.aws_secret_access_key.encode('utf-8'),
+                                     string_to_sign.encode('utf-8'),
+                                     hashlib.sha1).digest()).decode('ascii')
+        headers = {'Authorization': 'AWS '+self.conn.aws_access_key_id+':'+signature,
+                   'Date': string_date,
+                   'Host': self.conn.host+':'+str(self.conn.port),
+                   'Content-Type': content_type}
+        if self.conn.is_secure:
+            http_conn = http_client.HTTPSConnection(self.conn.host, self.conn.port,
+                    context=ssl.create_default_context(cafile='./cert.pem'))
+        else:
+            http_conn = http_client.HTTPConnection(self.conn.host, self.conn.port)
+        http_conn.request(method, resource, body, headers)
+        response = http_conn.getresponse()
+        data = response.read()
+        status = response.status
+        http_conn.close()
+        dict_response = xmltodict.parse(data)
+        return dict_response, status
+
+    def set_config(self):
+        """set topic"""
+        result = self.client.create_topic(Name=self.topic_name, Attributes=self.attributes)
+        self.topic_arn = result['TopicArn']
+        return self.topic_arn
+
+    def del_config(self):
+        """delete topic"""
+        result = self.client.delete_topic(TopicArn=self.topic_arn)
+        return result['ResponseMetadata']['HTTPStatusCode']
+    
+    def get_list(self):
+        """list all topics"""
+        # note that boto3 supports list_topics(), however, the result only show ARNs
+        parameters = {'Action': 'ListTopics'}
+        body = urlparse.urlencode(parameters)
+        string_date = strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime())
+        content_type = 'application/x-www-form-urlencoded; charset=utf-8'
+        resource = '/'
+        method = 'POST'
+        string_to_sign = method + '\n\n' + content_type + '\n' + string_date + '\n' + resource
+        log.debug('StringTosign: %s', string_to_sign) 
+        signature = base64.b64encode(hmac.new(self.conn.aws_secret_access_key.encode('utf-8'),
+                                     string_to_sign.encode('utf-8'),
+                                     hashlib.sha1).digest()).decode('ascii')
+        headers = {'Authorization': 'AWS '+self.conn.aws_access_key_id+':'+signature,
+                   'Date': string_date,
+                   'Host': self.conn.host+':'+str(self.conn.port),
+                   'Content-Type': content_type}
+        if self.conn.is_secure:
+            http_conn = http_client.HTTPSConnection(self.conn.host, self.conn.port,
+                    context=ssl.create_default_context(cafile='./cert.pem'))
+        else:
+            http_conn = http_client.HTTPConnection(self.conn.host, self.conn.port)
+        http_conn.request(method, resource, body, headers)
+        response = http_conn.getresponse()
+        data = response.read()
+        status = response.status
+        http_conn.close()
+        dict_response = xmltodict.parse(data)
+        return dict_response, status
+
+
+class PSNotification:
+    """class to set/get/delete a notification
+    PUT /notifications/bucket/<bucket>?topic=<topic-name>[&events=<event>[,<event>]]
+    GET /notifications/bucket/<bucket>
+    DELETE /notifications/bucket/<bucket>?topic=<topic-name>
+    """
+    def __init__(self, conn, bucket_name, topic_name, events=''):
+        self.conn = conn
+        assert bucket_name.strip()
+        assert topic_name.strip()
+        self.resource = '/notifications/bucket/'+bucket_name
+        if events.strip():
+            self.parameters = {'topic': topic_name, 'events': events}
+        else:
+            self.parameters = {'topic': topic_name}
+
+    def send_request(self, method, parameters=None):
+        """send request to radosgw"""
+        return make_request(self.conn, method, self.resource, parameters)
+
+    def get_config(self):
+        """get notification info"""
+        return self.send_request('GET')
+
+    def set_config(self):
+        """set notification"""
+        return self.send_request('PUT', self.parameters)
+
+    def del_config(self):
+        """delete notification"""
+        return self.send_request('DELETE', self.parameters)
+
+
+class PSNotificationS3:
+    """class to set/get/delete an S3 notification
+    PUT /<bucket>?notification
+    GET /<bucket>?notification[=<notification>]
+    DELETE /<bucket>?notification[=<notification>]
+    """
+    def __init__(self, conn, bucket_name, topic_conf_list):
+        self.conn = conn
+        assert bucket_name.strip()
+        self.bucket_name = bucket_name
+        self.resource = '/'+bucket_name
+        self.topic_conf_list = topic_conf_list
+        self.client = boto3.client('s3',
+                                   endpoint_url='http://'+conn.host+':'+str(conn.port),
+                                   aws_access_key_id=conn.aws_access_key_id,
+                                   aws_secret_access_key=conn.aws_secret_access_key,
+                                   config=Config(signature_version='s3'))
+
+    def send_request(self, method, parameters=None):
+        """send request to radosgw"""
+        return make_request(self.conn, method, self.resource,
+                            parameters=parameters, sign_parameters=True)
+
+    def get_config(self, notification=None):
+        """get notification info"""
+        parameters = None
+        if notification is None:
+            response = self.client.get_bucket_notification_configuration(Bucket=self.bucket_name)
+            status = response['ResponseMetadata']['HTTPStatusCode']
+            return response, status
+        parameters = {'notification': notification}
+        response, status = self.send_request('GET', parameters=parameters)
+        dict_response = xmltodict.parse(response)
+        return dict_response, status
+
+    def set_config(self):
+        """set notification"""
+        response = self.client.put_bucket_notification_configuration(Bucket=self.bucket_name,
+                                                                     NotificationConfiguration={
+                                                                         'TopicConfigurations': self.topic_conf_list
+                                                                     })
+        status = response['ResponseMetadata']['HTTPStatusCode']
+        return response, status
+
+    def del_config(self, notification=None):
+        """delete notification"""
+        parameters = {'notification': notification}
+
+        return self.send_request('DELETE', parameters)
+
+
+class PSSubscription:
+    """class to set/get/delete a subscription:
+    PUT /subscriptions/<sub-name>?topic=<topic-name>[&push-endpoint=<endpoint>&[<arg1>=<value1>...]]
+    GET /subscriptions/<sub-name>
+    DELETE /subscriptions/<sub-name>
+    also to get list of events, and ack them:
+    GET /subscriptions/<sub-name>?events[&max-entries=<max-entries>][&marker=<marker>]
+    POST /subscriptions/<sub-name>?ack&event-id=<event-id>
+    """
+    def __init__(self, conn, sub_name, topic_name, endpoint=None, endpoint_args=None):
+        self.conn = conn
+        assert topic_name.strip()
+        self.resource = '/subscriptions/'+sub_name
+        if endpoint is not None:
+            self.parameters = {'topic': topic_name, 'push-endpoint': endpoint}
+            self.extra_parameters = endpoint_args
+        else:
+            self.parameters = {'topic': topic_name}
+            self.extra_parameters = None
+
+    def send_request(self, method, parameters=None, extra_parameters=None):
+        """send request to radosgw"""
+        return make_request(self.conn, method, self.resource, 
+                            parameters=parameters,
+                            extra_parameters=extra_parameters)
+
+    def get_config(self):
+        """get subscription info"""
+        return self.send_request('GET')
+
+    def set_config(self):
+        """set subscription"""
+        return self.send_request('PUT', parameters=self.parameters, extra_parameters=self.extra_parameters)
+
+    def del_config(self, topic=False):
+        """delete subscription"""
+        if topic:
+            return self.send_request('DELETE', self.parameters)
+        return self.send_request('DELETE')
+
+    def get_events(self, max_entries=None, marker=None):
+        """ get events from subscription """
+        parameters = {'events': None}
+        if max_entries is not None:
+            parameters['max-entries'] = max_entries
+        if marker is not None:
+            parameters['marker'] = marker
+        return self.send_request('GET', parameters)
+
+    def ack_events(self, event_id):
+        """ ack events in a subscription """
+        parameters = {'ack': None, 'event-id': event_id}
+        return self.send_request('POST', parameters)
+
+
+class PSZoneConfig:
+    """ pubsub zone configuration """
+    def __init__(self, cfg, section):
+        self.full_sync = cfg.get(section, 'start_with_full_sync')
+        self.retention_days = cfg.get(section, 'retention_days')
diff --git a/src/test/rgw/rgw_multi/zone_rados.py b/src/test/rgw/rgw_multi/zone_rados.py
new file mode 100644
index 00000000..e645e9ab
--- /dev/null
+++ b/src/test/rgw/rgw_multi/zone_rados.py
@@ -0,0 +1,112 @@
+import logging
+from boto.s3.deletemarker import DeleteMarker
+
+try:
+    from itertools import izip_longest as zip_longest
+except ImportError:
+    from itertools import zip_longest
+
+from nose.tools import eq_ as eq
+
+from .multisite import *
+
+log = logging.getLogger(__name__)
+
+def check_object_eq(k1, k2, check_extra = True):
+    assert k1
+    assert k2
+    log.debug('comparing key name=%s', k1.name)
+    eq(k1.name, k2.name)
+    eq(k1.version_id, k2.version_id)
+    eq(k1.is_latest, k2.is_latest)
+    eq(k1.last_modified, k2.last_modified)
+    if isinstance(k1, DeleteMarker):
+        assert isinstance(k2, DeleteMarker)
+        return
+
+    eq(k1.get_contents_as_string(), k2.get_contents_as_string())
+    eq(k1.metadata, k2.metadata)
+    eq(k1.cache_control, k2.cache_control)
+    eq(k1.content_type, k2.content_type)
+    eq(k1.content_encoding, k2.content_encoding)
+    eq(k1.content_disposition, k2.content_disposition)
+    eq(k1.content_language, k2.content_language)
+    eq(k1.etag, k2.etag)
+    if check_extra:
+        eq(k1.owner.id, k2.owner.id)
+        eq(k1.owner.display_name, k2.owner.display_name)
+    eq(k1.storage_class, k2.storage_class)
+    eq(k1.size, k2.size)
+    eq(k1.encrypted, k2.encrypted)
+
+class RadosZone(Zone):
+    def __init__(self, name, zonegroup = None, cluster = None, data = None, zone_id = None, gateways = None):
+        super(RadosZone, self).__init__(name, zonegroup, cluster, data, zone_id, gateways)
+
+    def  tier_type(self):
+        return "rados"
+
+
+    class Conn(ZoneConn):
+        def __init__(self, zone, credentials):
+            super(RadosZone.Conn, self).__init__(zone, credentials)
+
+        def get_bucket(self, name):
+            return self.conn.get_bucket(name)
+
+        def create_bucket(self, name):
+            return self.conn.create_bucket(name)
+
+        def delete_bucket(self, name):
+            return self.conn.delete_bucket(name)
+
+        def check_bucket_eq(self, zone_conn, bucket_name):
+            log.info('comparing bucket=%s zones={%s, %s}', bucket_name, self.name, zone_conn.name)
+            b1 = self.get_bucket(bucket_name)
+            b2 = zone_conn.get_bucket(bucket_name)
+
+            b1_versions = b1.list_versions()
+            log.debug('bucket1 objects:')
+            for o in b1_versions:
+                log.debug('o=%s', o.name)
+
+            b2_versions = b2.list_versions()
+            log.debug('bucket2 objects:')
+            for o in b2_versions:
+                log.debug('o=%s', o.name)
+
+            for k1, k2 in zip_longest(b1_versions, b2_versions):
+                if k1 is None:
+                    log.critical('key=%s is missing from zone=%s', k2.name, self.name)
+                    assert False
+                if k2 is None:
+                    log.critical('key=%s is missing from zone=%s', k1.name, zone_conn.name)
+                    assert False
+
+                check_object_eq(k1, k2)
+
+                if isinstance(k1, DeleteMarker):
+                    # verify that HEAD sees a delete marker
+                    assert b1.get_key(k1.name) is None
+                    assert b2.get_key(k2.name) is None
+                else:
+                    # now get the keys through a HEAD operation, verify that the available data is the same
+                    k1_head = b1.get_key(k1.name, version_id=k1.version_id)
+                    k2_head = b2.get_key(k2.name, version_id=k2.version_id)
+                    check_object_eq(k1_head, k2_head, False)
+
+                    if k1.version_id:
+                        # compare the olh to make sure they agree about the current version
+                        k1_olh = b1.get_key(k1.name)
+                        k2_olh = b2.get_key(k2.name)
+                        # if there's a delete marker, HEAD will return None
+                        if k1_olh or k2_olh:
+                            check_object_eq(k1_olh, k2_olh, False)
+
+            log.info('success, bucket identical: bucket=%s zones={%s, %s}', bucket_name, self.name, zone_conn.name)
+
+            return True
+
+    def get_conn(self, credentials):
+        return self.Conn(self, credentials)
+
diff --git a/src/test/rgw/test-ceph-diff-sorted.sh b/src/test/rgw/test-ceph-diff-sorted.sh
new file mode 100755
index 00000000..dddf4ae1
--- /dev/null
+++ b/src/test/rgw/test-ceph-diff-sorted.sh
@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+
+# set -e -x
+
+. "`dirname $0`/test-rgw-common.sh"
+
+temp_prefix="/tmp/`basename $0`-$$"
+
+short=${temp_prefix}-short
+short_w_blank=${temp_prefix}-short-w-blank
+long=${temp_prefix}-long
+unsorted=${temp_prefix}-unsorted
+empty=${temp_prefix}-empty
+fake=${temp_prefix}-fake
+
+out1=${temp_prefix}-out1
+out2=${temp_prefix}-out2
+
+cat >"${short}" <<EOF
+bear
+fox
+hippo
+zebra
+EOF
+
+cat >"${short_w_blank}" <<EOF
+bear
+fox
+hippo
+
+zebra
+EOF
+
+cat >"${long}" <<EOF
+badger
+cuttlefish
+fox
+llama
+octopus
+penguine
+seal
+squid
+whale
+yak
+zebra
+EOF
+
+cat >"${unsorted}" <<EOF
+bear
+hippo
+fox
+zebra
+EOF
+
+touch $empty
+
+#### testing ####
+
+# test perfect match
+ceph-diff-sorted $long $long >"${out1}"
+$assert $? -eq 0
+$assert $(cat $out1 | wc -l) -eq 0
+
+# test non-match; use /bin/diff to verify
+/bin/diff $short $long >"${out2}"
+ceph-diff-sorted $short $long >"${out1}"
+$assert $? -eq 1
+$assert $(cat $out1 | grep '^<' | wc -l) -eq $(cat $out2 | grep '^<' | wc -l)
+$assert $(cat $out1 | grep '^>' | wc -l) -eq $(cat $out2 | grep '^>' | wc -l)
+
+/bin/diff $long $short >"${out2}"
+ceph-diff-sorted $long $short >"${out1}"
+$assert $? -eq 1
+$assert $(cat $out1 | grep '^<' | wc -l) -eq $(cat $out2 | grep '^<' | wc -l)
+$assert $(cat $out1 | grep '^>' | wc -l) -eq $(cat $out2 | grep '^>' | wc -l)
+
+# test w blank line
+ceph-diff-sorted $short $short_w_blank 2>/dev/null
+$assert $? -eq 4
+
+ceph-diff-sorted $short_w_blank $short 2>/dev/null
+$assert $? -eq 4
+
+# test unsorted input
+ceph-diff-sorted $short $unsorted >"${out2}" 2>/dev/null
+$assert $? -eq 4
+
+ceph-diff-sorted $unsorted $short >"${out2}" 2>/dev/null
+$assert $? -eq 4
+
+# test bad # of args
+ceph-diff-sorted 2>/dev/null
+$assert $? -eq 2
+
+ceph-diff-sorted $short 2>/dev/null
+$assert $? -eq 2
+
+# test bad file path
+
+ceph-diff-sorted $short $fake 2>/dev/null
+$assert $? -eq 3
+
+ceph-diff-sorted $fake $short 2>/dev/null
+$assert $? -eq 3
+
+#### clean-up ####
+
+/bin/rm -f $short $short_w_blank $long $unsorted $empty $out1 $out2
diff --git a/src/test/rgw/test-rgw-call.sh b/src/test/rgw/test-rgw-call.sh
new file mode 100755
index 00000000..49399ebc
--- /dev/null
+++ b/src/test/rgw/test-rgw-call.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+. "`dirname $0`/test-rgw-common.sh"
+. "`dirname $0`/test-rgw-meta-sync.sh"
+
+# Do not use eval here. We have eval in test-rgw-common.sh:x(), so adding
+# one here creates a double-eval situation. Passing arguments with spaces
+# becomes impossible when double-eval strips escaping and quotes.
+$@
diff --git a/src/test/rgw/test-rgw-common.sh b/src/test/rgw/test-rgw-common.sh
new file mode 100644
index 00000000..0f8918f2
--- /dev/null
+++ b/src/test/rgw/test-rgw-common.sh
@@ -0,0 +1,156 @@
+#!/usr/bin/env bash
+
+rgw_flags="--debug-rgw=20 --debug-ms=1"
+
+function _assert {
+  src=$1; shift
+  lineno=$1; shift
+  [ "$@" ] || echo "$src: $lineno: assert failed: $@" || exit 1
+}
+
+assert="eval _assert \$BASH_SOURCE \$LINENO"
+
+function var_to_python_json_index {
+  echo "['$1']" | sed "s/\./'\]\['/g"
+}
+
+function json_extract {
+var=""
+[ "$1" != "" ] && var=$(var_to_python_json_index $1)
+shift
+python - <<END
+import json
+s='$@'
+data = json.loads(s) 
+print data$var
+END
+}
+
+function python_array_len {
+python - <<END
+arr=$@
+print len(arr)
+END
+}
+
+function project_python_array_field {
+var=$(var_to_python_json_index $1)
+shift
+python - <<END
+arr=$@
+s='( '
+for x in arr:
+    s += '"' + str(x$var) + '" '
+s += ')'
+print s
+END
+}
+
+
+x() {
+  # echo "x " "$@" >&2
+  eval "$@"
+}
+
+
+script_dir=`dirname $0`
+root_path=`(cd $script_dir/../..; pwd)`
+
+mstart=$root_path/mstart.sh
+mstop=$root_path/mstop.sh
+mrun=$root_path/mrun
+mrgw=$root_path/mrgw.sh
+
+function start_ceph_cluster {
+  [ $# -ne 1 ] && echo "start_ceph_cluster() needs 1 param" && exit 1
+
+  echo "$mstart $1"
+}
+
+function rgw_admin {
+  [ $# -lt 1 ] && echo "rgw_admin() needs 1 param" && exit 1
+
+  echo "$mrun $1 radosgw-admin"
+}
+
+function rgw {
+  [ $# -ne 2 ] && echo "rgw() needs 2 params" && exit 1
+
+  echo "$mrgw $1 $2 $rgw_flags"
+}
+
+function init_first_zone {
+  [ $# -ne 7 ] && echo "init_first_zone() needs 7 params" && exit 1
+
+  cid=$1
+  realm=$2
+  zg=$3
+  zone=$4
+  endpoints=$5
+
+  access_key=$6
+  secret=$7
+
+# initialize realm
+  x $(rgw_admin $cid) realm create --rgw-realm=$realm
+
+# create zonegroup, zone
+  x $(rgw_admin $cid) zonegroup create --rgw-zonegroup=$zg --master --default
+  x $(rgw_admin $cid) zone create --rgw-zonegroup=$zg --rgw-zone=$zone --access-key=${access_key} --secret=${secret} --endpoints=$endpoints --default
+  x $(rgw_admin $cid) user create --uid=zone.user --display-name="Zone User" --access-key=${access_key} --secret=${secret} --system
+
+  x $(rgw_admin $cid) period update --commit
+}
+
+function init_zone_in_existing_zg {
+  [ $# -ne 8 ] && echo "init_zone_in_existing_zg() needs 8 params" && exit 1
+
+  cid=$1
+  realm=$2
+  zg=$3
+  zone=$4
+  master_zg_zone1_port=$5
+  endpoints=$6
+
+  access_key=$7
+  secret=$8
+
+  x $(rgw_admin $cid) realm pull --url=http://localhost:$master_zg_zone1_port --access-key=${access_key} --secret=${secret} --default
+  x $(rgw_admin $cid) zonegroup default --rgw-zonegroup=$zg
+  x $(rgw_admin $cid) zone create --rgw-zonegroup=$zg --rgw-zone=$zone --access-key=${access_key} --secret=${secret} --endpoints=$endpoints
+  x $(rgw_admin $cid) period update --commit --url=http://localhost:$master_zg_zone1_port --access-key=${access_key} --secret=${secret}
+}
+
+function init_first_zone_in_slave_zg {
+  [ $# -ne 8 ] && echo "init_first_zone_in_slave_zg() needs 8 params" && exit 1
+
+  cid=$1
+  realm=$2
+  zg=$3
+  zone=$4
+  master_zg_zone1_port=$5
+  endpoints=$6
+
+  access_key=$7
+  secret=$8
+
+# create zonegroup, zone
+  x $(rgw_admin $cid) realm pull --url=http://localhost:$master_zg_zone1_port --access-key=${access_key} --secret=${secret}
+  x $(rgw_admin $cid) realm default --rgw-realm=$realm
+  x $(rgw_admin $cid) zonegroup create --rgw-realm=$realm --rgw-zonegroup=$zg --endpoints=$endpoints --default
+  x $(rgw_admin $cid) zonegroup default --rgw-zonegroup=$zg
+
+  x $(rgw_admin $cid) zone create --rgw-zonegroup=$zg --rgw-zone=$zone --access-key=${access_key} --secret=${secret} --endpoints=$endpoints
+  x $(rgw_admin $cid) zone default --rgw-zone=$zone
+  x $(rgw_admin $cid) zonegroup add --rgw-zonegroup=$zg --rgw-zone=$zone
+
+  x $(rgw_admin $cid) user create --uid=zone.user --display-name="Zone User" --access-key=${access_key} --secret=${secret} --system
+  x $(rgw_admin $cid) period update --commit --url=localhost:$master_zg_zone1_port --access-key=${access_key} --secret=${secret}
+
+}
+
+function call_rgw_admin {
+  cid=$1
+  shift 1
+  x $(rgw_admin $cid) "$@"
+}
diff --git a/src/test/rgw/test-rgw-meta-sync.sh b/src/test/rgw/test-rgw-meta-sync.sh
new file mode 100755
index 00000000..18f42529
--- /dev/null
+++ b/src/test/rgw/test-rgw-meta-sync.sh
@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+
+. "`dirname $0`/test-rgw-common.sh"
+
+set -e
+
+function get_metadata_sync_status {
+  cid=$1
+  realm=$2
+
+  meta_sync_status_json=`$(rgw_admin $cid) --rgw-realm=$realm metadata sync status`
+
+  global_sync_status=$(json_extract sync_status.info.status $meta_sync_status_json)
+  num_shards=$(json_extract sync_status.info.num_shards $meta_sync_status_json)
+
+  echo "sync_status: $global_sync_status"
+
+  sync_markers=$(json_extract sync_status.markers $meta_sync_status_json)
+
+  num_shards2=$(python_array_len $sync_markers)
+
+  [ "$global_sync_status" == "sync" ] && $assert $num_shards2 -eq $num_shards
+
+  sync_states=$(project_python_array_field val.state $sync_markers)
+  eval secondary_status=$(project_python_array_field val.marker $sync_markers)
+}
+
+function get_metadata_log_status {
+  cid=$1
+  realm=$2
+
+  master_mdlog_status_json=`$(rgw_admin $cid) --rgw-realm=$realm mdlog status`
+  master_meta_status=$(json_extract "" $master_mdlog_status_json)
+
+  eval master_status=$(project_python_array_field marker $master_meta_status)
+}
+
+function wait_for_meta_sync {
+  master_id=$1
+  cid=$2
+  realm=$3
+
+  get_metadata_log_status $master_id $realm
+  echo "master_status=${master_status[*]}"
+
+  while true; do
+    get_metadata_sync_status $cid $realm
+
+    echo "secondary_status=${secondary_status[*]}"
+
+    fail=0
+    for i in `seq 0 $((num_shards-1))`; do
+      if [ "${master_status[$i]}" \> "${secondary_status[$i]}" ]; then
+        echo "shard $i not done syncing (${master_status[$i]} > ${secondary_status[$i]})"
+        fail=1
+        break
+      fi
+    done
+
+    [ $fail -eq 0 ] && echo "Success" && return || echo "Sync not complete"
+
+    sleep 5
+  done
+}
+
diff --git a/src/test/rgw/test-rgw-multisite.sh b/src/test/rgw/test-rgw-multisite.sh
new file mode 100755
index 00000000..94beef95
--- /dev/null
+++ b/src/test/rgw/test-rgw-multisite.sh
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+[ $# -lt 1 ] && echo "usage: $0 <num-clusters>" && exit 1
+
+num_clusters=$1
+
+[ $num_clusters -lt 1 ] && echo "clusters num must be at least 1" && exit 1
+
+. "`dirname $0`/test-rgw-common.sh"
+. "`dirname $0`/test-rgw-meta-sync.sh"
+
+set -e
+
+realm_name=earth
+zg=zg1
+
+system_access_key="1234567890"
+system_secret="pencil"
+
+# bring up first cluster
+x $(start_ceph_cluster c1) -n
+
+# create realm, zonegroup, zone, start rgw
+init_first_zone c1 $realm_name $zg ${zg}-1 8001 $system_access_key $system_secret
+x $(rgw c1 8001)
+
+output=`$(rgw_admin c1) realm get`
+
+echo realm_status=$output
+
+# bring up next clusters
+
+i=2
+while [ $i -le $num_clusters ]; do
+  x $(start_ceph_cluster c$i) -n
+
+  # create new zone, start rgw
+  init_zone_in_existing_zg c$i $realm_name $zg ${zg}-${i} 8001 $((8000+$i)) $zone_port $system_access_key $system_secret
+  x $(rgw c$i $((8000+$i)))
+
+  i=$((i+1))
+done
+
+i=2
+while [ $i -le $num_clusters ]; do
+  wait_for_meta_sync c1 c$i $realm_name
+
+  i=$((i+1))
+done
+
diff --git a/src/test/rgw/test_http_manager.cc b/src/test/rgw/test_http_manager.cc
new file mode 100644
index 00000000..ed49e4c8
--- /dev/null
+++ b/src/test/rgw/test_http_manager.cc
@@ -0,0 +1,94 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2015 Red Hat
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+#include "rgw/rgw_rados.h"
+#include "rgw/rgw_http_client.h"
+#include "global/global_init.h"
+#include "common/ceph_argparse.h"
+#include <curl/curl.h>
+#include <boost/asio/ip/tcp.hpp>
+#include <boost/asio/write.hpp>
+#include <thread>
+#include <gtest/gtest.h>
+
+TEST(HTTPManager, ReadTruncated)
+{
+  using tcp = boost::asio::ip::tcp;
+  tcp::endpoint endpoint(tcp::v4(), 0);
+  boost::asio::io_context ioctx;
+  tcp::acceptor acceptor(ioctx);
+  acceptor.open(endpoint.protocol());
+  acceptor.bind(endpoint);
+  acceptor.listen();
+
+  std::thread server{[&] {
+    tcp::socket socket{ioctx};
+    acceptor.accept(socket);
+    std::string_view response =
+        "HTTP/1.1 200 OK\r\n"
+        "Content-Length: 1024\r\n"
+        "\r\n"
+        "short body";
+    boost::asio::write(socket, boost::asio::buffer(response));
+  }};
+  const auto url = std::string{"http://127.0.0.1:"} + std::to_string(acceptor.local_endpoint().port());
+
+  RGWHTTPClient client{g_ceph_context, "GET", url};
+  EXPECT_EQ(-EAGAIN, RGWHTTP::process(&client));
+
+  server.join();
+}
+
+TEST(HTTPManager, SignalThread)
+{
+  auto cct = g_ceph_context;
+  RGWHTTPManager http(cct);
+
+  ASSERT_EQ(0, http.start());
+
+  // default pipe buffer size according to man pipe
+  constexpr size_t max_pipe_buffer_size = 65536;
+  // each signal writes 4 bytes to the pipe
+  constexpr size_t max_pipe_signals = max_pipe_buffer_size / sizeof(uint32_t);
+  // add_request and unregister_request
+  constexpr size_t pipe_signals_per_request = 2;
+  // number of http requests to fill the pipe buffer
+  constexpr size_t max_requests = max_pipe_signals / pipe_signals_per_request;
+
+  // send one extra request to test that we don't deadlock
+  constexpr size_t num_requests = max_requests + 1;
+
+  for (size_t i = 0; i < num_requests; i++) {
+    RGWHTTPClient client{cct, "PUT", "http://127.0.0.1:80"};
+    http.add_request(&client);
+  }
+}
+
+int main(int argc, char** argv)
+{
+  vector<const char*> args;
+  argv_to_vec(argc, (const char **)argv, args);
+
+  auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT,
+			 CODE_ENVIRONMENT_UTILITY,
+			 CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+  common_init_finish(g_ceph_context);
+
+  rgw_http_client_init(cct->get());
+  rgw_setup_saved_curl_handles();
+  ::testing::InitGoogleTest(&argc, argv);
+  int r = RUN_ALL_TESTS();
+  rgw_release_all_curl_handles();
+  rgw_http_client_cleanup();
+  return r;
+}
diff --git a/src/test/rgw/test_multen.py b/src/test/rgw/test_multen.py
new file mode 100644
index 00000000..91464d33
--- /dev/null
+++ b/src/test/rgw/test_multen.py
@@ -0,0 +1,400 @@
+# Test of mult-tenancy
+
+import json
+import sys
+
+from boto.s3.connection import S3Connection, OrdinaryCallingFormat
+
+# XXX once we're done, break out the common code into a library module
+#     See  https://github.com/ceph/ceph/pull/8646
+import test_multi as t
+
+class TestException(Exception):
+    pass
+
+#
+# Create a traditional user, S3-only, global (empty) tenant
+#
+def test2(cluster):
+    uid = "tester2"
+    display_name = "'Test User 2'"
+    access_key = "tester2KEY"
+    s3_secret = "test3pass"
+    cmd = t.build_cmd('--uid', uid,
+                    '--display-name', display_name,
+                    '--access-key', access_key,
+                    '--secret', s3_secret,
+                    "user create")
+    out, ret = cluster.rgw_admin(cmd, check_retcode=False)
+    if ret != 0:
+        raise TestException("failed command: user create --uid %s" % uid)
+
+    try:
+        outj = json.loads(out.decode('utf-8'))
+    except ValueError:
+        raise TestException("invalid json after: user create --uid %s" % uid)
+    if not isinstance(outj, dict):
+        raise TestException("bad json after: user create --uid %s" % uid)
+    if outj['user_id'] != uid:
+        raise TestException(
+            "command: user create --uid %s, returned user_id %s" %
+            (uid, outj['user_id']))
+
+#
+# Create a tenantized user with --tenant foo
+#
+def test3(cluster):
+    tid = "testx3"
+    uid = "tester3"
+    display_name = "Test_User_3"
+    access_key = "tester3KEY"
+    s3_secret = "test3pass"
+    cmd = t.build_cmd(
+        '--tenant', tid,
+        '--uid', uid,
+        '--display-name', display_name,
+        '--access-key', access_key,
+        '--secret', s3_secret,
+        "user create")
+    out, ret = cluster.rgw_admin(cmd, check_retcode=False)
+    if ret != 0:
+        raise TestException("failed command: user create --uid %s" % uid)
+
+    try:
+        outj = json.loads(out.decode('utf-8'))
+    except ValueError:
+        raise TestException("invalid json after: user create --uid %s" % uid)
+    if not isinstance(outj, dict):
+        raise TestException("bad json after: user create --uid %s" % uid)
+    tid_uid = "%s$%s" % (tid, uid)
+    if outj['user_id'] != tid_uid:
+        raise TestException(
+            "command: user create --uid %s, returned user_id %s" %
+            (tid_uid, outj['user_id']))
+
+#
+# Create a tenantized user with a subuser
+#
+# N.B. The aim of this test is not just to create a subuser, but to create
+# the key with a separate command, which does not use --tenant, but extracts
+# the tenant from the subuser. No idea why we allow this. There was some kind
+# of old script that did this.
+#
+def test4(cluster):
+    tid = "testx4"
+    uid = "tester4"
+    subid = "test4"
+
+    display_name = "Test_User_4"
+    cmd = t.build_cmd(
+        '--tenant', tid,
+        '--uid', uid,
+        '--display-name', display_name,
+        '--subuser', '%s:%s' % (uid, subid),
+        '--key-type', 'swift',
+        '--access', 'full',
+        "user create")
+    out, ret = cluster.rgw_admin(cmd, check_retcode=False)
+    if ret != 0:
+        raise TestException("failed command: user create --uid %s" % uid)
+
+    try:
+        outj = json.loads(out.decode('utf-8'))
+    except ValueError:
+        raise TestException("invalid json after: user create --uid %s" % uid)
+    if not isinstance(outj, dict):
+        raise TestException("bad json after: user create --uid %s" % uid)
+    tid_uid = "%s$%s" % (tid, uid)
+    if outj['user_id'] != tid_uid:
+        raise TestException(
+            "command: user create --uid %s, returned user_id %s" %
+            (tid_uid, outj['user_id']))
+
+    # Note that this tests a way to identify a fully-qualified subuser
+    # without --tenant and --uid. This is a historic use that we support.
+    swift_secret = "test3pass"
+    cmd = t.build_cmd(
+        '--subuser', "'%s$%s:%s'" % (tid, uid, subid),
+        '--key-type', 'swift',
+        '--secret', swift_secret,
+        "key create")
+    out, ret = cluster.rgw_admin(cmd, check_retcode=False)
+    if ret != 0:
+        raise TestException("failed command: key create --uid %s" % uid)
+
+    try:
+        outj = json.loads(out.decode('utf-8'))
+    except ValueError:
+        raise TestException("invalid json after: key create --uid %s" % uid)
+    if not isinstance(outj, dict):
+        raise TestException("bad json after: key create --uid %s" % uid)
+    tid_uid = "%s$%s" % (tid, uid)
+    if outj['user_id'] != tid_uid:
+        raise TestException(
+            "command: key create --uid %s, returned user_id %s" %
+            (tid_uid, outj['user_id']))
+    # These tests easily can throw KeyError, needs a try: XXX
+    skj = outj['swift_keys'][0]
+    if skj['secret_key'] != swift_secret:
+        raise TestException(
+            "command: key create --uid %s, returned swift key %s" %
+            (tid_uid, skj['secret_key']))
+
+#
+# Access the cluster, create containers in two tenants, verify it all works.
+#
+
+def test5_add_s3_key(cluster, tid, uid):
+    secret = "%spass" % uid
+    if tid:
+        tid_uid = "%s$%s" % (tid, uid)
+    else:
+        tid_uid = uid
+
+    cmd = t.build_cmd(
+        '--uid', "'%s'" % (tid_uid,),
+        '--access-key', uid,
+        '--secret', secret,
+        "key create")
+    out, ret = cluster.rgw_admin(cmd, check_retcode=False)
+    if ret != 0:
+        raise TestException("failed command: key create --uid %s" % uid)
+
+    try:
+        outj = json.loads(out.decode('utf-8'))
+    except ValueError:
+        raise TestException("invalid json after: key create --uid %s" % uid)
+    if not isinstance(outj, dict):
+        raise TestException("bad json after: key create --uid %s" % uid)
+    if outj['user_id'] != tid_uid:
+        raise TestException(
+            "command: key create --uid %s, returned user_id %s" %
+            (uid, outj['user_id']))
+    skj = outj['keys'][0]
+    if skj['secret_key'] != secret:
+        raise TestException(
+            "command: key create --uid %s, returned s3 key %s" %
+            (uid, skj['secret_key']))
+
+def test5_add_swift_key(cluster, tid, uid, subid):
+    secret = "%spass" % uid
+    if tid:
+        tid_uid = "%s$%s" % (tid, uid)
+    else:
+        tid_uid = uid
+
+    cmd = t.build_cmd(
+        '--subuser', "'%s:%s'" % (tid_uid, subid),
+        '--key-type', 'swift',
+        '--secret', secret,
+        "key create")
+    out, ret = cluster.rgw_admin(cmd, check_retcode=False)
+    if ret != 0:
+        raise TestException("failed command: key create --uid %s" % uid)
+
+    try:
+        outj = json.loads(out.decode('utf-8'))
+    except ValueError:
+        raise TestException("invalid json after: key create --uid %s" % uid)
+    if not isinstance(outj, dict):
+        raise TestException("bad json after: key create --uid %s" % uid)
+    if outj['user_id'] != tid_uid:
+        raise TestException(
+            "command: key create --uid %s, returned user_id %s" %
+            (uid, outj['user_id']))
+    # XXX checking wrong thing here (S3 key)
+    skj = outj['keys'][0]
+    if skj['secret_key'] != secret:
+        raise TestException(
+            "command: key create --uid %s, returned s3 key %s" %
+            (uid, skj['secret_key']))
+
+def test5_make_user(cluster, tid, uid, subid):
+    """
+    :param tid: Tenant ID string or None for the legacy tenant
+    :param uid: User ID string
+    :param subid: Subuser ID, may be None for S3-only users
+    """
+    display_name = "'Test User %s'" % uid
+
+    cmd = ""
+    if tid:
+        cmd = t.build_cmd(cmd,
+            '--tenant', tid)
+    cmd = t.build_cmd(cmd,
+        '--uid', uid,
+        '--display-name', display_name)
+    if subid:
+        cmd = t.build_cmd(cmd,
+            '--subuser', '%s:%s' % (uid, subid),
+            '--key-type', 'swift')
+    cmd = t.build_cmd(cmd,
+        '--access', 'full',
+        "user create")
+
+    out, ret = cluster.rgw_admin(cmd, check_retcode=False)
+    if ret != 0:
+        raise TestException("failed command: user create --uid %s" % uid)
+    try:
+        outj = json.loads(out.decode('utf-8'))
+    except ValueError:
+        raise TestException("invalid json after: user create --uid %s" % uid)
+    if not isinstance(outj, dict):
+        raise TestException("bad json after: user create --uid %s" % uid)
+    if tid:
+        tid_uid = "%s$%s" % (tid, uid)
+    else:
+        tid_uid = uid
+    if outj['user_id'] != tid_uid:
+        raise TestException(
+            "command: user create --uid %s, returned user_id %s" %
+            (tid_uid, outj['user_id']))
+
+    #
+    # For now, this uses hardcoded passwords based on uid.
+    # They are all different for ease of debugging in case something crosses.
+    #
+    test5_add_s3_key(cluster, tid, uid)
+    if subid:
+        test5_add_swift_key(cluster, tid, uid, subid)
+
+def test5_poke_s3(cluster):
+
+    bucketname = "test5cont1"
+    objname = "obj1"
+
+    # Not sure if we like useless information printed, but the rest of the
+    # test framework is insanely talkative when it executes commands.
+    # So, to keep it in line and have a marker when things go wrong, this.
+    print("PUT bucket %s object %s for tenant A (empty)" %
+          (bucketname, objname))
+    c = S3Connection(
+        aws_access_key_id="tester5a",
+        aws_secret_access_key="tester5apass",
+        is_secure=False,
+        host="localhost",
+        port = cluster.port,
+        calling_format = OrdinaryCallingFormat())
+
+    bucket = c.create_bucket(bucketname)
+
+    key = bucket.new_key(objname)
+    headers = { "Content-Type": "text/plain" }
+    key.set_contents_from_string(b"Test5A\n", headers)
+    key.set_acl('public-read')
+
+    #
+    # Now it's getting interesting. We're logging into a tenantized user.
+    #
+    print("PUT bucket %s object %s for tenant B" % (bucketname, objname))
+    c = S3Connection(
+        aws_access_key_id="tester5b1",
+        aws_secret_access_key="tester5b1pass",
+        is_secure=False,
+        host="localhost",
+        port = cluster.port,
+        calling_format = OrdinaryCallingFormat())
+
+    bucket = c.create_bucket(bucketname)
+    bucket.set_canned_acl('public-read')
+
+    key = bucket.new_key(objname)
+    headers = { "Content-Type": "text/plain" }
+    key.set_contents_from_string(b"Test5B\n", headers)
+    key.set_acl('public-read')
+
+    #
+    # Finally, let's fetch a couple of objects and verify that they
+    # are what they should be and we didn't get them overwritten.
+    # Note that we access one of objects across tenants using the colon.
+    #
+    print("GET bucket %s object %s for tenants A and B" %
+          (bucketname, objname))
+    c = S3Connection(
+        aws_access_key_id="tester5a",
+        aws_secret_access_key="tester5apass",
+        is_secure=False,
+        host="localhost",
+        port = cluster.port,
+        calling_format = OrdinaryCallingFormat())
+
+    bucket = c.get_bucket(bucketname)
+
+    key = bucket.get_key(objname)
+    body = key.get_contents_as_string()
+    if body != b"Test5A\n":
+        raise TestException("failed body check, bucket %s object %s" %
+                            (bucketname, objname))
+
+    bucket = c.get_bucket("test5b:"+bucketname)
+    key = bucket.get_key(objname)
+    body = key.get_contents_as_string()
+    if body != b"Test5B\n":
+        raise TestException(
+            "failed body check, tenant %s bucket %s object %s" %
+            ("test5b", bucketname, objname))
+
+    print("Poke OK")
+
+
+def test5(cluster):
+    # Plan:
+    # 0. create users tester5a and test5b$tester5b1 test5b$tester5b2
+    # 1. create buckets "test5cont" under test5a and test5b
+    # 2. create objects in the buckets
+    # 3. access objects (across users in container test5b)
+
+    test5_make_user(cluster, None, "tester5a", "test5a")
+    test5_make_user(cluster, "test5b", "tester5b1", "test5b1")
+    test5_make_user(cluster, "test5b", "tester5b2", "test5b2")
+
+    test5_poke_s3(cluster)
+
+
+# XXX this parse_args boolean makes no sense. we should pass argv[] instead,
+#     possibly empty. (copied from test_multi, correct it there too)
+def init(parse_args):
+
+    #argv = []
+    #if parse_args:
+    #    argv = sys.argv[1:]
+    #args = parser.parse_args(argv)
+
+    #rgw_multi = RGWMulti(int(args.num_zones))
+    #rgw_multi.setup(not args.no_bootstrap)
+
+    # __init__():
+    port = 8001
+    clnum = 1  # number of clusters
+    clid = 1   # 1-based
+    cluster = t.RGWCluster(clid, port)
+
+    # setup():
+    cluster.start()
+    cluster.start_rgw()
+
+    # The cluster is always reset at this point, so we don't need to list
+    # users or delete pre-existing users.
+
+    try:
+        test2(cluster)
+        test3(cluster)
+        test4(cluster)
+        test5(cluster)
+    except TestException as e:
+        cluster.stop_rgw()
+        cluster.stop()
+        sys.stderr.write("FAIL\n")
+        sys.stderr.write("%s\n" % str(e))
+        return 1
+
+    # teardown():
+    cluster.stop_rgw()
+    cluster.stop()
+    return 0
+
+def setup_module():
+    return init(False)
+
+if __name__ == "__main__":
+    sys.exit(init(True))
diff --git a/src/test/rgw/test_multi.md b/src/test/rgw/test_multi.md
new file mode 100644
index 00000000..6acba7d7
--- /dev/null
+++ b/src/test/rgw/test_multi.md
@@ -0,0 +1,59 @@
+# Multi Site Test Framework
+This framework allows you to write and run tests against a **local** multi-cluster environment. The framework is using the `mstart.sh` script in order to setup the environment according to a configuration file, and then uses the [nose](https://nose.readthedocs.io/en/latest/) test framework to actually run the tests.
+Tests are written in python2.7, but can invoke shell scripts, binaries etc.
+## Running Tests
+Entry point for all tests is `/path/to/ceph/src/test/rgw/test_multi.py`. And the actual tests are located inside the `/path/to/ceph/src/test/rgw/rgw_multi` subdirectory.
+So, to run all tests use:
+```
+$ cd /path/to/ceph/src/test/rgw/
+$ nosetests test_multi.py
+```
+This will assume a configuration file called `/path/to/ceph/src/test/rgw/test_multi.conf` exists.
+To use a different configuration file, set the `RGW_MULTI_TEST_CONF` environment variable to point to that file.
+Since we use the same entry point file for all tests, running specific tests is possible using the following format:
+```
+$ nosetests test_multi.py:<specific_test_name>
+```
+To run miltiple tests based on wildcard string, use the following format:
+```
+$ nosetests test_multi.py -m "<wildcard string>"
+```
+Note that the test to run, does not have to be inside the `test_multi.py` file.
+Note that different options for running specific and multiple tests exists in the [nose documentation](https://nose.readthedocs.io/en/latest/usage.html#options), as well as other options to control the execution of the tests.
+## Configuration
+### Environment Variables
+Following RGW environment variables are taken into consideration when running the tests:
+ - `RGW_FRONTEND`: used to change frontend to 'civetweb' or 'beast' (default)
+ - `RGW_VALGRIND`: used to run the radosgw under valgrind. e.g. RGW_VALGRIND=yes
+Other environment variables used to configure elements other than RGW can also be used as they are used in vstart.sh. E.g. MON, OSD, MGR, MSD
+The configuration file for the run has 3 sections:
+### Default
+This section holds the following parameters:
+ - `num_zonegroups`: number of zone groups (integer, default 1)
+ - `num_zones`: number of regular zones in each group (integer, default 3)
+ - `num_ps_zones`: number of pubsub zones in each group (integer, default 0)         
+ - `num_az_zones`: number of archive zones (integer, default 0, max value 1)
+ - `gateways_per_zone`: number of RADOS gateways per zone (integer, default 2)
+ - `no_bootstrap`: whether to assume that the cluster is already up and does not need to be setup again. If set to "false", it will try to re-run the cluster, so, `mstop.sh` must be called beforehand. Should be set to false, anytime the configuration is changed. Otherwise, and assuming the cluster is already up, it should be set to "true" to save on execution time (boolean, default false)
+ - `log_level`: console log level of the logs in the tests, note that any program invoked from the test my emit logs regardless of that setting (integer, default 20)
+    - 20 and up -> DEBUG
+    - 10 and up -> INFO
+    - 5 and up -> WARNING
+    - 1 and up -> ERROR
+    - CRITICAL is always logged
+- `log_file`: log file name. If not set, only console logs exists (string, default None)
+- `file_log_level`: file log level of the logs in the tests. Similar to `log_level`
+- `tenant`: name of tenant (string, default None)
+- `checkpoint_retries`: *TODO* (integer, default 60)
+- `checkpoint_delay`: *TODO* (integer, default 5)
+- `reconfigure_delay`: *TODO* (integer, default 5)          
+### Elasticsearch
+*TODO*
+### Cloud
+*TODO*
+### PubSub
+*TODO*
+## Writing Tests
+New tests should be added into the `/path/to/ceph/src/test/rgw/rgw_multi` subdirectory.
+- Base classes are in: `/path/to/ceph/src/test/rgw/rgw_multi/multisite.py`
+- `/path/to/ceph/src/test/rgw/rgw_multi/tests.py` holds the majority of the tests, but also many utility and infrastructure functions that could be used in other tests files 
diff --git a/src/test/rgw/test_multi.py b/src/test/rgw/test_multi.py
new file mode 100644
index 00000000..5b1e7f2e
--- /dev/null
+++ b/src/test/rgw/test_multi.py
@@ -0,0 +1,418 @@
+import subprocess
+import os
+import random
+import string
+import argparse
+import sys
+import logging
+try:
+    import configparser
+except ImportError:
+    import ConfigParser as configparser
+
+import nose.core
+
+from rgw_multi import multisite
+from rgw_multi.zone_rados import RadosZone as RadosZone
+from rgw_multi.zone_es import ESZone as ESZone
+from rgw_multi.zone_es import ESZoneConfig as ESZoneConfig
+from rgw_multi.zone_cloud import CloudZone as CloudZone
+from rgw_multi.zone_cloud import CloudZoneConfig as CloudZoneConfig
+from rgw_multi.zone_ps import PSZone as PSZone
+from rgw_multi.zone_ps import PSZoneConfig as PSZoneConfig
+
+# make tests from rgw_multi.tests available to nose
+from rgw_multi.tests import *
+from rgw_multi.tests_es import *
+from rgw_multi.tests_ps import *
+
+mstart_path = os.getenv('MSTART_PATH')
+if mstart_path is None:
+    mstart_path = os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + '/../..') + '/'
+
+test_path = os.path.normpath(os.path.dirname(os.path.realpath(__file__))) + '/'
+
+# configure logging for the tests module
+log = logging.getLogger('rgw_multi.tests')
+
+def bash(cmd, **kwargs):
+    log.debug('running cmd: %s', ' '.join(cmd))
+    check_retcode = kwargs.pop('check_retcode', True)
+    kwargs['stdout'] = subprocess.PIPE
+    process = subprocess.Popen(cmd, **kwargs)
+    s = process.communicate()[0].decode('utf-8')
+    log.debug('command returned status=%d stdout=%s', process.returncode, s)
+    if check_retcode:
+        assert(process.returncode == 0)
+    return (s, process.returncode)
+
+class Cluster(multisite.Cluster):
+    """ cluster implementation based on mstart/mrun scripts """
+    def __init__(self, cluster_id):
+        super(Cluster, self).__init__()
+        self.cluster_id = cluster_id
+        self.needs_reset = True
+
+    def admin(self, args = None, **kwargs):
+        """ radosgw-admin command """
+        cmd = [test_path + 'test-rgw-call.sh', 'call_rgw_admin', self.cluster_id]
+        if args:
+            cmd += args
+        cmd += ['--debug-rgw', str(kwargs.pop('debug_rgw', 0))]
+        cmd += ['--debug-ms', str(kwargs.pop('debug_ms', 0))]
+        if kwargs.pop('read_only', False):
+            cmd += ['--rgw-cache-enabled', 'false']
+        return bash(cmd, **kwargs)
+
+    def start(self):
+        cmd = [mstart_path + 'mstart.sh', self.cluster_id]
+        env = None
+        if self.needs_reset:
+            env = os.environ.copy()
+            env['CEPH_NUM_MDS'] = '0'
+            cmd += ['-n']
+            # cmd += ['-o']
+            # cmd += ['rgw_cache_enabled=false']
+        bash(cmd, env=env)
+        self.needs_reset = False
+
+    def stop(self):
+        cmd = [mstart_path + 'mstop.sh', self.cluster_id]
+        bash(cmd)
+
+class Gateway(multisite.Gateway):
+    """ gateway implementation based on mrgw/mstop scripts """
+    def __init__(self, client_id = None, *args, **kwargs):
+        super(Gateway, self).__init__(*args, **kwargs)
+        self.id = client_id
+
+    def start(self, args = None):
+        """ start the gateway """
+        assert(self.cluster)
+        env = os.environ.copy()
+        # to change frontend, set RGW_FRONTEND env variable
+        # e.g. RGW_FRONTEND=civetweb
+        # to run test under valgrind memcheck, set RGW_VALGRIND to 'yes'
+        # e.g. RGW_VALGRIND=yes
+        cmd = [mstart_path + 'mrgw.sh', self.cluster.cluster_id, str(self.port), str(self.ssl_port)]
+        if self.id:
+            cmd += ['-i', self.id]
+        cmd += ['--debug-rgw=20', '--debug-ms=1']
+        if args:
+            cmd += args
+        bash(cmd, env=env)
+
+    def stop(self):
+        """ stop the gateway """
+        assert(self.cluster)
+        cmd = [mstart_path + 'mstop.sh', self.cluster.cluster_id, 'radosgw', self.id]
+        bash(cmd)
+
+def gen_access_key():
+    return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(16))
+
+def gen_secret():
+    return ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(32))
+
+def gen_credentials():
+    return multisite.Credentials(gen_access_key(), gen_secret())
+
+def cluster_name(cluster_num):
+    return 'c' + str(cluster_num)
+
+def zonegroup_name(zonegroup_num):
+    return string.ascii_lowercase[zonegroup_num]
+
+def zone_name(zonegroup_num, zone_num):
+    return zonegroup_name(zonegroup_num) + str(zone_num + 1)
+
+def gateway_port(zonegroup_num, gateway_num):
+    return 8000 + 100 * zonegroup_num + gateway_num
+
+def gateway_name(zonegroup_num, zone_num, gateway_num):
+    return zone_name(zonegroup_num, zone_num) + '-' + str(gateway_num + 1)
+
+def zone_endpoints(zonegroup_num, zone_num, gateways_per_zone):
+    endpoints = []
+    base = gateway_port(zonegroup_num, zone_num * gateways_per_zone)
+    for i in range(0, gateways_per_zone):
+        endpoints.append('http://localhost:' + str(base + i))
+    return endpoints
+
+def get_log_level(log_level):
+    if log_level >= 20:
+        return logging.DEBUG
+    if log_level >= 10:
+        return logging.INFO
+    if log_level >= 5:
+        return logging.WARN
+    if log_level >= 1:
+        return logging.ERROR
+    return logging.CRITICAL
+
+def setup_logging(log_level_console, log_file, log_level_file):
+    if log_file:
+        formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
+        fh = logging.FileHandler(log_file)
+        fh.setFormatter(formatter)
+        fh.setLevel(get_log_level(log_level_file))
+        log.addHandler(fh)
+
+    formatter = logging.Formatter('%(levelname)s %(message)s')
+    ch = logging.StreamHandler()
+    ch.setFormatter(formatter)
+    ch.setLevel(get_log_level(log_level_console))
+    log.addHandler(ch)
+    log.setLevel(get_log_level(log_level_console))
+
+def init(parse_args):
+    cfg = configparser.RawConfigParser({
+                                         'num_zonegroups': 1,
+                                         'num_zones': 3,
+                                         'num_ps_zones': 0,
+                                         'num_az_zones': 0,
+                                         'gateways_per_zone': 2,
+                                         'no_bootstrap': 'false',
+                                         'log_level': 20,
+                                         'log_file': None,
+                                         'file_log_level': 20,
+                                         'tenant': None,
+                                         'checkpoint_retries': 60,
+                                         'checkpoint_delay': 5,
+                                         'reconfigure_delay': 5,
+                                         'use_ssl': 'false',
+                                         })
+    try:
+        path = os.environ['RGW_MULTI_TEST_CONF']
+    except KeyError:
+        path = test_path + 'test_multi.conf'
+
+    try:
+        with open(path) as f:
+            cfg.readfp(f)
+    except:
+        print('WARNING: error reading test config. Path can be set through the RGW_MULTI_TEST_CONF env variable')
+        pass
+
+    parser = argparse.ArgumentParser(
+            description='Run rgw multi-site tests',
+            usage='test_multi [--num-zonegroups <num>] [--num-zones <num>] [--no-bootstrap]')
+
+    section = 'DEFAULT'
+    parser.add_argument('--num-zonegroups', type=int, default=cfg.getint(section, 'num_zonegroups'))
+    parser.add_argument('--num-zones', type=int, default=cfg.getint(section, 'num_zones'))
+    parser.add_argument('--gateways-per-zone', type=int, default=cfg.getint(section, 'gateways_per_zone'))
+    parser.add_argument('--no-bootstrap', action='store_true', default=cfg.getboolean(section, 'no_bootstrap'))
+    parser.add_argument('--log-level', type=int, default=cfg.getint(section, 'log_level'))
+    parser.add_argument('--log-file', type=str, default=cfg.get(section, 'log_file'))
+    parser.add_argument('--file-log-level', type=int, default=cfg.getint(section, 'file_log_level'))
+    parser.add_argument('--tenant', type=str, default=cfg.get(section, 'tenant'))
+    parser.add_argument('--checkpoint-retries', type=int, default=cfg.getint(section, 'checkpoint_retries'))
+    parser.add_argument('--checkpoint-delay', type=int, default=cfg.getint(section, 'checkpoint_delay'))
+    parser.add_argument('--reconfigure-delay', type=int, default=cfg.getint(section, 'reconfigure_delay'))
+    parser.add_argument('--num-ps-zones', type=int, default=cfg.getint(section, 'num_ps_zones'))
+    parser.add_argument('--use-ssl', type=bool, default=cfg.getboolean(section, 'use_ssl'))
+
+
+    es_cfg = []
+    cloud_cfg = []
+    ps_cfg = []
+    az_cfg = []
+
+    for s in cfg.sections():
+        if s.startswith('elasticsearch'):
+            es_cfg.append(ESZoneConfig(cfg, s))
+        elif s.startswith('cloud'):
+            cloud_cfg.append(CloudZoneConfig(cfg, s))
+        elif s.startswith('pubsub'):
+            ps_cfg.append(PSZoneConfig(cfg, s))
+
+    argv = []
+
+    if parse_args:
+        argv = sys.argv[1:]
+
+    args = parser.parse_args(argv)
+    bootstrap = not args.no_bootstrap
+
+    setup_logging(args.log_level, args.log_file, args.file_log_level)
+
+    # start first cluster
+    c1 = Cluster(cluster_name(1))
+    if bootstrap:
+        c1.start()
+    clusters = []
+    clusters.append(c1)
+
+    admin_creds = gen_credentials()
+    admin_user = multisite.User('zone.user')
+
+    user_creds = gen_credentials()
+    user = multisite.User('tester', tenant=args.tenant)
+
+    realm = multisite.Realm('r')
+    if bootstrap:
+        # create the realm on c1
+        realm.create(c1)
+    else:
+        realm.get(c1)
+    period = multisite.Period(realm=realm)
+    realm.current_period = period
+
+    num_es_zones = len(es_cfg)
+    num_cloud_zones = len(cloud_cfg)
+    num_ps_zones_from_conf = len(ps_cfg)
+    num_ps_zones = args.num_ps_zones if num_ps_zones_from_conf == 0 else num_ps_zones_from_conf 
+
+    num_zones = args.num_zones + num_es_zones + num_cloud_zones + args.num_ps_zones
+
+    use_ssl = cfg.getboolean(section, 'use_ssl')
+
+    if use_ssl and bootstrap:
+        cmd = ['openssl', 'req', 
+                '-x509', 
+                '-newkey', 'rsa:4096', 
+                '-sha256', 
+                '-nodes', 
+                '-keyout', 'key.pem', 
+                '-out', 'cert.pem', 
+                '-subj', '/CN=localhost', 
+                '-days', '3650']
+        bash(cmd)
+        # append key to cert
+        fkey = open('./key.pem', 'r')
+        if fkey.mode == 'r':
+            fcert = open('./cert.pem', 'a')
+            fcert.write(fkey.read())
+            fcert.close()
+        fkey.close()
+
+    for zg in range(0, args.num_zonegroups):
+        zonegroup = multisite.ZoneGroup(zonegroup_name(zg), period)
+        period.zonegroups.append(zonegroup)
+
+        is_master_zg = zg == 0
+        if is_master_zg:
+            period.master_zonegroup = zonegroup
+
+        for z in range(0, num_zones):
+            is_master = z == 0
+            # start a cluster, or use c1 for first zone
+            cluster = None
+            if is_master_zg and is_master:
+                cluster = c1
+            else:
+                cluster = Cluster(cluster_name(len(clusters) + 1))
+                clusters.append(cluster)
+                if bootstrap:
+                    cluster.start()
+                    # pull realm configuration from the master's gateway
+                    gateway = realm.meta_master_zone().gateways[0]
+                    realm.pull(cluster, gateway, admin_creds)
+
+            endpoints = zone_endpoints(zg, z, args.gateways_per_zone)
+            if is_master:
+                if bootstrap:
+                    # create the zonegroup on its first zone's cluster
+                    arg = []
+                    if is_master_zg:
+                        arg += ['--master']
+                    if len(endpoints): # use master zone's endpoints
+                        arg += ['--endpoints', ','.join(endpoints)]
+                    zonegroup.create(cluster, arg)
+                else:
+                    zonegroup.get(cluster)
+
+            es_zone = (z >= args.num_zones and z < args.num_zones + num_es_zones)
+            cloud_zone = (z >= args.num_zones + num_es_zones and z < args.num_zones + num_es_zones + num_cloud_zones)
+            ps_zone = (z >= args.num_zones + num_es_zones + num_cloud_zones)
+
+            # create the zone in its zonegroup
+            zone = multisite.Zone(zone_name(zg, z), zonegroup, cluster)
+            if es_zone:
+                zone_index = z - args.num_zones
+                zone = ESZone(zone_name(zg, z), es_cfg[zone_index].endpoint, zonegroup, cluster)
+            elif cloud_zone:
+                zone_index = z - args.num_zones - num_es_zones
+                ccfg = cloud_cfg[zone_index]
+                zone = CloudZone(zone_name(zg, z), ccfg.endpoint, ccfg.credentials, ccfg.source_bucket,
+                                 ccfg.target_path, zonegroup, cluster)
+            elif ps_zone:
+                zone_index = z - args.num_zones - num_es_zones - num_cloud_zones
+                if num_ps_zones_from_conf == 0:
+                    zone = PSZone(zone_name(zg, z), zonegroup, cluster)
+                else:
+                    pscfg = ps_cfg[zone_index]
+                    zone = PSZone(zone_name(zg, z), zonegroup, cluster,
+                                  full_sync=pscfg.full_sync, retention_days=pscfg.retention_days)
+            else:
+                zone = RadosZone(zone_name(zg, z), zonegroup, cluster)
+
+            if bootstrap:
+                arg = admin_creds.credential_args()
+                if is_master:
+                    arg += ['--master']
+                if len(endpoints):
+                    arg += ['--endpoints', ','.join(endpoints)]
+                zone.create(cluster, arg)
+            else:
+                zone.get(cluster)
+            zonegroup.zones.append(zone)
+            if is_master:
+                zonegroup.master_zone = zone
+
+            zonegroup.zones_by_type.setdefault(zone.tier_type(), []).append(zone)
+
+            if zone.is_read_only():
+                zonegroup.ro_zones.append(zone)
+            else:
+                zonegroup.rw_zones.append(zone)
+
+            # update/commit the period
+            if bootstrap:
+                period.update(zone, commit=True)
+
+            ssl_port_offset = 1000
+            # start the gateways
+            for g in range(0, args.gateways_per_zone):
+                port = gateway_port(zg, g + z * args.gateways_per_zone)
+                client_id = gateway_name(zg, z, g)
+                gateway = Gateway(client_id, 'localhost', port, cluster, zone, 
+                        ssl_port = port+ssl_port_offset if use_ssl else 0)
+                if bootstrap:
+                    gateway.start()
+                zone.gateways.append(gateway)
+
+            if is_master_zg and is_master:
+                if bootstrap:
+                    # create admin user
+                    arg = ['--display-name', '"Zone User"', '--system']
+                    arg += admin_creds.credential_args()
+                    admin_user.create(zone, arg)
+                    # create test user
+                    arg = ['--display-name', '"Test User"']
+                    arg += user_creds.credential_args()
+                    user.create(zone, arg)
+                else:
+                    # read users and update keys
+                    admin_user.info(zone)
+                    admin_creds = admin_user.credentials[0]
+                    arg = []
+                    user.info(zone, arg)
+                    user_creds = user.credentials[0]
+
+    if not bootstrap:
+        period.get(c1)
+
+    config = Config(checkpoint_retries=args.checkpoint_retries,
+                    checkpoint_delay=args.checkpoint_delay,
+                    reconfigure_delay=args.reconfigure_delay,
+                    tenant=args.tenant)
+    init_multi(realm, user, config)
+
+def setup_module():
+    init(False)
+
+if __name__ == "__main__":
+    init(True)
+
diff --git a/src/test/rgw/test_rgw_amqp.cc b/src/test/rgw/test_rgw_amqp.cc
new file mode 100644
index 00000000..0f11b817
--- /dev/null
+++ b/src/test/rgw/test_rgw_amqp.cc
@@ -0,0 +1,456 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "rgw/rgw_amqp.h"
+#include "common/ceph_context.h"
+#include "amqp_mock.h"
+#include <gtest/gtest.h>
+#include <chrono>
+#include <thread>
+#include <atomic>
+
+using namespace rgw;
+
+const std::chrono::milliseconds wait_time(10);
+const std::chrono::milliseconds long_wait_time = wait_time*50;
+
+
+class CctCleaner {
+  CephContext* cct;
+public:
+  CctCleaner(CephContext* _cct) : cct(_cct) {}
+  ~CctCleaner() { 
+#ifdef WITH_SEASTAR
+    delete cct; 
+#else
+    cct->put(); 
+#endif
+  }
+};
+
+auto cct = new CephContext(CEPH_ENTITY_TYPE_CLIENT);
+
+CctCleaner cleaner(cct);
+
+class TestAMQP : public ::testing::Test {
+protected:
+  amqp::connection_ptr_t conn = nullptr;
+  unsigned current_dequeued = 0U;
+
+  void SetUp() override {
+    ASSERT_TRUE(amqp::init(cct));
+  }
+
+  void TearDown() override {
+    amqp::shutdown();
+  }
+
+  // wait for at least one new (since last drain) message to be dequeueud
+  // and then wait for all pending answers to be received
+  void wait_until_drained() {  
+    while (amqp::get_dequeued() == current_dequeued) {
+      std::this_thread::sleep_for(wait_time);
+    }
+    while (amqp::get_inflight() > 0) {
+      std::this_thread::sleep_for(wait_time);
+    }
+    current_dequeued = amqp::get_dequeued();
+  }
+};
+
+TEST_F(TestAMQP, ConnectionOK)
+{
+  const auto connection_number = amqp::get_connection_count();
+  conn = amqp::connect("amqp://localhost", "ex1");
+  EXPECT_TRUE(conn);
+  EXPECT_EQ(amqp::get_connection_count(), connection_number + 1);
+  auto rc = amqp::publish(conn, "topic", "message");
+  EXPECT_EQ(rc, 0);
+}
+
+TEST_F(TestAMQP, ConnectionReuse)
+{
+  amqp::connection_ptr_t conn1 = amqp::connect("amqp://localhost", "ex1");
+  EXPECT_TRUE(conn1);
+  const auto connection_number = amqp::get_connection_count();
+  amqp::connection_ptr_t conn2 = amqp::connect("amqp://localhost", "ex1");
+  EXPECT_TRUE(conn2);
+  EXPECT_EQ(amqp::get_connection_count(), connection_number);
+  auto rc = amqp::publish(conn1, "topic", "message");
+  EXPECT_EQ(rc, 0);
+}
+
+TEST_F(TestAMQP, NameResolutionFail)
+{
+  const auto connection_number = amqp::get_connection_count();
+  conn = amqp::connect("amqp://kaboom", "ex1");
+  EXPECT_TRUE(conn);
+  EXPECT_EQ(amqp::get_connection_count(), connection_number + 1);
+  auto rc = amqp::publish(conn, "topic", "message");
+  EXPECT_LT(rc, 0);
+}
+
+TEST_F(TestAMQP, InvalidPort)
+{
+  const auto connection_number = amqp::get_connection_count();
+  conn = amqp::connect("amqp://localhost:1234", "ex1");
+  EXPECT_TRUE(conn);
+  EXPECT_EQ(amqp::get_connection_count(), connection_number + 1);
+  auto rc = amqp::publish(conn, "topic", "message");
+  EXPECT_LT(rc, 0);
+}
+
+TEST_F(TestAMQP, InvalidHost)
+{
+  const auto connection_number = amqp::get_connection_count();
+  conn = amqp::connect("amqp://0.0.0.1", "ex1");
+  EXPECT_TRUE(conn);
+  EXPECT_EQ(amqp::get_connection_count(), connection_number + 1);
+  auto rc = amqp::publish(conn, "topic", "message");
+  EXPECT_LT(rc, 0);
+}
+
+TEST_F(TestAMQP, InvalidVhost)
+{
+  const auto connection_number = amqp::get_connection_count();
+  conn = amqp::connect("amqp://localhost/kaboom", "ex1");
+  EXPECT_TRUE(conn);
+  EXPECT_EQ(amqp::get_connection_count(), connection_number + 1);
+  auto rc = amqp::publish(conn, "topic", "message");
+  EXPECT_LT(rc, 0);
+}
+
+TEST_F(TestAMQP, UserPassword)
+{
+  amqp_mock::set_valid_host("127.0.0.1");
+  {
+    const auto connection_number = amqp::get_connection_count();
+    conn = amqp::connect("amqp://foo:bar@127.0.0.1", "ex1");
+    EXPECT_TRUE(conn);
+    EXPECT_EQ(amqp::get_connection_count(), connection_number + 1);
+    auto rc = amqp::publish(conn, "topic", "message");
+    EXPECT_LT(rc, 0);
+  }
+  // now try the same connection with default user/password
+  amqp_mock::set_valid_host("127.0.0.2");
+  {
+    const auto connection_number = amqp::get_connection_count();
+    conn = amqp::connect("amqp://guest:guest@127.0.0.2", "ex1");
+    EXPECT_TRUE(conn);
+    EXPECT_EQ(amqp::get_connection_count(), connection_number + 1);
+    auto rc = amqp::publish(conn, "topic", "message");
+    EXPECT_EQ(rc, 0);
+  }
+  amqp_mock::set_valid_host("localhost");
+}
+
+TEST_F(TestAMQP, URLParseError)
+{
+  const auto connection_number = amqp::get_connection_count();
+  conn = amqp::connect("http://localhost", "ex1");
+  EXPECT_FALSE(conn);
+  EXPECT_EQ(amqp::get_connection_count(), connection_number);
+  auto rc = amqp::publish(conn, "topic", "message");
+  EXPECT_LT(rc, 0);
+}
+
+TEST_F(TestAMQP, ExchangeMismatch)
+{
+  const auto connection_number = amqp::get_connection_count();
+  conn = amqp::connect("http://localhost", "ex2");
+  EXPECT_FALSE(conn);
+  EXPECT_EQ(amqp::get_connection_count(), connection_number);
+  auto rc = amqp::publish(conn, "topic", "message");
+  EXPECT_LT(rc, 0);
+}
+
+TEST_F(TestAMQP, MaxConnections)
+{
+  // fill up all connections
+  std::vector<amqp::connection_ptr_t> connections;
+  auto remaining_connections = amqp::get_max_connections() - amqp::get_connection_count();
+  while (remaining_connections > 0) {
+    const auto host = "127.10.0." + std::to_string(remaining_connections);
+    amqp_mock::set_valid_host(host);
+    amqp::connection_ptr_t conn = amqp::connect("amqp://" + host, "ex1");
+    EXPECT_TRUE(conn);
+    auto rc = amqp::publish(conn, "topic", "message");
+    EXPECT_EQ(rc, 0);
+    --remaining_connections;
+    connections.push_back(conn);
+  }
+  EXPECT_EQ(amqp::get_connection_count(), amqp::get_max_connections());
+  // try to add another connection
+  {
+    const std::string host = "toomany";
+    amqp_mock::set_valid_host(host);
+    amqp::connection_ptr_t conn = amqp::connect("amqp://" + host, "ex1");
+    EXPECT_FALSE(conn);
+    auto rc = amqp::publish(conn, "topic", "message");
+    EXPECT_LT(rc, 0);
+  }
+  EXPECT_EQ(amqp::get_connection_count(), amqp::get_max_connections());
+  amqp_mock::set_valid_host("localhost");
+  // delete connections to make space for new ones
+  for (auto conn : connections) {
+    EXPECT_TRUE(amqp::disconnect(conn));
+  }
+  // wait for them to be deleted
+  std::this_thread::sleep_for(long_wait_time);
+  EXPECT_LT(amqp::get_connection_count(), amqp::get_max_connections());
+}
+
+std::atomic<bool> callback_invoked = false;
+
+std::atomic<int> callbacks_invoked = 0;
+
+// note: because these callback are shared among different "publish" calls
+// they should be used on different connections
+
+void my_callback_expect_ack(int rc) {
+  EXPECT_EQ(0, rc);
+  callback_invoked = true;
+}
+
+void my_callback_expect_nack(int rc) {
+  EXPECT_LT(rc, 0);
+  callback_invoked = true;
+}
+
+void my_callback_expect_multiple_acks(int rc) {
+  EXPECT_EQ(0, rc);
+  ++callbacks_invoked;
+}
+
+class dynamic_callback_wrapper {
+    dynamic_callback_wrapper() = default;
+public:
+    static dynamic_callback_wrapper* create() {
+        return new dynamic_callback_wrapper;
+    }
+    void callback(int rc) {
+      EXPECT_EQ(0, rc);
+      ++callbacks_invoked;
+      delete this;
+    }
+};
+
+void my_callback_expect_close_or_ack(int rc) {
+  // deleting the connection should trigger the callback with -4098
+  // but due to race conditions, some my get an ack
+  EXPECT_TRUE(-4098 == rc || 0 == rc);
+}
+
+TEST_F(TestAMQP, ReceiveAck)
+{
+  callback_invoked = false;
+  const std::string host("localhost1");
+  amqp_mock::set_valid_host(host);
+  conn = amqp::connect("amqp://" + host, "ex1");
+  EXPECT_TRUE(conn);
+  auto rc = publish_with_confirm(conn, "topic", "message", my_callback_expect_ack);
+  EXPECT_EQ(rc, 0);
+  wait_until_drained();
+  EXPECT_TRUE(callback_invoked);
+  amqp_mock::set_valid_host("localhost");
+}
+
+TEST_F(TestAMQP, ImplicitConnectionClose)
+{
+  callback_invoked = false;
+  const std::string host("localhost1");
+  amqp_mock::set_valid_host(host);
+  conn = amqp::connect("amqp://" + host, "ex1");
+  EXPECT_TRUE(conn);
+  const auto NUMBER_OF_CALLS = 2000;
+  for (auto i = 0; i < NUMBER_OF_CALLS; ++i) {
+    auto rc = publish_with_confirm(conn, "topic", "message", my_callback_expect_close_or_ack);
+    EXPECT_EQ(rc, 0);
+  }
+  wait_until_drained();
+  // deleting the connection object should close the connection
+  conn.reset(nullptr);
+  amqp_mock::set_valid_host("localhost");
+}
+
+TEST_F(TestAMQP, ReceiveMultipleAck)
+{
+  callbacks_invoked = 0;
+  const std::string host("localhost1");
+  amqp_mock::set_valid_host(host);
+  conn = amqp::connect("amqp://" + host, "ex1");
+  EXPECT_TRUE(conn);
+  const auto NUMBER_OF_CALLS = 100;
+  for (auto i=0; i < NUMBER_OF_CALLS; ++i) {
+    auto rc = publish_with_confirm(conn, "topic", "message", my_callback_expect_multiple_acks);
+    EXPECT_EQ(rc, 0);
+  }
+  wait_until_drained();
+  EXPECT_EQ(callbacks_invoked, NUMBER_OF_CALLS);
+  callbacks_invoked = 0;
+  amqp_mock::set_valid_host("localhost");
+}
+
+TEST_F(TestAMQP, ReceiveAckForMultiple)
+{
+  callbacks_invoked = 0;
+  const std::string host("localhost1");
+  amqp_mock::set_valid_host(host);
+  conn = amqp::connect("amqp://" + host, "ex1");
+  EXPECT_TRUE(conn);
+  amqp_mock::set_multiple(59);
+  const auto NUMBER_OF_CALLS = 100;
+  for (auto i=0; i < NUMBER_OF_CALLS; ++i) {
+    auto rc = publish_with_confirm(conn, "topic", "message", my_callback_expect_multiple_acks);
+    EXPECT_EQ(rc, 0);
+  }
+  wait_until_drained();
+  EXPECT_EQ(callbacks_invoked, NUMBER_OF_CALLS);
+  callbacks_invoked = 0;
+  amqp_mock::set_valid_host("localhost");
+}
+
+TEST_F(TestAMQP, DynamicCallback)
+{
+  callbacks_invoked = 0;
+  const std::string host("localhost1");
+  amqp_mock::set_valid_host(host);
+  conn = amqp::connect("amqp://" + host, "ex1");
+  EXPECT_TRUE(conn);
+  amqp_mock::set_multiple(59);
+  const auto NUMBER_OF_CALLS = 100;
+  for (auto i=0; i < NUMBER_OF_CALLS; ++i) {
+    auto rc = publish_with_confirm(conn, "topic", "message",
+            std::bind(&dynamic_callback_wrapper::callback, dynamic_callback_wrapper::create(), std::placeholders::_1));
+    EXPECT_EQ(rc, 0);
+  }
+  wait_until_drained();
+  EXPECT_EQ(callbacks_invoked, NUMBER_OF_CALLS);
+  callbacks_invoked = 0;
+  amqp_mock::set_valid_host("localhost");
+}
+
+TEST_F(TestAMQP, ReceiveNack)
+{
+  callback_invoked = false;
+  amqp_mock::REPLY_ACK = false;
+  const std::string host("localhost2");
+  amqp_mock::set_valid_host(host);
+  conn = amqp::connect("amqp://" + host, "ex1");
+  EXPECT_TRUE(conn);
+  auto rc = publish_with_confirm(conn, "topic", "message", my_callback_expect_nack);
+  EXPECT_EQ(rc, 0);
+  wait_until_drained();
+  EXPECT_TRUE(callback_invoked);
+  amqp_mock::REPLY_ACK = true;
+  callback_invoked = false;
+  amqp_mock::set_valid_host("localhost");
+}
+
+TEST_F(TestAMQP, FailWrite)
+{
+  callback_invoked = false;
+  amqp_mock::FAIL_NEXT_WRITE = true;
+  const std::string host("localhost2");
+  amqp_mock::set_valid_host(host);
+  conn = amqp::connect("amqp://" + host, "ex1");
+  EXPECT_TRUE(conn);
+  auto rc = publish_with_confirm(conn, "topic", "message", my_callback_expect_nack);
+  EXPECT_EQ(rc, 0);
+  wait_until_drained();
+  EXPECT_TRUE(callback_invoked);
+  amqp_mock::FAIL_NEXT_WRITE = false;
+  callback_invoked = false;
+  amqp_mock::set_valid_host("localhost");
+}
+
+TEST_F(TestAMQP, ClosedConnection)
+{
+  callback_invoked = false;
+  const auto current_connections = amqp::get_connection_count();
+  const std::string host("localhost3");
+  amqp_mock::set_valid_host(host);
+  conn = amqp::connect("amqp://" + host, "ex1");
+  EXPECT_TRUE(conn);
+  EXPECT_EQ(amqp::get_connection_count(), current_connections + 1);
+  EXPECT_TRUE(amqp::disconnect(conn));
+  std::this_thread::sleep_for(long_wait_time);
+  // make sure number of connections decreased back
+  EXPECT_EQ(amqp::get_connection_count(), current_connections);
+  auto rc = publish_with_confirm(conn, "topic", "message", my_callback_expect_nack);
+  EXPECT_LT(rc, 0);
+  std::this_thread::sleep_for(long_wait_time);
+  EXPECT_FALSE(callback_invoked);
+  callback_invoked = false;
+  amqp_mock::set_valid_host("localhost");
+}
+
+TEST_F(TestAMQP, RetryInvalidHost)
+{
+  const std::string host = "192.168.0.1";
+  const auto connection_number = amqp::get_connection_count();
+  conn = amqp::connect("amqp://"+host, "ex1");
+  EXPECT_TRUE(conn);
+  EXPECT_EQ(amqp::get_connection_count(), connection_number + 1);
+  auto rc = amqp::publish(conn, "topic", "message");
+  EXPECT_LT(rc, 0);
+  // now next retry should be ok
+  amqp_mock::set_valid_host(host);
+  std::this_thread::sleep_for(long_wait_time);
+  rc = amqp::publish(conn, "topic", "message");
+  EXPECT_EQ(rc, 0);
+  amqp_mock::set_valid_host("localhost");
+}
+
+TEST_F(TestAMQP, RetryInvalidPort)
+{
+  const int port = 9999;
+  const auto connection_number = amqp::get_connection_count();
+  conn = amqp::connect("amqp://localhost:" + std::to_string(port), "ex1");
+  EXPECT_TRUE(conn);
+  EXPECT_EQ(amqp::get_connection_count(), connection_number + 1);
+  auto rc = amqp::publish(conn, "topic", "message");
+  EXPECT_LT(rc, 0);
+  // now next retry should be ok
+  amqp_mock::set_valid_port(port);
+  std::this_thread::sleep_for(long_wait_time);
+  rc = amqp::publish(conn, "topic", "message");
+  EXPECT_EQ(rc, 0);
+  amqp_mock::set_valid_port(5672);
+}
+
+TEST_F(TestAMQP, RetryFailWrite)
+{
+  callback_invoked = false;
+  amqp_mock::FAIL_NEXT_WRITE = true;
+  const std::string host("localhost4");
+  amqp_mock::set_valid_host(host);
+  conn = amqp::connect("amqp://" + host, "ex1");
+  EXPECT_TRUE(conn);
+  auto rc = publish_with_confirm(conn, "topic", "message", my_callback_expect_nack);
+  EXPECT_EQ(rc, 0);
+  // set port to a different one, so that reconnect would fail
+  amqp_mock::set_valid_port(9999);
+  wait_until_drained();
+  EXPECT_TRUE(callback_invoked);
+  callback_invoked = false;
+  rc = publish_with_confirm(conn, "topic", "message", my_callback_expect_nack);
+  EXPECT_LT(rc, 0);
+  // expect immediate failure, no callback called after sleep
+  std::this_thread::sleep_for(long_wait_time);
+  EXPECT_FALSE(callback_invoked);
+  // set port to the right one so that reconnect would succeed
+  amqp_mock::set_valid_port(5672);
+  callback_invoked = false;
+  amqp_mock::FAIL_NEXT_WRITE = false;
+  // give time to reconnect
+  std::this_thread::sleep_for(long_wait_time);
+  // retry to publish should succeed now
+  rc = publish_with_confirm(conn, "topic", "message", my_callback_expect_ack);
+  EXPECT_EQ(rc, 0);
+  wait_until_drained();
+  EXPECT_TRUE(callback_invoked);
+  callback_invoked = false;
+  amqp_mock::set_valid_host("localhost");
+}
+
diff --git a/src/test/rgw/test_rgw_arn.cc b/src/test/rgw/test_rgw_arn.cc
new file mode 100644
index 00000000..334c5ecf
--- /dev/null
+++ b/src/test/rgw/test_rgw_arn.cc
@@ -0,0 +1,107 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "rgw/rgw_arn.h"
+#include <gtest/gtest.h>
+
+using namespace rgw;
+
+const int BASIC_ENTRIES = 6;
+
+const std::string basic_str[BASIC_ENTRIES] = {"arn:aws:s3:us-east-1:12345:resource",
+                                  "arn:aws:s3:us-east-1:12345:resourceType/resource",
+                                  "arn:aws:s3:us-east-1:12345:resourceType/resource/qualifier",
+                                  "arn:aws:s3:us-east-1:12345:resourceType/resource:qualifier",
+                                  "arn:aws:s3:us-east-1:12345:resourceType:resource",
+                                  "arn:aws:s3:us-east-1:12345:resourceType:resource/qualifier"};
+
+const std::string expected_basic_resource[BASIC_ENTRIES] = {"resource", 
+                                                "resourceType/resource", 
+                                                "resourceType/resource/qualifier",
+                                                "resourceType/resource:qualifier",
+                                                "resourceType:resource",
+                                                "resourceType:resource/qualifier"};
+TEST(TestARN, Basic)
+{
+  for (auto i = 0; i < BASIC_ENTRIES; ++i) {
+    boost::optional<ARN> arn = ARN::parse(basic_str[i]);
+    ASSERT_TRUE(arn);
+    EXPECT_EQ(arn->partition, Partition::aws);
+    EXPECT_EQ(arn->service, Service::s3);
+    EXPECT_STREQ(arn->region.c_str(), "us-east-1");
+    EXPECT_STREQ(arn->account.c_str(), "12345");
+    EXPECT_STREQ(arn->resource.c_str(), expected_basic_resource[i].c_str());
+  }
+}
+
+TEST(TestARN, ToString)
+{
+  for (auto i = 0; i < BASIC_ENTRIES; ++i) {
+    boost::optional<ARN> arn = ARN::parse(basic_str[i]);
+    ASSERT_TRUE(arn);
+    EXPECT_STREQ(to_string(*arn).c_str(), basic_str[i].c_str());
+  }
+}
+
+const std::string expected_basic_resource_type[BASIC_ENTRIES] = 
+    {"", "resourceType", "resourceType", "resourceType", "resourceType", "resourceType"};
+const std::string expected_basic_qualifier[BASIC_ENTRIES] = 
+    {"", "", "qualifier", "qualifier", "", "qualifier"};
+
+TEST(TestARNResource, Basic)
+{
+  for (auto i = 0; i < BASIC_ENTRIES; ++i) {
+    boost::optional<ARN> arn = ARN::parse(basic_str[i]);
+    ASSERT_TRUE(arn);
+    ASSERT_FALSE(arn->resource.empty());
+    boost::optional<ARNResource> resource = ARNResource::parse(arn->resource);
+    ASSERT_TRUE(resource);
+    EXPECT_STREQ(resource->resource.c_str(), "resource");
+    EXPECT_STREQ(resource->resource_type.c_str(), expected_basic_resource_type[i].c_str());
+    EXPECT_STREQ(resource->qualifier.c_str(), expected_basic_qualifier[i].c_str());
+  }
+}
+
+const int EMPTY_ENTRIES = 4;
+
+const std::string empty_str[EMPTY_ENTRIES] = {"arn:aws:s3:::resource",
+                                  "arn:aws:s3::12345:resource",
+                                  "arn:aws:s3:us-east-1::resource",
+                                  "arn:aws:s3:us-east-1:12345:"};
+
+TEST(TestARN, Empty)
+{
+  for (auto i = 0; i < EMPTY_ENTRIES; ++i) {
+    boost::optional<ARN> arn = ARN::parse(empty_str[i]);
+    ASSERT_TRUE(arn);
+    EXPECT_EQ(arn->partition, Partition::aws);
+    EXPECT_EQ(arn->service, Service::s3);
+    EXPECT_TRUE(arn->region.empty() || arn->region == "us-east-1");
+    EXPECT_TRUE(arn->account.empty() || arn->account == "12345");
+    EXPECT_TRUE(arn->resource.empty() || arn->resource == "resource");
+  }
+}
+
+const int WILDCARD_ENTRIES = 3;
+
+const std::string wildcard_str[WILDCARD_ENTRIES] = {"arn:aws:s3:*:*:resource",
+                                  "arn:aws:s3:*:12345:resource",
+                                  "arn:aws:s3:us-east-1:*:resource"};
+
+// FIXME: currently the following: "arn:aws:s3:us-east-1:12345:*"
+// does not fail, even if "wildcard" is not set to "true" 
+
+TEST(TestARN, Wildcard)
+{
+  for (auto i = 0; i < WILDCARD_ENTRIES; ++i) {
+    EXPECT_FALSE(ARN::parse(wildcard_str[i]));
+    boost::optional<ARN> arn = ARN::parse(wildcard_str[i], true);
+    ASSERT_TRUE(arn);
+    EXPECT_EQ(arn->partition, Partition::aws);
+    EXPECT_EQ(arn->service, Service::s3);
+    EXPECT_TRUE(arn->region == "*" || arn->region == "us-east-1");
+    EXPECT_TRUE(arn->account == "*" || arn->account == "12345");
+    EXPECT_TRUE(arn->resource == "*" || arn->resource == "resource");
+  }
+}
+
diff --git a/src/test/rgw/test_rgw_bencode.cc b/src/test/rgw/test_rgw_bencode.cc
new file mode 100644
index 00000000..66fa92a8
--- /dev/null
+++ b/src/test/rgw/test_rgw_bencode.cc
@@ -0,0 +1,63 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+#include "gtest/gtest.h"
+
+#include "rgw/rgw_torrent.h"
+
+TEST(Bencode, String)
+{
+  TorrentBencode decode;
+  bufferlist bl;
+
+  decode.bencode("foo", bl);
+  decode.bencode("bar", bl);
+  decode.bencode("baz", bl);
+
+  string s(bl.c_str(), bl.length());
+
+  ASSERT_STREQ("3:foo3:bar3:baz", s.c_str());
+}
+
+TEST(Bencode, Integers)
+{
+  TorrentBencode decode;
+  bufferlist bl;
+
+  decode.bencode(0, bl);
+  decode.bencode(-3, bl);
+  decode.bencode(7, bl);
+
+  string s(bl.c_str(), bl.length());
+
+  ASSERT_STREQ("i0ei-3ei7e", s.c_str());
+}
+
+TEST(Bencode, Dict)
+{
+  TorrentBencode decode;  
+  bufferlist bl;
+
+  decode.bencode_dict(bl);
+  decode.bencode("foo", 5, bl);
+  decode.bencode("bar", "baz", bl);
+  decode.bencode_end(bl);
+
+  string s(bl.c_str(), bl.length());
+
+  ASSERT_STREQ("d3:fooi5e3:bar3:baze", s.c_str());
+}
+
+TEST(Bencode, List)
+{
+  TorrentBencode decode;
+  bufferlist bl;
+
+  decode.bencode_list(bl);
+  decode.bencode("foo", 5, bl);
+  decode.bencode("bar", "baz", bl);
+  decode.bencode_end(bl);
+
+  string s(bl.c_str(), bl.length());
+
+  ASSERT_STREQ("l3:fooi5e3:bar3:baze", s.c_str());
+}
diff --git a/src/test/rgw/test_rgw_common.cc b/src/test/rgw/test_rgw_common.cc
new file mode 100644
index 00000000..731d624e
--- /dev/null
+++ b/src/test/rgw/test_rgw_common.cc
@@ -0,0 +1,91 @@
+#include "test_rgw_common.h"
+
+void test_rgw_add_placement(RGWZoneGroup *zonegroup, RGWZoneParams *zone_params, const std::string& name, bool is_default)
+{
+  zonegroup->placement_targets[name] = { name };
+
+  RGWZonePlacementInfo& pinfo = zone_params->placement_pools[name];
+  pinfo.index_pool = rgw_pool(name + ".index").to_str();
+
+  rgw_pool data_pool(name + ".data");
+  pinfo.storage_classes.set_storage_class(RGW_STORAGE_CLASS_STANDARD, &data_pool, nullptr);
+  pinfo.data_extra_pool = rgw_pool(name + ".extra").to_str();
+
+  if (is_default) {
+    zonegroup->default_placement = rgw_placement_rule(name, RGW_STORAGE_CLASS_STANDARD);
+  }
+}
+
+void test_rgw_init_env(RGWZoneGroup *zonegroup, RGWZoneParams *zone_params)
+{
+  test_rgw_add_placement(zonegroup, zone_params, "default-placement", true);
+
+}
+
+void test_rgw_populate_explicit_placement_bucket(rgw_bucket *b, const char *t, const char *n, const char *dp, const char *ip, const char *m, const char *id)
+{
+  b->tenant = t;
+  b->name = n;
+  b->marker = m;
+  b->bucket_id = id;
+  b->explicit_placement.data_pool = rgw_pool(dp);
+  b->explicit_placement.index_pool = rgw_pool(ip);
+}
+
+void test_rgw_populate_old_bucket(old_rgw_bucket *b, const char *t, const char *n, const char *dp, const char *ip, const char *m, const char *id)
+{
+  b->tenant = t;
+  b->name = n;
+  b->marker = m;
+  b->bucket_id = id;
+  b->data_pool = dp;
+  b->index_pool = ip;
+}
+
+std::string test_rgw_get_obj_oid(const rgw_obj& obj)
+{
+  std::string oid;
+  std::string loc;
+
+  get_obj_bucket_and_oid_loc(obj, oid, loc);
+  return oid;
+}
+
+void test_rgw_init_explicit_placement_bucket(rgw_bucket *bucket, const char *name)
+{
+  test_rgw_populate_explicit_placement_bucket(bucket, "", name, ".data-pool", ".index-pool", "marker", "bucket-id");
+}
+
+void test_rgw_init_old_bucket(old_rgw_bucket *bucket, const char *name)
+{
+  test_rgw_populate_old_bucket(bucket, "", name, ".data-pool", ".index-pool", "marker", "bucket-id");
+}
+
+void test_rgw_populate_bucket(rgw_bucket *b, const char *t, const char *n, const char *m, const char *id)
+{
+  b->tenant = t;
+  b->name = n;
+  b->marker = m;
+  b->bucket_id = id;
+}
+
+void test_rgw_init_bucket(rgw_bucket *bucket, const char *name)
+{
+  test_rgw_populate_bucket(bucket, "", name, "marker", "bucket-id");
+}
+
+rgw_obj test_rgw_create_obj(const rgw_bucket& bucket, const std::string& name, const std::string& instance, const std::string& ns)
+{
+  rgw_obj obj(bucket, name);
+  if (!instance.empty()) {
+    obj.key.set_instance(instance);
+  }
+  if (!ns.empty()) {
+    obj.key.ns = ns;
+  }
+  obj.bucket = bucket;
+
+  return obj;
+}
+
+
diff --git a/src/test/rgw/test_rgw_common.h b/src/test/rgw/test_rgw_common.h
new file mode 100644
index 00000000..c360e3b0
--- /dev/null
+++ b/src/test/rgw/test_rgw_common.h
@@ -0,0 +1,506 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 eNovance SAS <licensing@enovance.com>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+#include <iostream>
+#include "common/ceph_json.h"
+#include "common/Formatter.h"
+#include "rgw/rgw_common.h"
+#include "rgw/rgw_rados.h"
+#include "rgw/rgw_zone.h"
+
+#ifndef CEPH_TEST_RGW_COMMON_H
+#define CEPH_TEST_RGW_COMMON_H
+
+struct old_rgw_bucket {
+  std::string tenant;
+  std::string name;
+  std::string data_pool;
+  std::string data_extra_pool; /* if not set, then we should use data_pool instead */
+  std::string index_pool;
+  std::string marker;
+  std::string bucket_id;
+
+  std::string oid; /*
+                    * runtime in-memory only info. If not empty, points to the bucket instance object
+                    */
+
+  old_rgw_bucket() { }
+  // cppcheck-suppress noExplicitConstructor
+  old_rgw_bucket(const std::string& s) : name(s) {
+    data_pool = index_pool = s;
+    marker = "";
+  }
+  explicit old_rgw_bucket(const char *n) : name(n) {
+    data_pool = index_pool = n;
+    marker = "";
+  }
+  old_rgw_bucket(const char *t, const char *n, const char *dp, const char *ip, const char *m, const char *id, const char *h) :
+    tenant(t), name(n), data_pool(dp), index_pool(ip), marker(m), bucket_id(id) {}
+
+  void encode(bufferlist& bl) const {
+     ENCODE_START(8, 3, bl);
+    encode(name, bl);
+    encode(data_pool, bl);
+    encode(marker, bl);
+    encode(bucket_id, bl);
+    encode(index_pool, bl);
+    encode(data_extra_pool, bl);
+    encode(tenant, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::const_iterator& bl) {
+    DECODE_START_LEGACY_COMPAT_LEN(8, 3, 3, bl);
+    decode(name, bl);
+    decode(data_pool, bl);
+    if (struct_v >= 2) {
+      decode(marker, bl);
+      if (struct_v <= 3) {
+        uint64_t id;
+        decode(id, bl);
+        char buf[16];
+        snprintf(buf, sizeof(buf), "%llu", (long long)id);
+        bucket_id = buf;
+      } else {
+        decode(bucket_id, bl);
+      }
+    }
+    if (struct_v >= 5) {
+      decode(index_pool, bl);
+    } else {
+      index_pool = data_pool;
+    }
+    if (struct_v >= 7) {
+      decode(data_extra_pool, bl);
+    }
+    if (struct_v >= 8) {
+      decode(tenant, bl);
+    }
+    DECODE_FINISH(bl);
+  }
+
+  // format a key for the bucket/instance. pass delim=0 to skip a field
+  std::string get_key(char tenant_delim = '/',
+                      char id_delim = ':') const;
+
+  const std::string& get_data_extra_pool() {
+    if (data_extra_pool.empty()) {
+      return data_pool;
+    }
+    return data_extra_pool;
+  }
+
+  void dump(Formatter *f) const;
+  void decode_json(JSONObj *obj);
+  static void generate_test_instances(list<old_rgw_bucket*>& o);
+
+  bool operator<(const old_rgw_bucket& b) const {
+    return name.compare(b.name) < 0;
+  }
+};
+WRITE_CLASS_ENCODER(old_rgw_bucket)
+
+class old_rgw_obj {
+  std::string orig_obj;
+  std::string loc;
+  std::string object;
+  std::string instance;
+public:
+  const std::string& get_object() const { return object; }
+  const std::string& get_orig_obj() const { return orig_obj; }
+  const std::string& get_loc() const { return loc; }
+  const std::string& get_instance() const { return instance; }
+  old_rgw_bucket bucket;
+  std::string ns;
+
+  bool in_extra_data; /* in-memory only member, does not serialize */
+
+  // Represents the hash index source for this object once it is set (non-empty)
+  std::string index_hash_source;
+
+  old_rgw_obj() : in_extra_data(false) {}
+  old_rgw_obj(old_rgw_bucket& b, const std::string& o) : in_extra_data(false) {
+    init(b, o);
+  }
+  old_rgw_obj(old_rgw_bucket& b, const rgw_obj_key& k) : in_extra_data(false) {
+    from_index_key(b, k);
+  }
+  void init(old_rgw_bucket& b, const std::string& o) {
+    bucket = b;
+    set_obj(o);
+    reset_loc();
+  }
+  void init_ns(old_rgw_bucket& b, const std::string& o, const std::string& n) {
+    bucket = b;
+    set_ns(n);
+    set_obj(o);
+    reset_loc();
+  }
+  int set_ns(const char *n) {
+    if (!n)
+      return -EINVAL;
+    std::string ns_str(n);
+    return set_ns(ns_str);
+  }
+  int set_ns(const std::string& n) {
+    if (n[0] == '_')
+      return -EINVAL;
+    ns = n;
+    set_obj(orig_obj);
+    return 0;
+  }
+  int set_instance(const std::string& i) {
+    if (i[0] == '_')
+      return -EINVAL;
+    instance = i;
+    set_obj(orig_obj);
+    return 0;
+  }
+
+  int clear_instance() {
+    return set_instance(string());
+  }
+
+  void set_loc(const std::string& k) {
+    loc = k;
+  }
+
+  void reset_loc() {
+    loc.clear();
+    /*
+     * For backward compatibility. Older versions used to have object locator on all objects,
+     * however, the orig_obj was the effective object locator. This had the same effect as not
+     * having object locator at all for most objects but the ones that started with underscore as
+     * these were escaped.
+     */
+    if (orig_obj[0] == '_' && ns.empty()) {
+      loc = orig_obj;
+    }
+  }
+
+  bool have_null_instance() {
+    return instance == "null";
+  }
+
+  bool have_instance() {
+    return !instance.empty();
+  }
+
+  bool need_to_encode_instance() {
+    return have_instance() && !have_null_instance();
+  }
+
+  void set_obj(const std::string& o) {
+    object.reserve(128);
+
+    orig_obj = o;
+    if (ns.empty() && !need_to_encode_instance()) {
+      if (o.empty()) {
+        return;
+      }
+      if (o.size() < 1 || o[0] != '_') {
+        object = o;
+        return;
+      }
+      object = "_";
+      object.append(o);
+    } else {
+      object = "_";
+      object.append(ns);
+      if (need_to_encode_instance()) {
+        object.append(string(":") + instance);
+      }
+      object.append("_");
+      object.append(o);
+    }
+    reset_loc();
+  }
+
+  /*
+   * get the object's key name as being referred to by the bucket index.
+   */
+  std::string get_index_key_name() const {
+    if (ns.empty()) {
+      if (orig_obj.size() < 1 || orig_obj[0] != '_') {
+        return orig_obj;
+      }
+      return std::string("_") + orig_obj;
+    };
+
+    char buf[ns.size() + 16];
+    snprintf(buf, sizeof(buf), "_%s_", ns.c_str());
+    return std::string(buf) + orig_obj;
+  };
+
+  void from_index_key(old_rgw_bucket& b, const rgw_obj_key& key) {
+    if (key.name[0] != '_') {
+      init(b, key.name);
+      set_instance(key.instance);
+      return;
+    }
+    if (key.name[1] == '_') {
+      init(b, key.name.substr(1));
+      set_instance(key.instance);
+      return;
+    }
+    ssize_t pos = key.name.find('_', 1);
+    if (pos < 0) {
+      /* shouldn't happen, just use key */
+      init(b, key.name);
+      set_instance(key.instance);
+      return;
+    }
+
+    init_ns(b, key.name.substr(pos + 1), key.name.substr(1, pos -1));
+    set_instance(key.instance);
+  }
+
+  void get_index_key(rgw_obj_key *key) const {
+    key->name = get_index_key_name();
+    key->instance = instance;
+  }
+
+  static void parse_ns_field(string& ns, std::string& instance) {
+    int pos = ns.find(':');
+    if (pos >= 0) {
+      instance = ns.substr(pos + 1);
+      ns = ns.substr(0, pos);
+    } else {
+      instance.clear();
+    }
+  }
+
+  std::string& get_hash_object() {
+    return index_hash_source.empty() ? orig_obj : index_hash_source;
+  }
+  /**
+   * Translate a namespace-mangled object name to the user-facing name
+   * existing in the given namespace.
+   *
+   * If the object is part of the given namespace, it returns true
+   * and cuts down the name to the unmangled version. If it is not
+   * part of the given namespace, it returns false.
+   */
+  static bool translate_raw_obj_to_obj_in_ns(string& obj, std::string& instance, std::string& ns) {
+    if (obj[0] != '_') {
+      if (ns.empty()) {
+        return true;
+      }
+      return false;
+    }
+
+    std::string obj_ns;
+    bool ret = parse_raw_oid(obj, &obj, &instance, &obj_ns);
+    if (!ret) {
+      return ret;
+    }
+
+    return (ns == obj_ns);
+  }
+
+  static bool parse_raw_oid(const std::string& oid, std::string *obj_name, std::string *obj_instance, std::string *obj_ns) {
+    obj_instance->clear();
+    obj_ns->clear();
+    if (oid[0] != '_') {
+      *obj_name = oid;
+      return true;
+    }
+
+    if (oid.size() >= 2 && oid[1] == '_') {
+      *obj_name = oid.substr(1);
+      return true;
+    }
+
+    if (oid[0] != '_' || oid.size() < 3) // for namespace, min size would be 3: _x_
+      return false;
+
+    int pos = oid.find('_', 1);
+    if (pos <= 1) // if it starts with __, it's not in our namespace
+      return false;
+
+    *obj_ns = oid.substr(1, pos - 1);
+    parse_ns_field(*obj_ns, *obj_instance);
+
+    *obj_name = oid.substr(pos + 1);
+    return true;
+  }
+
+  /**
+   * Given a mangled object name and an empty namespace string, this
+   * function extracts the namespace into the string and sets the object
+   * name to be the unmangled version.
+   *
+   * It returns true after successfully doing so, or
+   * false if it fails.
+   */
+  static bool strip_namespace_from_object(string& obj, std::string& ns, std::string& instance) {
+    ns.clear();
+    instance.clear();
+    if (obj[0] != '_') {
+      return true;
+    }
+
+    size_t pos = obj.find('_', 1);
+    if (pos == std::string::npos) {
+      return false;
+    }
+
+    if (obj[1] == '_') {
+      obj = obj.substr(1);
+      return true;
+    }
+
+    size_t period_pos = obj.find('.');
+    if (period_pos < pos) {
+      return false;
+    }
+
+    ns = obj.substr(1, pos-1);
+    obj = obj.substr(pos+1, std::string::npos);
+
+    parse_ns_field(ns, instance);
+    return true;
+  }
+
+  void set_in_extra_data(bool val) {
+    in_extra_data = val;
+  }
+
+  bool is_in_extra_data() const {
+    return in_extra_data;
+  }
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(5, 3, bl);
+    encode(bucket.name, bl);
+    encode(loc, bl);
+    encode(ns, bl);
+    encode(object, bl);
+    encode(bucket, bl);
+    encode(instance, bl);
+    if (!ns.empty() || !instance.empty()) {
+      encode(orig_obj, bl);
+    }
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::const_iterator& bl) {
+    DECODE_START_LEGACY_COMPAT_LEN(5, 3, 3, bl);
+    decode(bucket.name, bl);
+    decode(loc, bl);
+    decode(ns, bl);
+    decode(object, bl);
+    if (struct_v >= 2)
+      decode(bucket, bl);
+    if (struct_v >= 4)
+      decode(instance, bl);
+    if (ns.empty() && instance.empty()) {
+      if (object[0] != '_') {
+        orig_obj = object;
+      } else {
+	orig_obj = object.substr(1);
+      }
+    } else {
+      if (struct_v >= 5) {
+        decode(orig_obj, bl);
+      } else {
+        ssize_t pos = object.find('_', 1);
+        if (pos < 0) {
+          throw buffer::error();
+        }
+        orig_obj = object.substr(pos);
+      }
+    }
+    DECODE_FINISH(bl);
+  }
+
+  bool operator==(const old_rgw_obj& o) const {
+    return (object.compare(o.object) == 0) &&
+           (bucket.name.compare(o.bucket.name) == 0) &&
+           (ns.compare(o.ns) == 0) &&
+           (instance.compare(o.instance) == 0);
+  }
+  bool operator<(const old_rgw_obj& o) const {
+    int r = bucket.name.compare(o.bucket.name);
+    if (r == 0) {
+      r = bucket.bucket_id.compare(o.bucket.bucket_id);
+      if (r == 0) {
+        r = object.compare(o.object);
+        if (r == 0) {
+          r = ns.compare(o.ns);
+          if (r == 0) {
+            r = instance.compare(o.instance);
+          }
+        }
+      }
+    }
+
+    return (r < 0);
+  }
+};
+WRITE_CLASS_ENCODER(old_rgw_obj)
+
+static inline void prepend_old_bucket_marker(const old_rgw_bucket& bucket, const string& orig_oid, string& oid)
+{
+  if (bucket.marker.empty() || orig_oid.empty()) {
+    oid = orig_oid;
+  } else {
+    oid = bucket.marker;
+    oid.append("_");
+    oid.append(orig_oid);
+  }
+}
+
+void test_rgw_init_env(RGWZoneGroup *zonegroup, RGWZoneParams *zone_params);
+
+struct test_rgw_env {
+  RGWZoneGroup zonegroup;
+  RGWZoneParams zone_params;
+  rgw_data_placement_target default_placement;
+
+  test_rgw_env() {
+    test_rgw_init_env(&zonegroup, &zone_params);
+    default_placement.data_pool = rgw_pool(zone_params.placement_pools[zonegroup.default_placement.name].get_standard_data_pool());
+    default_placement.data_extra_pool =  rgw_pool(zone_params.placement_pools[zonegroup.default_placement.name].data_extra_pool);
+  }
+
+  rgw_data_placement_target get_placement(const std::string& placement_id) {
+    const RGWZonePlacementInfo& pi = zone_params.placement_pools[placement_id];
+    rgw_data_placement_target pt;
+    pt.index_pool = pi.index_pool;
+    pt.data_pool = pi.get_standard_data_pool();
+    pt.data_extra_pool = pi.data_extra_pool;
+    return pt;
+  }
+
+  rgw_raw_obj get_raw(const rgw_obj& obj) {
+    rgw_obj_select s(obj);
+    return s.get_raw_obj(zonegroup, zone_params);
+  }
+
+  rgw_raw_obj get_raw(const rgw_obj_select& os) {
+    return os.get_raw_obj(zonegroup, zone_params);
+  }
+};
+
+void test_rgw_add_placement(RGWZoneGroup *zonegroup, RGWZoneParams *zone_params, const std::string& name, bool is_default);
+void test_rgw_populate_explicit_placement_bucket(rgw_bucket *b, const char *t, const char *n, const char *dp, const char *ip, const char *m, const char *id);
+void test_rgw_populate_old_bucket(old_rgw_bucket *b, const char *t, const char *n, const char *dp, const char *ip, const char *m, const char *id);
+
+std::string test_rgw_get_obj_oid(const rgw_obj& obj);
+void test_rgw_init_explicit_placement_bucket(rgw_bucket *bucket, const char *name);
+void test_rgw_init_old_bucket(old_rgw_bucket *bucket, const char *name);
+void test_rgw_populate_bucket(rgw_bucket *b, const char *t, const char *n, const char *m, const char *id);
+void test_rgw_init_bucket(rgw_bucket *bucket, const char *name);
+rgw_obj test_rgw_create_obj(const rgw_bucket& bucket, const std::string& name, const std::string& instance, const std::string& ns);
+
+#endif
+
diff --git a/src/test/rgw/test_rgw_compression.cc b/src/test/rgw/test_rgw_compression.cc
new file mode 100644
index 00000000..936b8f7c
--- /dev/null
+++ b/src/test/rgw/test_rgw_compression.cc
@@ -0,0 +1,184 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+#include "gtest/gtest.h"
+
+#include "rgw/rgw_compression.h"
+
+class ut_get_sink : public RGWGetObj_Filter {
+  bufferlist sink;
+public:
+  ut_get_sink() {}
+  virtual ~ut_get_sink() {}
+
+  int handle_data(bufferlist& bl, off_t bl_ofs, off_t bl_len) override
+  {
+    auto& bl_buffers = bl.buffers();
+    auto i = bl_buffers.begin();
+    while (bl_len > 0)
+    {
+      ceph_assert(i != bl_buffers.end());
+      off_t len = std::min<off_t>(bl_len, i->length());
+      sink.append(*i, 0, len);
+      bl_len -= len;
+      i++;
+    }
+    return 0;
+  }
+  bufferlist& get_sink()
+  {
+    return sink;
+  }
+};
+
+class ut_get_sink_size : public RGWGetObj_Filter {
+  size_t max_size = 0;
+public:
+  ut_get_sink_size() {}
+  virtual ~ut_get_sink_size() {}
+
+  int handle_data(bufferlist& bl, off_t bl_ofs, off_t bl_len) override
+  {
+    if (bl_len > (off_t)max_size)
+      max_size = bl_len;
+    return 0;
+  }
+  size_t get_size()
+  {
+    return max_size;
+  }
+};
+
+class ut_put_sink: public rgw::putobj::DataProcessor
+{
+  bufferlist sink;
+public:
+  int process(bufferlist&& bl, uint64_t ofs) override
+  {
+    sink.claim_append(bl);
+    return 0;
+  }
+  bufferlist&  get_sink()
+  {
+    return sink;
+  }
+};
+
+
+struct MockGetDataCB : public RGWGetObj_Filter {
+  int handle_data(bufferlist& bl, off_t bl_ofs, off_t bl_len) override {
+    return 0;
+  }
+} cb;
+
+using range_t = std::pair<off_t, off_t>;
+
+// call filter->fixup_range() and return the range as a pair. this makes it easy
+// to fit on a single line for ASSERT_EQ()
+range_t fixup_range(RGWGetObj_Decompress *filter, off_t ofs, off_t end)
+{
+  filter->fixup_range(ofs, end);
+  return {ofs, end};
+}
+
+
+TEST(Decompress, FixupRangePartial)
+{
+  RGWCompressionInfo cs_info;
+
+  // array of blocks with original len=8, compressed to len=6
+  auto& blocks = cs_info.blocks;
+  blocks.emplace_back(compression_block{0, 0, 6});
+  blocks.emplace_back(compression_block{8, 6, 6});
+  blocks.emplace_back(compression_block{16, 12, 6});
+  blocks.emplace_back(compression_block{24, 18, 6});
+
+  const bool partial = true;
+  RGWGetObj_Decompress decompress(g_ceph_context, &cs_info, partial, &cb);
+
+  // test translation from logical ranges to compressed ranges
+  ASSERT_EQ(range_t(0, 5), fixup_range(&decompress, 0, 1));
+  ASSERT_EQ(range_t(0, 5), fixup_range(&decompress, 1, 7));
+  ASSERT_EQ(range_t(0, 11), fixup_range(&decompress, 7, 8));
+  ASSERT_EQ(range_t(0, 11), fixup_range(&decompress, 0, 9));
+  ASSERT_EQ(range_t(0, 11), fixup_range(&decompress, 7, 9));
+  ASSERT_EQ(range_t(6, 11), fixup_range(&decompress, 8, 9));
+  ASSERT_EQ(range_t(6, 17), fixup_range(&decompress, 8, 16));
+  ASSERT_EQ(range_t(6, 17), fixup_range(&decompress, 8, 17));
+  ASSERT_EQ(range_t(12, 23), fixup_range(&decompress, 16, 24));
+  ASSERT_EQ(range_t(12, 23), fixup_range(&decompress, 16, 999));
+  ASSERT_EQ(range_t(18, 23), fixup_range(&decompress, 998, 999));
+}
+
+TEST(Compress, LimitedChunkSize)
+{
+  CompressorRef plugin;
+  plugin = Compressor::create(g_ceph_context, Compressor::COMP_ALG_ZLIB);
+  ASSERT_NE(plugin.get(), nullptr);
+
+  for (size_t s = 100 ; s < 10000000 ; s = s*5/4)
+  {
+    bufferptr bp(s);
+    bufferlist bl;
+    bl.append(bp);
+
+    ut_put_sink c_sink;
+    RGWPutObj_Compress compressor(g_ceph_context, plugin, &c_sink);
+    compressor.process(std::move(bl), 0);
+    compressor.process({}, s); // flush
+
+    RGWCompressionInfo cs_info;
+    cs_info.compression_type = plugin->get_type_name();
+    cs_info.orig_size = s;
+    cs_info.blocks = move(compressor.get_compression_blocks());
+
+    ut_get_sink_size d_sink;
+    RGWGetObj_Decompress decompress(g_ceph_context, &cs_info, false, &d_sink);
+
+    off_t f_begin = 0;
+    off_t f_end = s - 1;
+    decompress.fixup_range(f_begin, f_end);
+
+    decompress.handle_data(c_sink.get_sink(), 0, c_sink.get_sink().length());
+    bufferlist empty;
+    decompress.handle_data(empty, 0, 0);
+
+    ASSERT_LE(d_sink.get_size(), (size_t)g_ceph_context->_conf->rgw_max_chunk_size);
+  }
+}
+
+
+TEST(Compress, BillionZeros)
+{
+  CompressorRef plugin;
+  ut_put_sink c_sink;
+  plugin = Compressor::create(g_ceph_context, Compressor::COMP_ALG_ZLIB);
+  ASSERT_NE(plugin.get(), nullptr);
+  RGWPutObj_Compress compressor(g_ceph_context, plugin, &c_sink);
+
+  constexpr size_t size = 1000000;
+  bufferptr bp(size);
+  bufferlist bl;
+  bl.append(bp);
+
+  for (int i=0; i<1000;i++)
+    compressor.process(bufferlist{bl}, size*i);
+  compressor.process({}, size*1000); // flush
+
+  RGWCompressionInfo cs_info;
+  cs_info.compression_type = plugin->get_type_name();
+  cs_info.orig_size = size*1000;
+  cs_info.blocks = move(compressor.get_compression_blocks());
+
+  ut_get_sink d_sink;
+  RGWGetObj_Decompress decompress(g_ceph_context, &cs_info, false, &d_sink);
+
+  off_t f_begin = 0;
+  off_t f_end = size*1000 - 1;
+  decompress.fixup_range(f_begin, f_end);
+
+  decompress.handle_data(c_sink.get_sink(), 0, c_sink.get_sink().length());
+  bufferlist empty;
+  decompress.handle_data(empty, 0, 0);
+
+  ASSERT_EQ(d_sink.get_sink().length() , size*1000);
+}
diff --git a/src/test/rgw/test_rgw_crypto.cc b/src/test/rgw/test_rgw_crypto.cc
new file mode 100644
index 00000000..1ab95592
--- /dev/null
+++ b/src/test/rgw/test_rgw_crypto.cc
@@ -0,0 +1,811 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2016 Mirantis <akupczyk@mirantis.com>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+#include <iostream>
+#include "global/global_init.h"
+#include "common/ceph_argparse.h"
+#include "rgw/rgw_common.h"
+#include "rgw/rgw_rados.h"
+#include "rgw/rgw_crypt.h"
+#include <gtest/gtest.h>
+#include "include/ceph_assert.h"
+#define dout_subsys ceph_subsys_rgw
+
+using namespace std;
+
+
+std::unique_ptr<BlockCrypt> AES_256_CBC_create(CephContext* cct, const uint8_t* key, size_t len);
+
+
+class ut_get_sink : public RGWGetObj_Filter {
+  std::stringstream sink;
+public:
+  ut_get_sink() {}
+  virtual ~ut_get_sink() {}
+
+  int handle_data(bufferlist& bl, off_t bl_ofs, off_t bl_len) override
+  {
+    sink << boost::string_ref(bl.c_str()+bl_ofs, bl_len);
+    return 0;
+  }
+  std::string get_sink()
+  {
+    return sink.str();
+  }
+};
+
+class ut_put_sink: public rgw::putobj::DataProcessor
+{
+  std::stringstream sink;
+public:
+  int process(bufferlist&& bl, uint64_t ofs) override
+  {
+    sink << boost::string_ref(bl.c_str(),bl.length());
+    return 0;
+  }
+  std::string get_sink()
+  {
+    return sink.str();
+  }
+};
+
+
+class BlockCryptNone: public BlockCrypt {
+  size_t block_size = 256;
+public:
+  BlockCryptNone(){};
+  BlockCryptNone(size_t sz) : block_size(sz) {}
+  virtual ~BlockCryptNone(){};
+  size_t get_block_size() override
+  {
+    return block_size;
+  }
+  bool encrypt(bufferlist& input,
+                       off_t in_ofs,
+                       size_t size,
+                       bufferlist& output,
+                       off_t stream_offset) override
+  {
+    output.clear();
+    output.append(input.c_str(), input.length());
+    return true;
+  }
+  bool decrypt(bufferlist& input,
+                       off_t in_ofs,
+                       size_t size,
+                       bufferlist& output,
+                       off_t stream_offset) override
+  {
+    output.clear();
+    output.append(input.c_str(), input.length());
+    return true;
+  }
+};
+
+
+TEST(TestRGWCrypto, verify_AES_256_CBC_identity)
+{
+  //create some input for encryption
+  const off_t test_range = 1024*1024;
+  buffer::ptr buf(test_range);
+  char* p = buf.c_str();
+  for(size_t i = 0; i < buf.length(); i++)
+    p[i] = i + i*i + (i >> 2);
+
+  bufferlist input;
+  input.append(buf);
+
+  for (unsigned int step : {1, 2, 3, 5, 7, 11, 13, 17})
+  {
+    //make some random key
+    uint8_t key[32];
+    for(size_t i=0;i<sizeof(key);i++)
+      key[i]=i*step;
+
+    auto aes(AES_256_CBC_create(g_ceph_context, &key[0], 32));
+    ASSERT_NE(aes.get(), nullptr);
+
+    size_t block_size = aes->get_block_size();
+    ASSERT_NE(block_size, 0u);
+
+    for (size_t r = 97; r < 123 ; r++)
+    {
+      off_t begin = (r*r*r*r*r % test_range);
+      begin = begin - begin % block_size;
+      off_t end = begin + r*r*r*r*r*r*r % (test_range - begin);
+      if (r % 3)
+        end = end - end % block_size;
+      off_t offset = r*r*r*r*r*r*r*r % (1000*1000*1000);
+      offset = offset - offset % block_size;
+
+      ASSERT_EQ(begin % block_size, 0u);
+      ASSERT_LE(end, test_range);
+      ASSERT_EQ(offset % block_size, 0u);
+
+      bufferlist encrypted;
+      ASSERT_TRUE(aes->encrypt(input, begin, end - begin, encrypted, offset));
+      bufferlist decrypted;
+      ASSERT_TRUE(aes->decrypt(encrypted, 0, end - begin, decrypted, offset));
+
+      ASSERT_EQ(decrypted.length(), end - begin);
+      ASSERT_EQ(boost::string_ref(input.c_str() + begin, end - begin),
+                boost::string_ref(decrypted.c_str(), end - begin) );
+    }
+  }
+}
+
+
+TEST(TestRGWCrypto, verify_AES_256_CBC_identity_2)
+{
+  //create some input for encryption
+  const off_t test_range = 1024*1024;
+  buffer::ptr buf(test_range);
+  char* p = buf.c_str();
+  for(size_t i = 0; i < buf.length(); i++)
+    p[i] = i + i*i + (i >> 2);
+
+  bufferlist input;
+  input.append(buf);
+
+  for (unsigned int step : {1, 2, 3, 5, 7, 11, 13, 17})
+  {
+    //make some random key
+    uint8_t key[32];
+    for(size_t i=0;i<sizeof(key);i++)
+      key[i]=i*step;
+
+    auto aes(AES_256_CBC_create(g_ceph_context, &key[0], 32));
+    ASSERT_NE(aes.get(), nullptr);
+
+    size_t block_size = aes->get_block_size();
+    ASSERT_NE(block_size, 0u);
+
+    for (off_t end = 1; end < 6096 ; end+=3)
+    {
+      off_t begin = 0;
+      off_t offset = end*end*end*end*end % (1000*1000*1000);
+      offset = offset - offset % block_size;
+
+      ASSERT_EQ(begin % block_size, 0u);
+      ASSERT_LE(end, test_range);
+      ASSERT_EQ(offset % block_size, 0u);
+
+      bufferlist encrypted;
+      ASSERT_TRUE(aes->encrypt(input, begin, end, encrypted, offset));
+      bufferlist decrypted;
+      ASSERT_TRUE(aes->decrypt(encrypted, 0, end, decrypted, offset));
+
+      ASSERT_EQ(decrypted.length(), end);
+      ASSERT_EQ(boost::string_ref(input.c_str(), end),
+                boost::string_ref(decrypted.c_str(), end) );
+    }
+  }
+}
+
+
+TEST(TestRGWCrypto, verify_AES_256_CBC_identity_3)
+{
+  //create some input for encryption
+  const off_t test_range = 1024*1024;
+  buffer::ptr buf(test_range);
+  char* p = buf.c_str();
+  for(size_t i = 0; i < buf.length(); i++)
+    p[i] = i + i*i + (i >> 2);
+
+  bufferlist input;
+  input.append(buf);
+
+  for (unsigned int step : {1, 2, 3, 5, 7, 11, 13, 17})
+  {
+    //make some random key
+    uint8_t key[32];
+    for(size_t i=0;i<sizeof(key);i++)
+      key[i]=i*step;
+
+    auto aes(AES_256_CBC_create(g_ceph_context, &key[0], 32));
+    ASSERT_NE(aes.get(), nullptr);
+
+    size_t block_size = aes->get_block_size();
+    ASSERT_NE(block_size, 0u);
+    size_t rr = 111;
+    for (size_t r = 97; r < 123 ; r++)
+    {
+      off_t begin = 0;
+      off_t end = begin + r*r*r*r*r*r*r % (test_range - begin);
+      //sometimes make aligned
+      if (r % 3)
+        end = end - end % block_size;
+      off_t offset = r*r*r*r*r*r*r*r % (1000*1000*1000);
+      offset = offset - offset % block_size;
+
+      ASSERT_EQ(begin % block_size, 0u);
+      ASSERT_LE(end, test_range);
+      ASSERT_EQ(offset % block_size, 0u);
+
+      bufferlist encrypted1;
+      bufferlist encrypted2;
+
+      off_t pos = begin;
+      off_t chunk;
+      while (pos < end) {
+        chunk = block_size + (rr/3)*(rr+17)*(rr+71)*(rr+123)*(rr+131) % 50000;
+        chunk = chunk - chunk % block_size;
+        if (pos + chunk > end)
+          chunk = end - pos;
+        bufferlist tmp;
+        ASSERT_TRUE(aes->encrypt(input, pos, chunk, tmp, offset + pos));
+        encrypted1.append(tmp);
+        pos += chunk;
+        rr++;
+      }
+
+      pos = begin;
+      while (pos < end) {
+        chunk = block_size + (rr/3)*(rr+97)*(rr+151)*(rr+213)*(rr+251) % 50000;
+        chunk = chunk - chunk % block_size;
+        if (pos + chunk > end)
+          chunk = end - pos;
+        bufferlist tmp;
+        ASSERT_TRUE(aes->encrypt(input, pos, chunk, tmp, offset + pos));
+        encrypted2.append(tmp);
+        pos += chunk;
+        rr++;
+      }
+      ASSERT_EQ(encrypted1.length(), end);
+      ASSERT_EQ(encrypted2.length(), end);
+      ASSERT_EQ(boost::string_ref(encrypted1.c_str(), end),
+                boost::string_ref(encrypted2.c_str(), end) );
+    }
+  }
+}
+
+
+TEST(TestRGWCrypto, verify_AES_256_CBC_size_0_15)
+{
+  //create some input for encryption
+  const off_t test_range = 1024*1024;
+  buffer::ptr buf(test_range);
+  char* p = buf.c_str();
+  for(size_t i = 0; i < buf.length(); i++)
+    p[i] = i + i*i + (i >> 2);
+
+  bufferlist input;
+  input.append(buf);
+
+  for (unsigned int step : {1, 2, 3, 5, 7, 11, 13, 17})
+  {
+    //make some random key
+    uint8_t key[32];
+    for(size_t i=0;i<sizeof(key);i++)
+      key[i]=i*step;
+
+    auto aes(AES_256_CBC_create(g_ceph_context, &key[0], 32));
+    ASSERT_NE(aes.get(), nullptr);
+
+    size_t block_size = aes->get_block_size();
+    ASSERT_NE(block_size, 0u);
+    for (size_t r = 97; r < 123 ; r++)
+    {
+      off_t begin = 0;
+      off_t end = begin + r*r*r*r*r*r*r % (16);
+
+      off_t offset = r*r*r*r*r*r*r*r % (1000*1000*1000);
+      offset = offset - offset % block_size;
+
+      ASSERT_EQ(begin % block_size, 0u);
+      ASSERT_LE(end, test_range);
+      ASSERT_EQ(offset % block_size, 0u);
+
+      bufferlist encrypted;
+      bufferlist decrypted;
+      ASSERT_TRUE(aes->encrypt(input, 0, end, encrypted, offset));
+      ASSERT_TRUE(aes->encrypt(encrypted, 0, end, decrypted, offset));
+      ASSERT_EQ(encrypted.length(), end);
+      ASSERT_EQ(decrypted.length(), end);
+      ASSERT_EQ(boost::string_ref(input.c_str(), end),
+                boost::string_ref(decrypted.c_str(), end) );
+    }
+  }
+}
+
+
+TEST(TestRGWCrypto, verify_AES_256_CBC_identity_last_block)
+{
+  //create some input for encryption
+  const off_t test_range = 1024*1024;
+  buffer::ptr buf(test_range);
+  char* p = buf.c_str();
+  for(size_t i = 0; i < buf.length(); i++)
+    p[i] = i + i*i + (i >> 2);
+
+  bufferlist input;
+  input.append(buf);
+
+  for (unsigned int step : {1, 2, 3, 5, 7, 11, 13, 17})
+  {
+    //make some random key
+    uint8_t key[32];
+    for(size_t i=0;i<sizeof(key);i++)
+      key[i]=i*step;
+
+    auto aes(AES_256_CBC_create(g_ceph_context, &key[0], 32));
+    ASSERT_NE(aes.get(), nullptr);
+
+    size_t block_size = aes->get_block_size();
+    ASSERT_NE(block_size, 0u);
+    size_t rr = 111;
+    for (size_t r = 97; r < 123 ; r++)
+    {
+      off_t begin = 0;
+      off_t end = r*r*r*r*r*r*r % (test_range - 16);
+      end = end - end % block_size;
+      end = end + (r+3)*(r+5)*(r+7) % 16;
+
+      off_t offset = r*r*r*r*r*r*r*r % (1000*1000*1000);
+      offset = offset - offset % block_size;
+
+      ASSERT_EQ(begin % block_size, 0u);
+      ASSERT_LE(end, test_range);
+      ASSERT_EQ(offset % block_size, 0u);
+
+      bufferlist encrypted1;
+      bufferlist encrypted2;
+
+      off_t pos = begin;
+      off_t chunk;
+      while (pos < end) {
+        chunk = block_size + (rr/3)*(rr+17)*(rr+71)*(rr+123)*(rr+131) % 50000;
+        chunk = chunk - chunk % block_size;
+        if (pos + chunk > end)
+          chunk = end - pos;
+        bufferlist tmp;
+        ASSERT_TRUE(aes->encrypt(input, pos, chunk, tmp, offset + pos));
+        encrypted1.append(tmp);
+        pos += chunk;
+        rr++;
+      }
+      pos = begin;
+      while (pos < end) {
+        chunk = block_size + (rr/3)*(rr+97)*(rr+151)*(rr+213)*(rr+251) % 50000;
+        chunk = chunk - chunk % block_size;
+        if (pos + chunk > end)
+          chunk = end - pos;
+        bufferlist tmp;
+        ASSERT_TRUE(aes->encrypt(input, pos, chunk, tmp, offset + pos));
+        encrypted2.append(tmp);
+        pos += chunk;
+        rr++;
+      }
+      ASSERT_EQ(encrypted1.length(), end);
+      ASSERT_EQ(encrypted2.length(), end);
+      ASSERT_EQ(boost::string_ref(encrypted1.c_str(), end),
+                boost::string_ref(encrypted2.c_str(), end) );
+    }
+  }
+}
+
+
+TEST(TestRGWCrypto, verify_RGWGetObj_BlockDecrypt_ranges)
+{
+  //create some input for encryption
+  const off_t test_range = 1024*1024;
+  bufferptr buf(test_range);
+  char* p = buf.c_str();
+  for(size_t i = 0; i < buf.length(); i++)
+    p[i] = i + i*i + (i >> 2);
+
+  bufferlist input;
+  input.append(buf);
+
+  uint8_t key[32];
+  for(size_t i=0;i<sizeof(key);i++)
+    key[i] = i;
+
+  auto cbc = AES_256_CBC_create(g_ceph_context, &key[0], 32);
+  ASSERT_NE(cbc.get(), nullptr);
+  bufferlist encrypted;
+  ASSERT_TRUE(cbc->encrypt(input, 0, test_range, encrypted, 0));
+
+
+  for (off_t r = 93; r < 150; r++ )
+  {
+    ut_get_sink get_sink;
+    auto cbc = AES_256_CBC_create(g_ceph_context, &key[0], 32);
+    ASSERT_NE(cbc.get(), nullptr);
+    RGWGetObj_BlockDecrypt decrypt(g_ceph_context, &get_sink, std::move(cbc) );
+
+    //random ranges
+    off_t begin = (r/3)*r*(r+13)*(r+23)*(r+53)*(r+71) % test_range;
+    off_t end = begin + (r/5)*(r+7)*(r+13)*(r+101)*(r*103) % (test_range - begin) - 1;
+
+    off_t f_begin = begin;
+    off_t f_end = end;
+    decrypt.fixup_range(f_begin, f_end);
+    decrypt.handle_data(encrypted, f_begin, f_end - f_begin + 1);
+    decrypt.flush();
+    const std::string& decrypted = get_sink.get_sink();
+    size_t expected_len = end - begin + 1;
+    ASSERT_EQ(decrypted.length(), expected_len);
+    ASSERT_EQ(decrypted, boost::string_ref(input.c_str()+begin, expected_len));
+  }
+}
+
+
+TEST(TestRGWCrypto, verify_RGWGetObj_BlockDecrypt_chunks)
+{
+  //create some input for encryption
+  const off_t test_range = 1024*1024;
+  bufferptr buf(test_range);
+  char* p = buf.c_str();
+  for(size_t i = 0; i < buf.length(); i++)
+    p[i] = i + i*i + (i >> 2);
+
+  bufferlist input;
+  input.append(buf);
+
+  uint8_t key[32];
+  for(size_t i=0;i<sizeof(key);i++)
+    key[i] = i;
+
+  auto cbc = AES_256_CBC_create(g_ceph_context, &key[0], 32);
+  ASSERT_NE(cbc.get(), nullptr);
+  bufferlist encrypted;
+  ASSERT_TRUE(cbc->encrypt(input, 0, test_range, encrypted, 0));
+
+  for (off_t r = 93; r < 150; r++ )
+  {
+    ut_get_sink get_sink;
+    auto cbc = AES_256_CBC_create(g_ceph_context, &key[0], 32);
+    ASSERT_NE(cbc.get(), nullptr);
+    RGWGetObj_BlockDecrypt decrypt(g_ceph_context, &get_sink, std::move(cbc) );
+
+    //random
+    off_t begin = (r/3)*r*(r+13)*(r+23)*(r+53)*(r+71) % test_range;
+    off_t end = begin + (r/5)*(r+7)*(r+13)*(r+101)*(r*103) % (test_range - begin) - 1;
+
+    off_t f_begin = begin;
+    off_t f_end = end;
+    decrypt.fixup_range(f_begin, f_end);
+    off_t pos = f_begin;
+    do
+    {
+      off_t size = 2 << ((pos * 17 + pos / 113 + r) % 16);
+      size = (pos + 1117) * (pos + 2229) % size + 1;
+      if (pos + size > f_end + 1)
+        size = f_end + 1 - pos;
+
+      decrypt.handle_data(encrypted, pos, size);
+      pos = pos + size;
+    } while (pos < f_end + 1);
+    decrypt.flush();
+
+    const std::string& decrypted = get_sink.get_sink();
+    size_t expected_len = end - begin + 1;
+    ASSERT_EQ(decrypted.length(), expected_len);
+    ASSERT_EQ(decrypted, boost::string_ref(input.c_str()+begin, expected_len));
+  }
+}
+
+
+using range_t = std::pair<off_t, off_t>;
+
+// call filter->fixup_range() and return the range as a pair. this makes it easy
+// to fit on a single line for ASSERT_EQ()
+range_t fixup_range(RGWGetObj_BlockDecrypt *decrypt, off_t ofs, off_t end)
+{
+  decrypt->fixup_range(ofs, end);
+  return {ofs, end};
+}
+
+TEST(TestRGWCrypto, check_RGWGetObj_BlockDecrypt_fixup)
+{
+  ut_get_sink get_sink;
+  auto nonecrypt = std::unique_ptr<BlockCrypt>(new BlockCryptNone);
+  RGWGetObj_BlockDecrypt decrypt(g_ceph_context, &get_sink,
+                                 std::move(nonecrypt));
+  ASSERT_EQ(fixup_range(&decrypt,0,0),     range_t(0,255));
+  ASSERT_EQ(fixup_range(&decrypt,1,256),   range_t(0,511));
+  ASSERT_EQ(fixup_range(&decrypt,0,255),   range_t(0,255));
+  ASSERT_EQ(fixup_range(&decrypt,255,256), range_t(0,511));
+  ASSERT_EQ(fixup_range(&decrypt,511,1023), range_t(256,1023));
+  ASSERT_EQ(fixup_range(&decrypt,513,1024), range_t(512,1024+255));
+}
+
+using parts_len_t = std::vector<size_t>;
+
+class TestRGWGetObj_BlockDecrypt : public RGWGetObj_BlockDecrypt {
+  using RGWGetObj_BlockDecrypt::RGWGetObj_BlockDecrypt;
+public:
+  void set_parts_len(parts_len_t&& other) {
+    parts_len = std::move(other);
+  }
+};
+
+std::vector<size_t> create_mp_parts(size_t obj_size, size_t mp_part_len){
+  std::vector<size_t> parts_len;
+  size_t part_size;
+  size_t ofs=0;
+
+  while (ofs < obj_size){
+    part_size = std::min(mp_part_len, (obj_size - ofs));
+    ofs += part_size;
+    parts_len.push_back(part_size);
+  }
+  return parts_len;
+}
+
+const size_t part_size = 5*1024*1024;
+const size_t obj_size = 30*1024*1024;
+
+TEST(TestRGWCrypto, check_RGWGetObj_BlockDecrypt_fixup_simple)
+{
+
+  ut_get_sink get_sink;
+  auto nonecrypt = std::make_unique<BlockCryptNone>(4096);
+  TestRGWGetObj_BlockDecrypt decrypt(g_ceph_context, &get_sink,
+				     std::move(nonecrypt));
+  decrypt.set_parts_len(create_mp_parts(obj_size, part_size));
+  ASSERT_EQ(fixup_range(&decrypt,0,0),     range_t(0,4095));
+  ASSERT_EQ(fixup_range(&decrypt,1,4096),   range_t(0,8191));
+  ASSERT_EQ(fixup_range(&decrypt,0,4095),   range_t(0,4095));
+  ASSERT_EQ(fixup_range(&decrypt,4095,4096), range_t(0,8191));
+
+  // ranges are end-end inclusive, we request bytes just spanning short of first
+  // part to exceeding the first part, part_size - 1 is aligned to a 4095 boundary
+  ASSERT_EQ(fixup_range(&decrypt, 0, part_size - 2), range_t(0, part_size -1));
+  ASSERT_EQ(fixup_range(&decrypt, 0, part_size - 1), range_t(0, part_size -1));
+  ASSERT_EQ(fixup_range(&decrypt, 0, part_size),     range_t(0, part_size + 4095));
+  ASSERT_EQ(fixup_range(&decrypt, 0, part_size + 1), range_t(0, part_size + 4095));
+
+  // request bytes spanning 2 parts
+  ASSERT_EQ(fixup_range(&decrypt, part_size -2, part_size + 2),
+	    range_t(part_size - 4096, part_size + 4095));
+
+  // request last byte
+  ASSERT_EQ(fixup_range(&decrypt, obj_size - 1, obj_size -1),
+	    range_t(obj_size - 4096, obj_size -1));
+
+}
+
+TEST(TestRGWCrypto, check_RGWGetObj_BlockDecrypt_fixup_non_aligned_obj_size)
+{
+
+  ut_get_sink get_sink;
+  auto nonecrypt = std::make_unique<BlockCryptNone>(4096);
+  TestRGWGetObj_BlockDecrypt decrypt(g_ceph_context, &get_sink,
+				     std::move(nonecrypt));
+  auto na_obj_size = obj_size + 1;
+  decrypt.set_parts_len(create_mp_parts(na_obj_size, part_size));
+
+  // these should be unaffected here
+  ASSERT_EQ(fixup_range(&decrypt, 0, part_size - 2), range_t(0, part_size -1));
+  ASSERT_EQ(fixup_range(&decrypt, 0, part_size - 1), range_t(0, part_size -1));
+  ASSERT_EQ(fixup_range(&decrypt, 0, part_size),     range_t(0, part_size + 4095));
+  ASSERT_EQ(fixup_range(&decrypt, 0, part_size + 1), range_t(0, part_size + 4095));
+
+
+  // request last 2 bytes; spanning 2 parts
+  ASSERT_EQ(fixup_range(&decrypt, na_obj_size -2 , na_obj_size -1),
+	    range_t(na_obj_size - 1 - 4096, na_obj_size - 1));
+
+  // request last byte, spans last 1B part only
+  ASSERT_EQ(fixup_range(&decrypt, na_obj_size -1, na_obj_size - 1),
+	    range_t(na_obj_size - 1, na_obj_size -1));
+
+}
+
+TEST(TestRGWCrypto, check_RGWGetObj_BlockDecrypt_fixup_non_aligned_part_size)
+{
+
+  ut_get_sink get_sink;
+  auto nonecrypt = std::make_unique<BlockCryptNone>(4096);
+  TestRGWGetObj_BlockDecrypt decrypt(g_ceph_context, &get_sink,
+				     std::move(nonecrypt));
+  auto na_part_size = part_size + 1;
+  decrypt.set_parts_len(create_mp_parts(obj_size, na_part_size));
+
+  // na_part_size -2, ie. part_size -1  is aligned to 4095 boundary
+  ASSERT_EQ(fixup_range(&decrypt, 0, na_part_size - 2), range_t(0, na_part_size -2));
+  // even though na_part_size -1 should not align to a 4095 boundary, the range
+  // should not span the next part
+  ASSERT_EQ(fixup_range(&decrypt, 0, na_part_size - 1), range_t(0, na_part_size -1));
+
+  ASSERT_EQ(fixup_range(&decrypt, 0, na_part_size),     range_t(0, na_part_size + 4095));
+  ASSERT_EQ(fixup_range(&decrypt, 0, na_part_size + 1), range_t(0, na_part_size + 4095));
+
+  // request spanning 2 parts
+  ASSERT_EQ(fixup_range(&decrypt, na_part_size - 2, na_part_size + 2),
+	    range_t(na_part_size - 1 - 4096, na_part_size + 4095));
+
+  // request last byte, this will be interesting, since this a multipart upload
+  // with 5MB+1 size, the last part is actually 5 bytes short of 5 MB, which
+  // should be considered for the ranges alignment; an easier way to look at
+  // this will be that the last offset aligned to a 5MiB part will be 5MiB -
+  // 4095, this is a part that is 5MiB - 5 B
+  ASSERT_EQ(fixup_range(&decrypt, obj_size - 1, obj_size -1),
+	    range_t(obj_size +5 -4096, obj_size -1));
+
+}
+
+TEST(TestRGWCrypto, check_RGWGetObj_BlockDecrypt_fixup_non_aligned)
+{
+
+  ut_get_sink get_sink;
+  auto nonecrypt = std::make_unique<BlockCryptNone>(4096);
+  TestRGWGetObj_BlockDecrypt decrypt(g_ceph_context, &get_sink,
+				     std::move(nonecrypt));
+  auto na_part_size = part_size + 1;
+  auto na_obj_size = obj_size + 7; // (6*(5MiB + 1) + 1) for the last 1B overflow
+  decrypt.set_parts_len(create_mp_parts(na_obj_size, na_part_size));
+
+  // na_part_size -2, ie. part_size -1  is aligned to 4095 boundary
+  ASSERT_EQ(fixup_range(&decrypt, 0, na_part_size - 2), range_t(0, na_part_size -2));
+  // even though na_part_size -1 should not align to a 4095 boundary, the range
+  // should not span the next part
+  ASSERT_EQ(fixup_range(&decrypt, 0, na_part_size - 1), range_t(0, na_part_size -1));
+
+  ASSERT_EQ(fixup_range(&decrypt, 0, na_part_size),     range_t(0, na_part_size + 4095));
+  ASSERT_EQ(fixup_range(&decrypt, 0, na_part_size + 1), range_t(0, na_part_size + 4095));
+
+  // request last byte, spans last 1B part only
+  ASSERT_EQ(fixup_range(&decrypt, na_obj_size -1, na_obj_size - 1),
+	    range_t(na_obj_size - 1, na_obj_size -1));
+
+  ASSERT_EQ(fixup_range(&decrypt, na_obj_size -2, na_obj_size -1),
+	    range_t(na_obj_size - 2, na_obj_size -1));
+
+}
+
+TEST(TestRGWCrypto, check_RGWGetObj_BlockDecrypt_fixup_invalid_ranges)
+{
+
+  ut_get_sink get_sink;
+  auto nonecrypt = std::make_unique<BlockCryptNone>(4096);
+  TestRGWGetObj_BlockDecrypt decrypt(g_ceph_context, &get_sink,
+				     std::move(nonecrypt));
+
+  decrypt.set_parts_len(create_mp_parts(obj_size, part_size));
+
+  // the ranges below would be mostly unreachable in current code as rgw
+  // would've returned a 411 before reaching, but we're just doing this to make
+  // sure we don't have invalid access
+  ASSERT_EQ(fixup_range(&decrypt, obj_size - 1, obj_size + 100),
+            range_t(obj_size - 4096, obj_size - 1));
+  ASSERT_EQ(fixup_range(&decrypt, obj_size, obj_size + 1),
+            range_t(obj_size - 1, obj_size - 1));
+  ASSERT_EQ(fixup_range(&decrypt, obj_size+1, obj_size + 100),
+            range_t(obj_size - 1, obj_size - 1));
+
+}
+
+TEST(TestRGWCrypto, verify_RGWPutObj_BlockEncrypt_chunks)
+{
+  //create some input for encryption
+  const off_t test_range = 1024*1024;
+  bufferptr buf(test_range);
+  char* p = buf.c_str();
+  for(size_t i = 0; i < buf.length(); i++)
+    p[i] = i + i*i + (i >> 2);
+
+  bufferlist input;
+  input.append(buf);
+
+  uint8_t key[32];
+  for(size_t i=0;i<sizeof(key);i++)
+    key[i] = i;
+
+  for (off_t r = 93; r < 150; r++ )
+  {
+    ut_put_sink put_sink;
+    auto cbc = AES_256_CBC_create(g_ceph_context, &key[0], 32);
+    ASSERT_NE(cbc.get(), nullptr);
+    RGWPutObj_BlockEncrypt encrypt(g_ceph_context, &put_sink,
+                                   std::move(cbc) );
+
+    off_t test_size = (r/5)*(r+7)*(r+13)*(r+101)*(r*103) % (test_range - 1) + 1;
+    off_t pos = 0;
+    do
+    {
+      off_t size = 2 << ((pos * 17 + pos / 113 + r) % 16);
+      size = (pos + 1117) * (pos + 2229) % size + 1;
+      if (pos + size > test_size)
+        size = test_size - pos;
+
+      bufferlist bl;
+      bl.append(input.c_str()+pos, size);
+      encrypt.process(std::move(bl), pos);
+
+      pos = pos + size;
+    } while (pos < test_size);
+    encrypt.process({}, pos);
+
+    ASSERT_EQ(put_sink.get_sink().length(), static_cast<size_t>(test_size));
+
+    cbc = AES_256_CBC_create(g_ceph_context, &key[0], 32);
+    ASSERT_NE(cbc.get(), nullptr);
+
+    bufferlist encrypted;
+    bufferlist decrypted;
+    encrypted.append(put_sink.get_sink());
+    ASSERT_TRUE(cbc->decrypt(encrypted, 0, test_size, decrypted, 0));
+
+    ASSERT_EQ(decrypted.length(), test_size);
+    ASSERT_EQ(boost::string_ref(decrypted.c_str(), test_size),
+              boost::string_ref(input.c_str(), test_size));
+  }
+}
+
+
+TEST(TestRGWCrypto, verify_Encrypt_Decrypt)
+{
+  uint8_t key[32];
+  for(size_t i=0;i<sizeof(key);i++)
+    key[i]=i;
+
+  size_t fi_a = 0;
+  size_t fi_b = 1;
+  size_t test_size;
+  do
+  {
+    //fibonacci
+    size_t tmp = fi_b;
+    fi_b = fi_a + fi_b;
+    fi_a = tmp;
+
+    test_size = fi_b;
+
+    uint8_t* test_in = new uint8_t[test_size];
+    //fill with something
+    memset(test_in, test_size & 0xff, test_size);
+
+    ut_put_sink put_sink;
+    RGWPutObj_BlockEncrypt encrypt(g_ceph_context, &put_sink,
+				   AES_256_CBC_create(g_ceph_context, &key[0], 32) );
+    bufferlist bl;
+    bl.append((char*)test_in, test_size);
+    encrypt.process(std::move(bl), 0);
+    encrypt.process({}, test_size);
+    ASSERT_EQ(put_sink.get_sink().length(), test_size);
+
+    bl.append(put_sink.get_sink().data(), put_sink.get_sink().length());
+    ASSERT_EQ(bl.length(), test_size);
+
+    ut_get_sink get_sink;
+    RGWGetObj_BlockDecrypt decrypt(g_ceph_context, &get_sink,
+                                   AES_256_CBC_create(g_ceph_context, &key[0], 32) );
+
+    off_t bl_ofs = 0;
+    off_t bl_end = test_size - 1;
+    decrypt.fixup_range(bl_ofs, bl_end);
+    decrypt.handle_data(bl, 0, bl.length());
+    decrypt.flush();
+    ASSERT_EQ(get_sink.get_sink().length(), test_size);
+    ASSERT_EQ(get_sink.get_sink(), boost::string_ref((char*)test_in,test_size));
+  }
+  while (test_size < 20000);
+}
+
+
+int main(int argc, char **argv) {
+  vector<const char*> args;
+  argv_to_vec(argc, (const char **)argv, args);
+
+  auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT,
+			 CODE_ENVIRONMENT_UTILITY,
+			 CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+  common_init_finish(g_ceph_context);
+
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
+
diff --git a/src/test/rgw/test_rgw_dmclock_scheduler.cc b/src/test/rgw/test_rgw_dmclock_scheduler.cc
new file mode 100644
index 00000000..b7e9b0ea
--- /dev/null
+++ b/src/test/rgw/test_rgw_dmclock_scheduler.cc
@@ -0,0 +1,433 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2018 Red Hat, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+//#define BOOST_ASIO_ENABLE_HANDLER_TRACKING
+
+#include "rgw/rgw_dmclock_sync_scheduler.h"
+#include "rgw/rgw_dmclock_async_scheduler.h"
+
+#include <optional>
+#ifdef HAVE_BOOST_CONTEXT
+#include <boost/asio/spawn.hpp>
+#endif
+#include <gtest/gtest.h>
+#include "acconfig.h"
+#include "global/global_context.h"
+
+namespace rgw::dmclock {
+
+using boost::system::error_code;
+
+// return a lambda that can be used as a callback to capture its arguments
+auto capture(std::optional<error_code>& opt_ec,
+             std::optional<PhaseType>& opt_phase)
+{
+  return [&] (error_code ec, PhaseType phase) {
+    opt_ec = ec;
+    opt_phase = phase;
+  };
+}
+
+TEST(Queue, SyncRequest)
+{
+  ClientCounters counters(g_ceph_context);
+  auto client_info_f = [] (client_id client) -> ClientInfo* {
+                         static ClientInfo clients[] = {
+                                                        {1, 1, 1}, //admin: satisfy by reservation
+                                                        {0, 1, 1}, //auth: satisfy by priority
+                         };
+                         return &clients[static_cast<size_t>(client)];
+                       };
+  std::atomic <bool> ready = false;
+  auto server_ready_f = [&ready]() -> bool { return ready.load();};
+
+  SyncScheduler queue(g_ceph_context, std::ref(counters),
+		      client_info_f, server_ready_f,
+		      std::ref(SyncScheduler::handle_request_cb)
+		      );
+
+
+  auto now = get_time();
+  ready = true;
+  queue.add_request(client_id::admin, {}, now, 1);
+  queue.add_request(client_id::auth, {}, now, 1);
+
+  // We can't see the queue at length 1 as the queue len is decremented as the
+  //request is processed
+  EXPECT_EQ(0u, counters(client_id::admin)->get(queue_counters::l_qlen));
+  EXPECT_EQ(1u, counters(client_id::admin)->get(queue_counters::l_res));
+  EXPECT_EQ(0u, counters(client_id::admin)->get(queue_counters::l_prio));
+  EXPECT_EQ(0u, counters(client_id::admin)->get(queue_counters::l_limit));
+  EXPECT_EQ(0u, counters(client_id::admin)->get(queue_counters::l_cancel));
+
+  EXPECT_EQ(0u, counters(client_id::auth)->get(queue_counters::l_qlen));
+  EXPECT_EQ(0u, counters(client_id::auth)->get(queue_counters::l_res));
+  EXPECT_EQ(1u, counters(client_id::auth)->get(queue_counters::l_prio));
+  EXPECT_EQ(0u, counters(client_id::auth)->get(queue_counters::l_limit));
+  EXPECT_EQ(0u, counters(client_id::auth)->get(queue_counters::l_cancel));
+}
+
+#ifdef HAVE_BOOST_CONTEXT
+TEST(Queue, RateLimit)
+{
+  boost::asio::io_context context;
+  ClientCounters counters(g_ceph_context);
+  AsyncScheduler queue(g_ceph_context, context, std::ref(counters), nullptr,
+                  [] (client_id client) -> ClientInfo* {
+      static ClientInfo clients[] = {
+        {1, 1, 1}, // admin
+        {0, 1, 1}, // auth
+      };
+      return &clients[static_cast<size_t>(client)];
+    }, AtLimit::Reject);
+
+  std::optional<error_code> ec1, ec2, ec3, ec4;
+  std::optional<PhaseType> p1, p2, p3, p4;
+
+  auto now = get_time();
+  queue.async_request(client_id::admin, {}, now, 1, capture(ec1, p1));
+  queue.async_request(client_id::admin, {}, now, 1, capture(ec2, p2));
+  queue.async_request(client_id::auth, {}, now, 1, capture(ec3, p3));
+  queue.async_request(client_id::auth, {}, now, 1, capture(ec4, p4));
+  EXPECT_FALSE(ec1);
+  EXPECT_FALSE(ec2);
+  EXPECT_FALSE(ec3);
+  EXPECT_FALSE(ec4);
+
+  EXPECT_EQ(1u, counters(client_id::admin)->get(queue_counters::l_qlen));
+  EXPECT_EQ(1u, counters(client_id::auth)->get(queue_counters::l_qlen));
+
+  context.poll();
+  EXPECT_TRUE(context.stopped());
+
+  ASSERT_TRUE(ec1);
+  EXPECT_EQ(boost::system::errc::success, *ec1);
+  ASSERT_TRUE(p1);
+  EXPECT_EQ(PhaseType::reservation, *p1);
+
+  ASSERT_TRUE(ec2);
+  EXPECT_EQ(boost::system::errc::resource_unavailable_try_again, *ec2);
+
+  ASSERT_TRUE(ec3);
+  EXPECT_EQ(boost::system::errc::success, *ec3);
+  ASSERT_TRUE(p3);
+  EXPECT_EQ(PhaseType::priority, *p3);
+
+  ASSERT_TRUE(ec4);
+  EXPECT_EQ(boost::system::errc::resource_unavailable_try_again, *ec4);
+
+  EXPECT_EQ(0u, counters(client_id::admin)->get(queue_counters::l_qlen));
+  EXPECT_EQ(1u, counters(client_id::admin)->get(queue_counters::l_res));
+  EXPECT_EQ(0u, counters(client_id::admin)->get(queue_counters::l_prio));
+  EXPECT_EQ(1u, counters(client_id::admin)->get(queue_counters::l_limit));
+  EXPECT_EQ(0u, counters(client_id::admin)->get(queue_counters::l_cancel));
+
+  EXPECT_EQ(0u, counters(client_id::auth)->get(queue_counters::l_qlen));
+  EXPECT_EQ(0u, counters(client_id::auth)->get(queue_counters::l_res));
+  EXPECT_EQ(1u, counters(client_id::auth)->get(queue_counters::l_prio));
+  EXPECT_EQ(1u, counters(client_id::auth)->get(queue_counters::l_limit));
+  EXPECT_EQ(0u, counters(client_id::auth)->get(queue_counters::l_cancel));
+}
+
+TEST(Queue, AsyncRequest)
+{
+  boost::asio::io_context context;
+  ClientCounters counters(g_ceph_context);
+  AsyncScheduler queue(g_ceph_context, context, std::ref(counters), nullptr,
+                  [] (client_id client) -> ClientInfo* {
+      static ClientInfo clients[] = {
+        {1, 1, 1}, // admin: satisfy by reservation
+        {0, 1, 1}, // auth: satisfy by priority
+      };
+      return &clients[static_cast<size_t>(client)];
+		  }, AtLimit::Reject
+		  );
+
+  std::optional<error_code> ec1, ec2;
+  std::optional<PhaseType> p1, p2;
+
+  auto now = get_time();
+  queue.async_request(client_id::admin, {}, now, 1, capture(ec1, p1));
+  queue.async_request(client_id::auth, {}, now, 1, capture(ec2, p2));
+  EXPECT_FALSE(ec1);
+  EXPECT_FALSE(ec2);
+
+  EXPECT_EQ(1u, counters(client_id::admin)->get(queue_counters::l_qlen));
+  EXPECT_EQ(1u, counters(client_id::auth)->get(queue_counters::l_qlen));
+
+  context.poll();
+  EXPECT_TRUE(context.stopped());
+
+  ASSERT_TRUE(ec1);
+  EXPECT_EQ(boost::system::errc::success, *ec1);
+  ASSERT_TRUE(p1);
+  EXPECT_EQ(PhaseType::reservation, *p1);
+
+  ASSERT_TRUE(ec2);
+  EXPECT_EQ(boost::system::errc::success, *ec2);
+  ASSERT_TRUE(p2);
+  EXPECT_EQ(PhaseType::priority, *p2);
+
+  EXPECT_EQ(0u, counters(client_id::admin)->get(queue_counters::l_qlen));
+  EXPECT_EQ(1u, counters(client_id::admin)->get(queue_counters::l_res));
+  EXPECT_EQ(0u, counters(client_id::admin)->get(queue_counters::l_prio));
+  EXPECT_EQ(0u, counters(client_id::admin)->get(queue_counters::l_limit));
+  EXPECT_EQ(0u, counters(client_id::admin)->get(queue_counters::l_cancel));
+
+  EXPECT_EQ(0u, counters(client_id::auth)->get(queue_counters::l_qlen));
+  EXPECT_EQ(0u, counters(client_id::auth)->get(queue_counters::l_res));
+  EXPECT_EQ(1u, counters(client_id::auth)->get(queue_counters::l_prio));
+  EXPECT_EQ(0u, counters(client_id::auth)->get(queue_counters::l_limit));
+  EXPECT_EQ(0u, counters(client_id::auth)->get(queue_counters::l_cancel));
+}
+
+
+TEST(Queue, Cancel)
+{
+  boost::asio::io_context context;
+  ClientCounters counters(g_ceph_context);
+  AsyncScheduler queue(g_ceph_context, context, std::ref(counters), nullptr,
+                  [] (client_id client) -> ClientInfo* {
+      static ClientInfo info{0, 1, 1};
+      return &info;
+    });
+
+  std::optional<error_code> ec1, ec2;
+  std::optional<PhaseType> p1, p2;
+
+  auto now = get_time();
+  queue.async_request(client_id::admin, {}, now, 1, capture(ec1, p1));
+  queue.async_request(client_id::auth, {}, now, 1, capture(ec2, p2));
+  EXPECT_FALSE(ec1);
+  EXPECT_FALSE(ec2);
+
+  EXPECT_EQ(1u, counters(client_id::admin)->get(queue_counters::l_qlen));
+  EXPECT_EQ(1u, counters(client_id::auth)->get(queue_counters::l_qlen));
+
+  queue.cancel();
+
+  EXPECT_FALSE(ec1);
+  EXPECT_FALSE(ec2);
+
+  context.poll();
+  EXPECT_TRUE(context.stopped());
+
+  ASSERT_TRUE(ec1);
+  EXPECT_EQ(boost::asio::error::operation_aborted, *ec1);
+  ASSERT_TRUE(ec2);
+  EXPECT_EQ(boost::asio::error::operation_aborted, *ec2);
+
+  EXPECT_EQ(0u, counters(client_id::admin)->get(queue_counters::l_qlen));
+  EXPECT_EQ(0u, counters(client_id::admin)->get(queue_counters::l_res));
+  EXPECT_EQ(0u, counters(client_id::admin)->get(queue_counters::l_prio));
+  EXPECT_EQ(0u, counters(client_id::admin)->get(queue_counters::l_limit));
+  EXPECT_EQ(1u, counters(client_id::admin)->get(queue_counters::l_cancel));
+
+  EXPECT_EQ(0u, counters(client_id::auth)->get(queue_counters::l_qlen));
+  EXPECT_EQ(0u, counters(client_id::auth)->get(queue_counters::l_res));
+  EXPECT_EQ(0u, counters(client_id::auth)->get(queue_counters::l_prio));
+  EXPECT_EQ(0u, counters(client_id::auth)->get(queue_counters::l_limit));
+  EXPECT_EQ(1u, counters(client_id::auth)->get(queue_counters::l_cancel));
+}
+
+TEST(Queue, CancelClient)
+{
+  boost::asio::io_context context;
+  ClientCounters counters(g_ceph_context);
+  AsyncScheduler queue(g_ceph_context, context, std::ref(counters), nullptr,
+                  [] (client_id client) -> ClientInfo* {
+      static ClientInfo info{0, 1, 1};
+      return &info;
+    });
+
+  std::optional<error_code> ec1, ec2;
+  std::optional<PhaseType> p1, p2;
+
+  auto now = get_time();
+  queue.async_request(client_id::admin, {}, now, 1, capture(ec1, p1));
+  queue.async_request(client_id::auth, {}, now, 1, capture(ec2, p2));
+  EXPECT_FALSE(ec1);
+  EXPECT_FALSE(ec2);
+
+  EXPECT_EQ(1u, counters(client_id::admin)->get(queue_counters::l_qlen));
+  EXPECT_EQ(1u, counters(client_id::auth)->get(queue_counters::l_qlen));
+
+  queue.cancel(client_id::admin);
+
+  EXPECT_FALSE(ec1);
+  EXPECT_FALSE(ec2);
+
+  context.poll();
+  EXPECT_TRUE(context.stopped());
+
+  ASSERT_TRUE(ec1);
+  EXPECT_EQ(boost::asio::error::operation_aborted, *ec1);
+
+  ASSERT_TRUE(ec2);
+  EXPECT_EQ(boost::system::errc::success, *ec2);
+  ASSERT_TRUE(p2);
+  EXPECT_EQ(PhaseType::priority, *p2);
+
+  EXPECT_EQ(0u, counters(client_id::admin)->get(queue_counters::l_qlen));
+  EXPECT_EQ(0u, counters(client_id::admin)->get(queue_counters::l_res));
+  EXPECT_EQ(0u, counters(client_id::admin)->get(queue_counters::l_prio));
+  EXPECT_EQ(0u, counters(client_id::admin)->get(queue_counters::l_limit));
+  EXPECT_EQ(1u, counters(client_id::admin)->get(queue_counters::l_cancel));
+
+  EXPECT_EQ(0u, counters(client_id::auth)->get(queue_counters::l_qlen));
+  EXPECT_EQ(0u, counters(client_id::auth)->get(queue_counters::l_res));
+  EXPECT_EQ(1u, counters(client_id::auth)->get(queue_counters::l_prio));
+  EXPECT_EQ(0u, counters(client_id::auth)->get(queue_counters::l_limit));
+  EXPECT_EQ(0u, counters(client_id::auth)->get(queue_counters::l_cancel));
+}
+
+TEST(Queue, CancelOnDestructor)
+{
+  boost::asio::io_context context;
+
+  std::optional<error_code> ec1, ec2;
+  std::optional<PhaseType> p1, p2;
+
+  ClientCounters counters(g_ceph_context);
+  {
+    AsyncScheduler queue(g_ceph_context, context, std::ref(counters), nullptr,
+                    [] (client_id client) -> ClientInfo* {
+        static ClientInfo info{0, 1, 1};
+        return &info;
+      });
+
+    auto now = get_time();
+    queue.async_request(client_id::admin, {}, now, 1, capture(ec1, p1));
+    queue.async_request(client_id::auth, {}, now, 1, capture(ec2, p2));
+
+    EXPECT_EQ(1u, counters(client_id::admin)->get(queue_counters::l_qlen));
+    EXPECT_EQ(1u, counters(client_id::auth)->get(queue_counters::l_qlen));
+  }
+
+  EXPECT_FALSE(ec1);
+  EXPECT_FALSE(ec2);
+
+  context.poll();
+  EXPECT_TRUE(context.stopped());
+
+  ASSERT_TRUE(ec1);
+  EXPECT_EQ(boost::asio::error::operation_aborted, *ec1);
+  ASSERT_TRUE(ec2);
+  EXPECT_EQ(boost::asio::error::operation_aborted, *ec2);
+
+  EXPECT_EQ(0u, counters(client_id::admin)->get(queue_counters::l_qlen));
+  EXPECT_EQ(0u, counters(client_id::admin)->get(queue_counters::l_res));
+  EXPECT_EQ(0u, counters(client_id::admin)->get(queue_counters::l_prio));
+  EXPECT_EQ(0u, counters(client_id::admin)->get(queue_counters::l_limit));
+  EXPECT_EQ(1u, counters(client_id::admin)->get(queue_counters::l_cancel));
+
+  EXPECT_EQ(0u, counters(client_id::auth)->get(queue_counters::l_qlen));
+  EXPECT_EQ(0u, counters(client_id::auth)->get(queue_counters::l_res));
+  EXPECT_EQ(0u, counters(client_id::auth)->get(queue_counters::l_prio));
+  EXPECT_EQ(0u, counters(client_id::auth)->get(queue_counters::l_limit));
+  EXPECT_EQ(1u, counters(client_id::auth)->get(queue_counters::l_cancel));
+}
+
+// return a lambda from capture() that's bound to run on the given executor
+template <typename Executor>
+auto capture(const Executor& ex, std::optional<error_code>& opt_ec,
+             std::optional<PhaseType>& opt_res)
+{
+  return boost::asio::bind_executor(ex, capture(opt_ec, opt_res));
+}
+
+TEST(Queue, CrossExecutorRequest)
+{
+  boost::asio::io_context queue_context;
+  ClientCounters counters(g_ceph_context);
+  AsyncScheduler queue(g_ceph_context, queue_context, std::ref(counters), nullptr,
+                  [] (client_id client) -> ClientInfo* {
+      static ClientInfo info{0, 1, 1};
+      return &info;
+    });
+
+  // create a separate execution context to use for all callbacks to test that
+  // pending requests maintain executor work guards on both executors
+  boost::asio::io_context callback_context;
+  auto ex2 = callback_context.get_executor();
+
+  std::optional<error_code> ec1, ec2;
+  std::optional<PhaseType> p1, p2;
+
+  auto now = get_time();
+  queue.async_request(client_id::admin, {}, now, 1, capture(ex2, ec1, p1));
+  queue.async_request(client_id::auth, {}, now, 1, capture(ex2, ec2, p2));
+
+  EXPECT_EQ(1u, counters(client_id::admin)->get(queue_counters::l_qlen));
+  EXPECT_EQ(1u, counters(client_id::auth)->get(queue_counters::l_qlen));
+
+  callback_context.poll();
+  // maintains work on callback executor while in queue
+  EXPECT_FALSE(callback_context.stopped());
+
+  EXPECT_FALSE(ec1);
+  EXPECT_FALSE(ec2);
+
+  queue_context.poll();
+  EXPECT_TRUE(queue_context.stopped());
+
+  EXPECT_FALSE(ec1); // no callbacks until callback executor runs
+  EXPECT_FALSE(ec2);
+
+  callback_context.poll();
+  EXPECT_TRUE(callback_context.stopped());
+
+  ASSERT_TRUE(ec1);
+  EXPECT_EQ(boost::system::errc::success, *ec1);
+  ASSERT_TRUE(p1);
+  EXPECT_EQ(PhaseType::priority, *p1);
+
+  ASSERT_TRUE(ec2);
+  EXPECT_EQ(boost::system::errc::success, *ec2);
+  ASSERT_TRUE(p2);
+  EXPECT_EQ(PhaseType::priority, *p2);
+}
+
+TEST(Queue, SpawnAsyncRequest)
+{
+  boost::asio::io_context context;
+
+  boost::asio::spawn(context, [&] (boost::asio::yield_context yield) {
+    ClientCounters counters(g_ceph_context);
+    AsyncScheduler queue(g_ceph_context, context, std::ref(counters), nullptr,
+                    [] (client_id client) -> ClientInfo* {
+        static ClientInfo clients[] = {
+          {1, 1, 1}, // admin: satisfy by reservation
+          {0, 1, 1}, // auth: satisfy by priority
+        };
+        return &clients[static_cast<size_t>(client)];
+      });
+
+    error_code ec1, ec2;
+    auto p1 = queue.async_request(client_id::admin, {}, get_time(), 1, yield[ec1]);
+    EXPECT_EQ(boost::system::errc::success, ec1);
+    EXPECT_EQ(PhaseType::reservation, p1);
+
+    auto p2 = queue.async_request(client_id::auth, {}, get_time(), 1, yield[ec2]);
+    EXPECT_EQ(boost::system::errc::success, ec2);
+    EXPECT_EQ(PhaseType::priority, p2);
+  });
+
+  context.poll();
+  EXPECT_TRUE(context.stopped());
+}
+
+#endif
+
+} // namespace rgw::dmclock
diff --git a/src/test/rgw/test_rgw_iam_policy.cc b/src/test/rgw/test_rgw_iam_policy.cc
new file mode 100644
index 00000000..b12ab7fd
--- /dev/null
+++ b/src/test/rgw/test_rgw_iam_policy.cc
@@ -0,0 +1,1284 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2015 Red Hat
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#include <string>
+
+#include <boost/intrusive_ptr.hpp>
+#include <boost/optional.hpp>
+
+#include <gtest/gtest.h>
+
+#include "include/stringify.h"
+#include "common/code_environment.h"
+#include "common/ceph_context.h"
+#include "global/global_init.h"
+#include "rgw/rgw_auth.h"
+#include "rgw/rgw_iam_policy.h"
+#include "rgw/rgw_op.h"
+
+
+using std::string;
+using std::vector;
+
+using boost::container::flat_set;
+using boost::intrusive_ptr;
+using boost::make_optional;
+using boost::none;
+
+using rgw::auth::Identity;
+using rgw::auth::Principal;
+
+using rgw::ARN;
+using rgw::IAM::Effect;
+using rgw::IAM::Environment;
+using rgw::Partition;
+using rgw::IAM::Policy;
+using rgw::IAM::s3All;
+using rgw::IAM::s3Count;
+using rgw::IAM::s3GetAccelerateConfiguration;
+using rgw::IAM::s3GetBucketAcl;
+using rgw::IAM::s3GetBucketCORS;
+using rgw::IAM::s3GetBucketLocation;
+using rgw::IAM::s3GetBucketLogging;
+using rgw::IAM::s3GetBucketNotification;
+using rgw::IAM::s3GetBucketPolicy;
+using rgw::IAM::s3GetBucketRequestPayment;
+using rgw::IAM::s3GetBucketTagging;
+using rgw::IAM::s3GetBucketVersioning;
+using rgw::IAM::s3GetBucketWebsite;
+using rgw::IAM::s3GetLifecycleConfiguration;
+using rgw::IAM::s3GetObject;
+using rgw::IAM::s3GetObjectAcl;
+using rgw::IAM::s3GetObjectVersionAcl;
+using rgw::IAM::s3GetObjectTorrent;
+using rgw::IAM::s3GetObjectTagging;
+using rgw::IAM::s3GetObjectVersion;
+using rgw::IAM::s3GetObjectVersionTagging;
+using rgw::IAM::s3GetObjectVersionTorrent;
+using rgw::IAM::s3GetReplicationConfiguration;
+using rgw::IAM::s3ListAllMyBuckets;
+using rgw::IAM::s3ListBucket;
+using rgw::IAM::s3ListBucket;
+using rgw::IAM::s3ListBucketMultipartUploads;
+using rgw::IAM::s3ListBucketVersions;
+using rgw::IAM::s3ListMultipartUploadParts;
+using rgw::IAM::None;
+using rgw::IAM::s3PutBucketAcl;
+using rgw::IAM::s3PutBucketPolicy;
+using rgw::Service;
+using rgw::IAM::s3GetBucketObjectLockConfiguration;
+using rgw::IAM::s3GetObjectRetention;
+using rgw::IAM::s3GetObjectLegalHold;
+using rgw::IAM::TokenID;
+using rgw::IAM::Version;
+using rgw::IAM::Action_t;
+using rgw::IAM::NotAction_t;
+using rgw::IAM::iamCreateRole;
+using rgw::IAM::iamDeleteRole;
+using rgw::IAM::iamAll;
+using rgw::IAM::stsAll;
+
+class FakeIdentity : public Identity {
+  const Principal id;
+public:
+
+  explicit FakeIdentity(Principal&& id) : id(std::move(id)) {}
+  uint32_t get_perms_from_aclspec(const DoutPrefixProvider* dpp, const aclspec_t& aclspec) const override {
+    ceph_abort();
+    return 0;
+  };
+
+  bool is_admin_of(const rgw_user& uid) const override {
+    ceph_abort();
+    return false;
+  }
+
+  bool is_owner_of(const rgw_user& uid) const override {
+    ceph_abort();
+    return false;
+  }
+
+  virtual uint32_t get_perm_mask() const override {
+    ceph_abort();
+    return 0;
+  }
+
+  uint32_t get_identity_type() const override {
+    abort();
+    return 0;
+  }
+
+  string get_acct_name() const override {
+    abort();
+    return 0;
+  }
+
+  string get_subuser() const override {
+    abort();
+    return 0;
+  }
+
+  void to_str(std::ostream& out) const override {
+    out << id;
+  }
+
+  bool is_identity(const flat_set<Principal>& ids) const override {
+    if (id.is_wildcard() && (!ids.empty())) {
+      return true;
+    }
+    return ids.find(id) != ids.end() || ids.find(Principal::wildcard()) != ids.end();
+  }
+};
+
+class PolicyTest : public ::testing::Test {
+protected:
+  intrusive_ptr<CephContext> cct;
+  static const string arbitrary_tenant;
+  static string example1;
+  static string example2;
+  static string example3;
+  static string example4;
+  static string example5;
+  static string example6;
+  static string example7;
+public:
+  PolicyTest() {
+    cct = new CephContext(CEPH_ENTITY_TYPE_CLIENT);
+  }
+};
+
+TEST_F(PolicyTest, Parse1) {
+  boost::optional<Policy> p;
+
+  ASSERT_NO_THROW(p = Policy(cct.get(), arbitrary_tenant,
+			     bufferlist::static_from_string(example1)));
+  ASSERT_TRUE(p);
+
+  EXPECT_EQ(p->text, example1);
+  EXPECT_EQ(p->version, Version::v2012_10_17);
+  EXPECT_FALSE(p->id);
+  EXPECT_FALSE(p->statements[0].sid);
+  EXPECT_FALSE(p->statements.empty());
+  EXPECT_EQ(p->statements.size(), 1U);
+  EXPECT_TRUE(p->statements[0].princ.empty());
+  EXPECT_TRUE(p->statements[0].noprinc.empty());
+  EXPECT_EQ(p->statements[0].effect, Effect::Allow);
+  Action_t act;
+  act[s3ListBucket] = 1;
+  EXPECT_EQ(p->statements[0].action, act);
+  EXPECT_EQ(p->statements[0].notaction, None);
+  ASSERT_FALSE(p->statements[0].resource.empty());
+  ASSERT_EQ(p->statements[0].resource.size(), 1U);
+  EXPECT_EQ(p->statements[0].resource.begin()->partition, Partition::aws);
+  EXPECT_EQ(p->statements[0].resource.begin()->service, Service::s3);
+  EXPECT_TRUE(p->statements[0].resource.begin()->region.empty());
+  EXPECT_EQ(p->statements[0].resource.begin()->account, arbitrary_tenant);
+  EXPECT_EQ(p->statements[0].resource.begin()->resource, "example_bucket");
+  EXPECT_TRUE(p->statements[0].notresource.empty());
+  EXPECT_TRUE(p->statements[0].conditions.empty());
+}
+
+TEST_F(PolicyTest, Eval1) {
+  auto p  = Policy(cct.get(), arbitrary_tenant,
+		   bufferlist::static_from_string(example1));
+  Environment e;
+
+  EXPECT_EQ(p.eval(e, none, s3ListBucket,
+		   ARN(Partition::aws, Service::s3,
+		       "", arbitrary_tenant, "example_bucket")),
+	    Effect::Allow);
+
+  EXPECT_EQ(p.eval(e, none, s3PutBucketAcl,
+		   ARN(Partition::aws, Service::s3,
+		       "", arbitrary_tenant, "example_bucket")),
+	    Effect::Pass);
+
+  EXPECT_EQ(p.eval(e, none, s3ListBucket,
+		   ARN(Partition::aws, Service::s3,
+		       "", arbitrary_tenant, "erroneous_bucket")),
+	    Effect::Pass);
+
+}
+
+TEST_F(PolicyTest, Parse2) {
+  boost::optional<Policy> p;
+
+  ASSERT_NO_THROW(p = Policy(cct.get(), arbitrary_tenant,
+			     bufferlist::static_from_string(example2)));
+  ASSERT_TRUE(p);
+
+  EXPECT_EQ(p->text, example2);
+  EXPECT_EQ(p->version, Version::v2012_10_17);
+  EXPECT_EQ(*p->id, "S3-Account-Permissions");
+  ASSERT_FALSE(p->statements.empty());
+  EXPECT_EQ(p->statements.size(), 1U);
+  EXPECT_EQ(*p->statements[0].sid, "1");
+  EXPECT_FALSE(p->statements[0].princ.empty());
+  EXPECT_EQ(p->statements[0].princ.size(), 1U);
+  EXPECT_EQ(*p->statements[0].princ.begin(),
+	    Principal::tenant("ACCOUNT-ID-WITHOUT-HYPHENS"));
+  EXPECT_TRUE(p->statements[0].noprinc.empty());
+  EXPECT_EQ(p->statements[0].effect, Effect::Allow);
+  Action_t act;
+  for (auto i = 0ULL; i < s3Count; i++)
+    act[i] = 1;
+  act[s3All] = 1;
+  EXPECT_EQ(p->statements[0].action, act);
+  EXPECT_EQ(p->statements[0].notaction, None);
+  ASSERT_FALSE(p->statements[0].resource.empty());
+  ASSERT_EQ(p->statements[0].resource.size(), 2U);
+  EXPECT_EQ(p->statements[0].resource.begin()->partition, Partition::aws);
+  EXPECT_EQ(p->statements[0].resource.begin()->service, Service::s3);
+  EXPECT_TRUE(p->statements[0].resource.begin()->region.empty());
+  EXPECT_EQ(p->statements[0].resource.begin()->account, arbitrary_tenant);
+  EXPECT_EQ(p->statements[0].resource.begin()->resource, "mybucket");
+  EXPECT_EQ((p->statements[0].resource.begin() + 1)->partition,
+	    Partition::aws);
+  EXPECT_EQ((p->statements[0].resource.begin() + 1)->service,
+	    Service::s3);
+  EXPECT_TRUE((p->statements[0].resource.begin() + 1)->region.empty());
+  EXPECT_EQ((p->statements[0].resource.begin() + 1)->account,
+	    arbitrary_tenant);
+  EXPECT_EQ((p->statements[0].resource.begin() + 1)->resource, "mybucket/*");
+  EXPECT_TRUE(p->statements[0].notresource.empty());
+  EXPECT_TRUE(p->statements[0].conditions.empty());
+}
+
+TEST_F(PolicyTest, Eval2) {
+  auto p  = Policy(cct.get(), arbitrary_tenant,
+		   bufferlist::static_from_string(example2));
+  Environment e;
+
+  auto trueacct = FakeIdentity(
+    Principal::tenant("ACCOUNT-ID-WITHOUT-HYPHENS"));
+
+  auto notacct = FakeIdentity(
+    Principal::tenant("some-other-account"));
+  for (auto i = 0ULL; i < s3Count; ++i) {
+    EXPECT_EQ(p.eval(e, trueacct, i,
+		     ARN(Partition::aws, Service::s3,
+			 "", arbitrary_tenant, "mybucket")),
+	      Effect::Allow);
+    EXPECT_EQ(p.eval(e, trueacct, i,
+		     ARN(Partition::aws, Service::s3,
+			 "", arbitrary_tenant, "mybucket/myobject")),
+	      Effect::Allow);
+
+    EXPECT_EQ(p.eval(e, notacct, i,
+		     ARN(Partition::aws, Service::s3,
+			 "", arbitrary_tenant, "mybucket")),
+	      Effect::Pass);
+    EXPECT_EQ(p.eval(e, notacct, i,
+		     ARN(Partition::aws, Service::s3,
+			 "", arbitrary_tenant, "mybucket/myobject")),
+	      Effect::Pass);
+
+    EXPECT_EQ(p.eval(e, trueacct, i,
+		     ARN(Partition::aws, Service::s3,
+			 "", arbitrary_tenant, "notyourbucket")),
+	      Effect::Pass);
+    EXPECT_EQ(p.eval(e, trueacct, i,
+		     ARN(Partition::aws, Service::s3,
+			 "", arbitrary_tenant, "notyourbucket/notyourobject")),
+	      Effect::Pass);
+
+  }
+}
+
+TEST_F(PolicyTest, Parse3) {
+  boost::optional<Policy> p;
+
+  ASSERT_NO_THROW(p = Policy(cct.get(), arbitrary_tenant,
+			     bufferlist::static_from_string(example3)));
+  ASSERT_TRUE(p);
+
+  EXPECT_EQ(p->text, example3);
+  EXPECT_EQ(p->version, Version::v2012_10_17);
+  EXPECT_FALSE(p->id);
+  ASSERT_FALSE(p->statements.empty());
+  EXPECT_EQ(p->statements.size(), 3U);
+
+  EXPECT_EQ(*p->statements[0].sid, "FirstStatement");
+  EXPECT_TRUE(p->statements[0].princ.empty());
+  EXPECT_TRUE(p->statements[0].noprinc.empty());
+  EXPECT_EQ(p->statements[0].effect, Effect::Allow);
+  Action_t act;
+  act[s3PutBucketPolicy] = 1;
+  EXPECT_EQ(p->statements[0].action, act);
+  EXPECT_EQ(p->statements[0].notaction, None);
+  ASSERT_FALSE(p->statements[0].resource.empty());
+  ASSERT_EQ(p->statements[0].resource.size(), 1U);
+  EXPECT_EQ(p->statements[0].resource.begin()->partition, Partition::wildcard);
+  EXPECT_EQ(p->statements[0].resource.begin()->service, Service::wildcard);
+  EXPECT_EQ(p->statements[0].resource.begin()->region, "*");
+  EXPECT_EQ(p->statements[0].resource.begin()->account, arbitrary_tenant);
+  EXPECT_EQ(p->statements[0].resource.begin()->resource, "*");
+  EXPECT_TRUE(p->statements[0].notresource.empty());
+  EXPECT_TRUE(p->statements[0].conditions.empty());
+
+  EXPECT_EQ(*p->statements[1].sid, "SecondStatement");
+  EXPECT_TRUE(p->statements[1].princ.empty());
+  EXPECT_TRUE(p->statements[1].noprinc.empty());
+  EXPECT_EQ(p->statements[1].effect, Effect::Allow);
+  Action_t act1;
+  act1[s3ListAllMyBuckets] = 1;
+  EXPECT_EQ(p->statements[1].action, act1);
+  EXPECT_EQ(p->statements[1].notaction, None);
+  ASSERT_FALSE(p->statements[1].resource.empty());
+  ASSERT_EQ(p->statements[1].resource.size(), 1U);
+  EXPECT_EQ(p->statements[1].resource.begin()->partition, Partition::wildcard);
+  EXPECT_EQ(p->statements[1].resource.begin()->service, Service::wildcard);
+  EXPECT_EQ(p->statements[1].resource.begin()->region, "*");
+  EXPECT_EQ(p->statements[1].resource.begin()->account, arbitrary_tenant);
+  EXPECT_EQ(p->statements[1].resource.begin()->resource, "*");
+  EXPECT_TRUE(p->statements[1].notresource.empty());
+  EXPECT_TRUE(p->statements[1].conditions.empty());
+
+  EXPECT_EQ(*p->statements[2].sid, "ThirdStatement");
+  EXPECT_TRUE(p->statements[2].princ.empty());
+  EXPECT_TRUE(p->statements[2].noprinc.empty());
+  EXPECT_EQ(p->statements[2].effect, Effect::Allow);
+  Action_t act2;
+  act2[s3ListMultipartUploadParts] = 1;
+  act2[s3ListBucket] = 1;
+  act2[s3ListBucketVersions] = 1;
+  act2[s3ListAllMyBuckets] = 1;
+  act2[s3ListBucketMultipartUploads] = 1;
+  act2[s3GetObject] = 1;
+  act2[s3GetObjectVersion] = 1;
+  act2[s3GetObjectAcl] = 1;
+  act2[s3GetObjectVersionAcl] = 1;
+  act2[s3GetObjectTorrent] = 1;
+  act2[s3GetObjectVersionTorrent] = 1;
+  act2[s3GetAccelerateConfiguration] = 1;
+  act2[s3GetBucketAcl] = 1;
+  act2[s3GetBucketCORS] = 1;
+  act2[s3GetBucketVersioning] = 1;
+  act2[s3GetBucketRequestPayment] = 1;
+  act2[s3GetBucketLocation] = 1;
+  act2[s3GetBucketPolicy] = 1;
+  act2[s3GetBucketNotification] = 1;
+  act2[s3GetBucketLogging] = 1;
+  act2[s3GetBucketTagging] = 1;
+  act2[s3GetBucketWebsite] = 1;
+  act2[s3GetLifecycleConfiguration] = 1;
+  act2[s3GetReplicationConfiguration] = 1;
+  act2[s3GetObjectTagging] = 1;
+  act2[s3GetObjectVersionTagging] = 1;
+  act2[s3GetBucketObjectLockConfiguration] = 1;
+  act2[s3GetObjectRetention] = 1;
+  act2[s3GetObjectLegalHold] = 1;
+
+  EXPECT_EQ(p->statements[2].action, act2);
+  EXPECT_EQ(p->statements[2].notaction, None);
+  ASSERT_FALSE(p->statements[2].resource.empty());
+  ASSERT_EQ(p->statements[2].resource.size(), 2U);
+  EXPECT_EQ(p->statements[2].resource.begin()->partition, Partition::aws);
+  EXPECT_EQ(p->statements[2].resource.begin()->service, Service::s3);
+  EXPECT_TRUE(p->statements[2].resource.begin()->region.empty());
+  EXPECT_EQ(p->statements[2].resource.begin()->account, arbitrary_tenant);
+  EXPECT_EQ(p->statements[2].resource.begin()->resource, "confidential-data");
+  EXPECT_EQ((p->statements[2].resource.begin() + 1)->partition,
+	    Partition::aws);
+  EXPECT_EQ((p->statements[2].resource.begin() + 1)->service, Service::s3);
+  EXPECT_TRUE((p->statements[2].resource.begin() + 1)->region.empty());
+  EXPECT_EQ((p->statements[2].resource.begin() + 1)->account,
+	    arbitrary_tenant);
+  EXPECT_EQ((p->statements[2].resource.begin() + 1)->resource,
+	    "confidential-data/*");
+  EXPECT_TRUE(p->statements[2].notresource.empty());
+  ASSERT_FALSE(p->statements[2].conditions.empty());
+  ASSERT_EQ(p->statements[2].conditions.size(), 1U);
+  EXPECT_EQ(p->statements[2].conditions[0].op, TokenID::Bool);
+  EXPECT_EQ(p->statements[2].conditions[0].key, "aws:MultiFactorAuthPresent");
+  EXPECT_FALSE(p->statements[2].conditions[0].ifexists);
+  ASSERT_FALSE(p->statements[2].conditions[0].vals.empty());
+  EXPECT_EQ(p->statements[2].conditions[0].vals.size(), 1U);
+  EXPECT_EQ(p->statements[2].conditions[0].vals[0], "true");
+}
+
+TEST_F(PolicyTest, Eval3) {
+  auto p  = Policy(cct.get(), arbitrary_tenant,
+		   bufferlist::static_from_string(example3));
+  Environment em;
+  Environment tr = { { "aws:MultiFactorAuthPresent", "true" } };
+  Environment fa = { { "aws:MultiFactorAuthPresent", "false" } };
+
+  Action_t s3allow;
+  s3allow[s3ListMultipartUploadParts] = 1;
+  s3allow[s3ListBucket] = 1;
+  s3allow[s3ListBucketVersions] = 1;
+  s3allow[s3ListAllMyBuckets] = 1;
+  s3allow[s3ListBucketMultipartUploads] = 1;
+  s3allow[s3GetObject] = 1;
+  s3allow[s3GetObjectVersion] = 1;
+  s3allow[s3GetObjectAcl] = 1;
+  s3allow[s3GetObjectVersionAcl] = 1;
+  s3allow[s3GetObjectTorrent] = 1;
+  s3allow[s3GetObjectVersionTorrent] = 1;
+  s3allow[s3GetAccelerateConfiguration] = 1;
+  s3allow[s3GetBucketAcl] = 1;
+  s3allow[s3GetBucketCORS] = 1;
+  s3allow[s3GetBucketVersioning] = 1;
+  s3allow[s3GetBucketRequestPayment] = 1;
+  s3allow[s3GetBucketLocation] = 1;
+  s3allow[s3GetBucketPolicy] = 1;
+  s3allow[s3GetBucketNotification] = 1;
+  s3allow[s3GetBucketLogging] = 1;
+  s3allow[s3GetBucketTagging] = 1;
+  s3allow[s3GetBucketWebsite] = 1;
+  s3allow[s3GetLifecycleConfiguration] = 1;
+  s3allow[s3GetReplicationConfiguration] = 1;
+  s3allow[s3GetObjectTagging] = 1;
+  s3allow[s3GetObjectVersionTagging] = 1;
+  s3allow[s3GetBucketObjectLockConfiguration] = 1;
+  s3allow[s3GetObjectRetention] = 1;
+  s3allow[s3GetObjectLegalHold] = 1;
+
+  EXPECT_EQ(p.eval(em, none, s3PutBucketPolicy,
+		   ARN(Partition::aws, Service::s3,
+		       "", arbitrary_tenant, "mybucket")),
+	    Effect::Allow);
+
+  EXPECT_EQ(p.eval(em, none, s3PutBucketPolicy,
+		   ARN(Partition::aws, Service::s3,
+		       "", arbitrary_tenant, "mybucket")),
+	    Effect::Allow);
+
+
+  for (auto op = 0ULL; op < s3Count; ++op) {
+    if ((op == s3ListAllMyBuckets) || (op == s3PutBucketPolicy)) {
+      continue;
+    }
+    EXPECT_EQ(p.eval(em, none, op,
+		     ARN(Partition::aws, Service::s3,
+			 "", arbitrary_tenant, "confidential-data")),
+	      Effect::Pass);
+    EXPECT_EQ(p.eval(tr, none, op,
+		     ARN(Partition::aws, Service::s3,
+			 "", arbitrary_tenant, "confidential-data")),
+	      s3allow[op] ? Effect::Allow : Effect::Pass);
+    EXPECT_EQ(p.eval(fa, none, op,
+		     ARN(Partition::aws, Service::s3,
+			 "", arbitrary_tenant, "confidential-data")),
+	      Effect::Pass);
+
+    EXPECT_EQ(p.eval(em, none, op,
+		     ARN(Partition::aws, Service::s3,
+			 "", arbitrary_tenant, "confidential-data/moo")),
+	      Effect::Pass);
+    EXPECT_EQ(p.eval(tr, none, op,
+		     ARN(Partition::aws, Service::s3,
+			 "", arbitrary_tenant, "confidential-data/moo")),
+	      s3allow[op] ? Effect::Allow : Effect::Pass);
+    EXPECT_EQ(p.eval(fa, none, op,
+		     ARN(Partition::aws, Service::s3,
+			 "", arbitrary_tenant, "confidential-data/moo")),
+	      Effect::Pass);
+
+    EXPECT_EQ(p.eval(em, none, op,
+		     ARN(Partition::aws, Service::s3,
+			 "", arbitrary_tenant, "really-confidential-data")),
+	      Effect::Pass);
+    EXPECT_EQ(p.eval(tr, none, op,
+		     ARN(Partition::aws, Service::s3,
+			 "", arbitrary_tenant, "really-confidential-data")),
+	      Effect::Pass);
+    EXPECT_EQ(p.eval(fa, none, op,
+		     ARN(Partition::aws, Service::s3,
+			 "", arbitrary_tenant, "really-confidential-data")),
+	      Effect::Pass);
+
+    EXPECT_EQ(p.eval(em, none, op,
+		     ARN(Partition::aws, Service::s3,
+			 "", arbitrary_tenant,
+			 "really-confidential-data/moo")), Effect::Pass);
+    EXPECT_EQ(p.eval(tr, none, op,
+		     ARN(Partition::aws, Service::s3,
+			 "", arbitrary_tenant,
+			 "really-confidential-data/moo")), Effect::Pass);
+    EXPECT_EQ(p.eval(fa, none, op,
+		     ARN(Partition::aws, Service::s3,
+			 "", arbitrary_tenant,
+			 "really-confidential-data/moo")), Effect::Pass);
+
+  }
+}
+
+TEST_F(PolicyTest, Parse4) {
+  boost::optional<Policy> p;
+
+  ASSERT_NO_THROW(p = Policy(cct.get(), arbitrary_tenant,
+			     bufferlist::static_from_string(example4)));
+  ASSERT_TRUE(p);
+
+  EXPECT_EQ(p->text, example4);
+  EXPECT_EQ(p->version, Version::v2012_10_17);
+  EXPECT_FALSE(p->id);
+  EXPECT_FALSE(p->statements[0].sid);
+  EXPECT_FALSE(p->statements.empty());
+  EXPECT_EQ(p->statements.size(), 1U);
+  EXPECT_TRUE(p->statements[0].princ.empty());
+  EXPECT_TRUE(p->statements[0].noprinc.empty());
+  EXPECT_EQ(p->statements[0].effect, Effect::Allow);
+  Action_t act;
+  act[iamCreateRole] = 1;
+  EXPECT_EQ(p->statements[0].action, act);
+  EXPECT_EQ(p->statements[0].notaction, None);
+  ASSERT_FALSE(p->statements[0].resource.empty());
+  ASSERT_EQ(p->statements[0].resource.size(), 1U);
+  EXPECT_EQ(p->statements[0].resource.begin()->partition, Partition::wildcard);
+  EXPECT_EQ(p->statements[0].resource.begin()->service, Service::wildcard);
+  EXPECT_EQ(p->statements[0].resource.begin()->region, "*");
+  EXPECT_EQ(p->statements[0].resource.begin()->account, arbitrary_tenant);
+  EXPECT_EQ(p->statements[0].resource.begin()->resource, "*");
+  EXPECT_TRUE(p->statements[0].notresource.empty());
+  EXPECT_TRUE(p->statements[0].conditions.empty());
+}
+
+TEST_F(PolicyTest, Eval4) {
+  auto p  = Policy(cct.get(), arbitrary_tenant,
+		   bufferlist::static_from_string(example4));
+  Environment e;
+
+  EXPECT_EQ(p.eval(e, none, iamCreateRole,
+		   ARN(Partition::aws, Service::iam,
+		       "", arbitrary_tenant, "role/example_role")),
+	    Effect::Allow);
+
+  EXPECT_EQ(p.eval(e, none, iamDeleteRole,
+		   ARN(Partition::aws, Service::iam,
+		       "", arbitrary_tenant, "role/example_role")),
+	    Effect::Pass);
+}
+
+TEST_F(PolicyTest, Parse5) {
+  boost::optional<Policy> p;
+
+  ASSERT_NO_THROW(p = Policy(cct.get(), arbitrary_tenant,
+			     bufferlist::static_from_string(example5)));
+  ASSERT_TRUE(p);
+  EXPECT_EQ(p->text, example5);
+  EXPECT_EQ(p->version, Version::v2012_10_17);
+  EXPECT_FALSE(p->id);
+  EXPECT_FALSE(p->statements[0].sid);
+  EXPECT_FALSE(p->statements.empty());
+  EXPECT_EQ(p->statements.size(), 1U);
+  EXPECT_TRUE(p->statements[0].princ.empty());
+  EXPECT_TRUE(p->statements[0].noprinc.empty());
+  EXPECT_EQ(p->statements[0].effect, Effect::Allow);
+  Action_t act;
+  for (auto i = s3All+1; i <= iamAll; i++)
+    act[i] = 1;
+  EXPECT_EQ(p->statements[0].action, act);
+  EXPECT_EQ(p->statements[0].notaction, None);
+  ASSERT_FALSE(p->statements[0].resource.empty());
+  ASSERT_EQ(p->statements[0].resource.size(), 1U);
+  EXPECT_EQ(p->statements[0].resource.begin()->partition, Partition::aws);
+  EXPECT_EQ(p->statements[0].resource.begin()->service, Service::iam);
+  EXPECT_EQ(p->statements[0].resource.begin()->region, "");
+  EXPECT_EQ(p->statements[0].resource.begin()->account, arbitrary_tenant);
+  EXPECT_EQ(p->statements[0].resource.begin()->resource, "role/example_role");
+  EXPECT_TRUE(p->statements[0].notresource.empty());
+  EXPECT_TRUE(p->statements[0].conditions.empty());
+}
+
+TEST_F(PolicyTest, Eval5) {
+  auto p  = Policy(cct.get(), arbitrary_tenant,
+		   bufferlist::static_from_string(example5));
+  Environment e;
+
+  EXPECT_EQ(p.eval(e, none, iamCreateRole,
+		   ARN(Partition::aws, Service::iam,
+		       "", arbitrary_tenant, "role/example_role")),
+	    Effect::Allow);
+
+  EXPECT_EQ(p.eval(e, none, s3ListBucket,
+		   ARN(Partition::aws, Service::iam,
+		       "", arbitrary_tenant, "role/example_role")),
+	    Effect::Pass);
+
+  EXPECT_EQ(p.eval(e, none, iamCreateRole,
+		   ARN(Partition::aws, Service::iam,
+		       "", "", "role/example_role")),
+	    Effect::Pass);
+}
+
+TEST_F(PolicyTest, Parse6) {
+  boost::optional<Policy> p;
+
+  ASSERT_NO_THROW(p = Policy(cct.get(), arbitrary_tenant,
+			     bufferlist::static_from_string(example6)));
+  ASSERT_TRUE(p);
+  EXPECT_EQ(p->text, example6);
+  EXPECT_EQ(p->version, Version::v2012_10_17);
+  EXPECT_FALSE(p->id);
+  EXPECT_FALSE(p->statements[0].sid);
+  EXPECT_FALSE(p->statements.empty());
+  EXPECT_EQ(p->statements.size(), 1U);
+  EXPECT_TRUE(p->statements[0].princ.empty());
+  EXPECT_TRUE(p->statements[0].noprinc.empty());
+  EXPECT_EQ(p->statements[0].effect, Effect::Allow);
+  Action_t act;
+  for (auto i = 0U; i <= stsAll; i++)
+    act[i] = 1;
+  EXPECT_EQ(p->statements[0].action, act);
+  EXPECT_EQ(p->statements[0].notaction, None);
+  ASSERT_FALSE(p->statements[0].resource.empty());
+  ASSERT_EQ(p->statements[0].resource.size(), 1U);
+  EXPECT_EQ(p->statements[0].resource.begin()->partition, Partition::aws);
+  EXPECT_EQ(p->statements[0].resource.begin()->service, Service::iam);
+  EXPECT_EQ(p->statements[0].resource.begin()->region, "");
+  EXPECT_EQ(p->statements[0].resource.begin()->account, arbitrary_tenant);
+  EXPECT_EQ(p->statements[0].resource.begin()->resource, "user/A");
+  EXPECT_TRUE(p->statements[0].notresource.empty());
+  EXPECT_TRUE(p->statements[0].conditions.empty());
+}
+
+TEST_F(PolicyTest, Eval6) {
+  auto p  = Policy(cct.get(), arbitrary_tenant,
+		   bufferlist::static_from_string(example6));
+  Environment e;
+
+  EXPECT_EQ(p.eval(e, none, iamCreateRole,
+		   ARN(Partition::aws, Service::iam,
+		       "", arbitrary_tenant, "user/A")),
+	    Effect::Allow);
+
+  EXPECT_EQ(p.eval(e, none, s3ListBucket,
+		   ARN(Partition::aws, Service::iam,
+		       "", arbitrary_tenant, "user/A")),
+	    Effect::Allow);
+}
+
+TEST_F(PolicyTest, Parse7) {
+  boost::optional<Policy> p;
+
+  ASSERT_NO_THROW(p = Policy(cct.get(), arbitrary_tenant,
+			     bufferlist::static_from_string(example7)));
+  ASSERT_TRUE(p);
+
+  EXPECT_EQ(p->text, example7);
+  EXPECT_EQ(p->version, Version::v2012_10_17);
+  ASSERT_FALSE(p->statements.empty());
+  EXPECT_EQ(p->statements.size(), 1U);
+  EXPECT_FALSE(p->statements[0].princ.empty());
+  EXPECT_EQ(p->statements[0].princ.size(), 1U);
+  EXPECT_TRUE(p->statements[0].noprinc.empty());
+  EXPECT_EQ(p->statements[0].effect, Effect::Allow);
+  Action_t act;
+  act[s3ListBucket] = 1;
+  EXPECT_EQ(p->statements[0].action, act);
+  EXPECT_EQ(p->statements[0].notaction, None);
+  ASSERT_FALSE(p->statements[0].resource.empty());
+  ASSERT_EQ(p->statements[0].resource.size(), 1U);
+  EXPECT_EQ(p->statements[0].resource.begin()->partition, Partition::aws);
+  EXPECT_EQ(p->statements[0].resource.begin()->service, Service::s3);
+  EXPECT_TRUE(p->statements[0].resource.begin()->region.empty());
+  EXPECT_EQ(p->statements[0].resource.begin()->account, arbitrary_tenant);
+  EXPECT_EQ(p->statements[0].resource.begin()->resource, "mybucket/*");
+  EXPECT_TRUE(p->statements[0].princ.begin()->is_user());
+  EXPECT_FALSE(p->statements[0].princ.begin()->is_wildcard());
+  EXPECT_EQ(p->statements[0].princ.begin()->get_tenant(), "");
+  EXPECT_EQ(p->statements[0].princ.begin()->get_id(), "A:subA");
+  EXPECT_TRUE(p->statements[0].notresource.empty());
+  EXPECT_TRUE(p->statements[0].conditions.empty());
+}
+
+TEST_F(PolicyTest, Eval7) {
+  auto p  = Policy(cct.get(), arbitrary_tenant,
+		   bufferlist::static_from_string(example7));
+  Environment e;
+
+  auto subacct = FakeIdentity(
+    Principal::user(std::move(""), "A:subA"));
+  auto parentacct = FakeIdentity(
+    Principal::user(std::move(""), "A"));
+  auto sub2acct = FakeIdentity(
+    Principal::user(std::move(""), "A:sub2A"));
+
+  EXPECT_EQ(p.eval(e, subacct, s3ListBucket,
+		   ARN(Partition::aws, Service::s3,
+		       "", arbitrary_tenant, "mybucket/*")),
+	    Effect::Allow);
+  
+  EXPECT_EQ(p.eval(e, parentacct, s3ListBucket,
+		   ARN(Partition::aws, Service::s3,
+		       "", arbitrary_tenant, "mybucket/*")),
+	    Effect::Pass);
+  
+  EXPECT_EQ(p.eval(e, sub2acct, s3ListBucket,
+		   ARN(Partition::aws, Service::s3,
+		       "", arbitrary_tenant, "mybucket/*")),
+	    Effect::Pass);
+}
+
+const string PolicyTest::arbitrary_tenant = "arbitrary_tenant";
+string PolicyTest::example1 = R"(
+{
+  "Version": "2012-10-17",
+  "Statement": {
+    "Effect": "Allow",
+    "Action": "s3:ListBucket",
+    "Resource": "arn:aws:s3:::example_bucket"
+  }
+}
+)";
+
+string PolicyTest::example2 = R"(
+{
+  "Version": "2012-10-17",
+  "Id": "S3-Account-Permissions",
+  "Statement": [{
+    "Sid": "1",
+    "Effect": "Allow",
+    "Principal": {"AWS": ["arn:aws:iam::ACCOUNT-ID-WITHOUT-HYPHENS:root"]},
+    "Action": "s3:*",
+    "Resource": [
+      "arn:aws:s3:::mybucket",
+      "arn:aws:s3:::mybucket/*"
+    ]
+  }]
+}
+)";
+
+string PolicyTest::example3 = R"(
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Sid": "FirstStatement",
+      "Effect": "Allow",
+      "Action": ["s3:PutBucketPolicy"],
+      "Resource": "*"
+    },
+    {
+      "Sid": "SecondStatement",
+      "Effect": "Allow",
+      "Action": "s3:ListAllMyBuckets",
+      "Resource": "*"
+    },
+    {
+      "Sid": "ThirdStatement",
+      "Effect": "Allow",
+      "Action": [
+	"s3:List*",
+	"s3:Get*"
+      ],
+      "Resource": [
+	"arn:aws:s3:::confidential-data",
+	"arn:aws:s3:::confidential-data/*"
+      ],
+      "Condition": {"Bool": {"aws:MultiFactorAuthPresent": "true"}}
+    }
+  ]
+}
+)";
+
+string PolicyTest::example4 = R"(
+{
+  "Version": "2012-10-17",
+  "Statement": {
+    "Effect": "Allow",
+    "Action": "iam:CreateRole",
+    "Resource": "*"
+  }
+}
+)";
+
+string PolicyTest::example5 = R"(
+{
+  "Version": "2012-10-17",
+  "Statement": {
+    "Effect": "Allow",
+    "Action": "iam:*",
+    "Resource": "arn:aws:iam:::role/example_role"
+  }
+}
+)";
+
+string PolicyTest::example6 = R"(
+{
+  "Version": "2012-10-17",
+  "Statement": {
+    "Effect": "Allow",
+    "Action": "*",
+    "Resource": "arn:aws:iam:::user/A"
+  }
+}
+)";
+
+string PolicyTest::example7 = R"(
+{
+  "Version": "2012-10-17",
+  "Statement": {
+    "Effect": "Allow",
+    "Principal": {"AWS": ["arn:aws:iam:::user/A:subA"]},
+    "Action": "s3:ListBucket",
+    "Resource": "arn:aws:s3:::mybucket/*"
+  }
+}
+)";
+class IPPolicyTest : public ::testing::Test {
+protected:
+  intrusive_ptr<CephContext> cct;
+  static const string arbitrary_tenant;
+  static string ip_address_allow_example;
+  static string ip_address_deny_example;
+  static string ip_address_full_example;
+  // 192.168.1.0/24
+  const rgw::IAM::MaskedIP allowedIPv4Range = { false, rgw::IAM::Address("11000000101010000000000100000000"), 24 };
+  // 192.168.1.1/32
+  const rgw::IAM::MaskedIP blacklistedIPv4 = { false, rgw::IAM::Address("11000000101010000000000100000001"), 32 };
+  // 2001:db8:85a3:0:0:8a2e:370:7334/128
+  const rgw::IAM::MaskedIP allowedIPv6 = { true, rgw::IAM::Address("00100000000000010000110110111000100001011010001100000000000000000000000000000000100010100010111000000011011100000111001100110100"), 128 };
+  // ::1
+  const rgw::IAM::MaskedIP blacklistedIPv6 = { true, rgw::IAM::Address(1), 128 };
+  // 2001:db8:85a3:0:0:8a2e:370:7330/124
+  const rgw::IAM::MaskedIP allowedIPv6Range = { true, rgw::IAM::Address("00100000000000010000110110111000100001011010001100000000000000000000000000000000100010100010111000000011011100000111001100110000"), 124 };
+public:
+  IPPolicyTest() {
+    cct = new CephContext(CEPH_ENTITY_TYPE_CLIENT);
+  }
+};
+const string IPPolicyTest::arbitrary_tenant = "arbitrary_tenant";
+
+TEST_F(IPPolicyTest, MaskedIPOperations) {
+  EXPECT_EQ(stringify(allowedIPv4Range), "192.168.1.0/24");
+  EXPECT_EQ(stringify(blacklistedIPv4), "192.168.1.1/32");
+  EXPECT_EQ(stringify(allowedIPv6), "2001:db8:85a3:0:0:8a2e:370:7334/128");
+  EXPECT_EQ(stringify(allowedIPv6Range), "2001:db8:85a3:0:0:8a2e:370:7330/124");
+  EXPECT_EQ(stringify(blacklistedIPv6), "0:0:0:0:0:0:0:1/128");
+  EXPECT_EQ(allowedIPv4Range, blacklistedIPv4);
+  EXPECT_EQ(allowedIPv6Range, allowedIPv6);
+}
+
+TEST_F(IPPolicyTest, asNetworkIPv4Range) {
+  auto actualIPv4Range = rgw::IAM::Condition::as_network("192.168.1.0/24");
+  ASSERT_TRUE(actualIPv4Range.is_initialized());
+  EXPECT_EQ(*actualIPv4Range, allowedIPv4Range);
+}
+
+TEST_F(IPPolicyTest, asNetworkIPv4) {
+  auto actualIPv4 = rgw::IAM::Condition::as_network("192.168.1.1");
+  ASSERT_TRUE(actualIPv4.is_initialized());
+  EXPECT_EQ(*actualIPv4, blacklistedIPv4);
+}
+
+TEST_F(IPPolicyTest, asNetworkIPv6Range) {
+  auto actualIPv6Range = rgw::IAM::Condition::as_network("2001:db8:85a3:0:0:8a2e:370:7330/124");
+  ASSERT_TRUE(actualIPv6Range.is_initialized());
+  EXPECT_EQ(*actualIPv6Range, allowedIPv6Range);
+}
+
+TEST_F(IPPolicyTest, asNetworkIPv6) {
+  auto actualIPv6 = rgw::IAM::Condition::as_network("2001:db8:85a3:0:0:8a2e:370:7334");
+  ASSERT_TRUE(actualIPv6.is_initialized());
+  EXPECT_EQ(*actualIPv6, allowedIPv6);
+}
+
+TEST_F(IPPolicyTest, asNetworkInvalid) {
+  EXPECT_FALSE(rgw::IAM::Condition::as_network(""));
+  EXPECT_FALSE(rgw::IAM::Condition::as_network("192.168.1.1/33"));
+  EXPECT_FALSE(rgw::IAM::Condition::as_network("2001:db8:85a3:0:0:8a2e:370:7334/129"));
+  EXPECT_FALSE(rgw::IAM::Condition::as_network("192.168.1.1:"));
+  EXPECT_FALSE(rgw::IAM::Condition::as_network("1.2.3.10000"));
+}
+
+TEST_F(IPPolicyTest, IPEnvironment) {
+  // Unfortunately RGWCivetWeb is too tightly tied to civetweb to test RGWCivetWeb::init_env.
+  RGWEnv rgw_env;
+  RGWUserInfo user;
+  RGWRados rgw_rados;
+  rgw_env.set("REMOTE_ADDR", "192.168.1.1");
+  rgw_env.set("HTTP_HOST", "1.2.3.4");
+  req_state rgw_req_state(cct.get(), &rgw_env, &user, 0);
+  rgw_build_iam_environment(&rgw_rados, &rgw_req_state);
+  auto ip = rgw_req_state.env.find("aws:SourceIp");
+  ASSERT_NE(ip, rgw_req_state.env.end());
+  EXPECT_EQ(ip->second, "192.168.1.1");
+
+  ASSERT_EQ(cct.get()->_conf.set_val("rgw_remote_addr_param", "SOME_VAR"), 0);
+  EXPECT_EQ(cct.get()->_conf->rgw_remote_addr_param, "SOME_VAR");
+  rgw_req_state.env.clear();
+  rgw_build_iam_environment(&rgw_rados, &rgw_req_state);
+  ip = rgw_req_state.env.find("aws:SourceIp");
+  EXPECT_EQ(ip, rgw_req_state.env.end());
+
+  rgw_env.set("SOME_VAR", "192.168.1.2");
+  rgw_req_state.env.clear();
+  rgw_build_iam_environment(&rgw_rados, &rgw_req_state);
+  ip = rgw_req_state.env.find("aws:SourceIp");
+  ASSERT_NE(ip, rgw_req_state.env.end());
+  EXPECT_EQ(ip->second, "192.168.1.2");
+
+  ASSERT_EQ(cct.get()->_conf.set_val("rgw_remote_addr_param", "HTTP_X_FORWARDED_FOR"), 0);
+  rgw_env.set("HTTP_X_FORWARDED_FOR", "192.168.1.3");
+  rgw_req_state.env.clear();
+  rgw_build_iam_environment(&rgw_rados, &rgw_req_state);
+  ip = rgw_req_state.env.find("aws:SourceIp");
+  ASSERT_NE(ip, rgw_req_state.env.end());
+  EXPECT_EQ(ip->second, "192.168.1.3");
+
+  rgw_env.set("HTTP_X_FORWARDED_FOR", "192.168.1.4, 4.3.2.1, 2001:db8:85a3:8d3:1319:8a2e:370:7348");
+  rgw_req_state.env.clear();
+  rgw_build_iam_environment(&rgw_rados, &rgw_req_state);
+  ip = rgw_req_state.env.find("aws:SourceIp");
+  ASSERT_NE(ip, rgw_req_state.env.end());
+  EXPECT_EQ(ip->second, "192.168.1.4");
+}
+
+TEST_F(IPPolicyTest, ParseIPAddress) {
+  boost::optional<Policy> p;
+
+  ASSERT_NO_THROW(p = Policy(cct.get(), arbitrary_tenant,
+			     bufferlist::static_from_string(ip_address_full_example)));
+  ASSERT_TRUE(p);
+
+  EXPECT_EQ(p->text, ip_address_full_example);
+  EXPECT_EQ(p->version, Version::v2012_10_17);
+  EXPECT_EQ(*p->id, "S3IPPolicyTest");
+  EXPECT_FALSE(p->statements.empty());
+  EXPECT_EQ(p->statements.size(), 1U);
+  EXPECT_EQ(*p->statements[0].sid, "IPAllow");
+  EXPECT_FALSE(p->statements[0].princ.empty());
+  EXPECT_EQ(p->statements[0].princ.size(), 1U);
+  EXPECT_EQ(*p->statements[0].princ.begin(),
+	    Principal::wildcard());
+  EXPECT_TRUE(p->statements[0].noprinc.empty());
+  EXPECT_EQ(p->statements[0].effect, Effect::Allow);
+  Action_t act;
+  act[s3ListBucket] = 1;
+  EXPECT_EQ(p->statements[0].action, act);
+  EXPECT_EQ(p->statements[0].notaction, None);
+  ASSERT_FALSE(p->statements[0].resource.empty());
+  ASSERT_EQ(p->statements[0].resource.size(), 2U);
+  EXPECT_EQ(p->statements[0].resource.begin()->partition, Partition::aws);
+  EXPECT_EQ(p->statements[0].resource.begin()->service, Service::s3);
+  EXPECT_TRUE(p->statements[0].resource.begin()->region.empty());
+  EXPECT_EQ(p->statements[0].resource.begin()->account, arbitrary_tenant);
+  EXPECT_EQ(p->statements[0].resource.begin()->resource, "example_bucket");
+  EXPECT_EQ((p->statements[0].resource.begin() + 1)->resource, "example_bucket/*");
+  EXPECT_TRUE(p->statements[0].notresource.empty());
+  ASSERT_FALSE(p->statements[0].conditions.empty());
+  ASSERT_EQ(p->statements[0].conditions.size(), 2U);
+  EXPECT_EQ(p->statements[0].conditions[0].op, TokenID::IpAddress);
+  EXPECT_EQ(p->statements[0].conditions[0].key, "aws:SourceIp");
+  ASSERT_FALSE(p->statements[0].conditions[0].vals.empty());
+  EXPECT_EQ(p->statements[0].conditions[0].vals.size(), 2U);
+  EXPECT_EQ(p->statements[0].conditions[0].vals[0], "192.168.1.0/24");
+  EXPECT_EQ(p->statements[0].conditions[0].vals[1], "::1");
+  boost::optional<rgw::IAM::MaskedIP> convertedIPv4 = rgw::IAM::Condition::as_network(p->statements[0].conditions[0].vals[0]);
+  EXPECT_TRUE(convertedIPv4.is_initialized());
+  if (convertedIPv4.is_initialized()) {
+    EXPECT_EQ(*convertedIPv4, allowedIPv4Range);
+  }
+
+  EXPECT_EQ(p->statements[0].conditions[1].op, TokenID::NotIpAddress);
+  EXPECT_EQ(p->statements[0].conditions[1].key, "aws:SourceIp");
+  ASSERT_FALSE(p->statements[0].conditions[1].vals.empty());
+  EXPECT_EQ(p->statements[0].conditions[1].vals.size(), 2U);
+  EXPECT_EQ(p->statements[0].conditions[1].vals[0], "192.168.1.1/32");
+  EXPECT_EQ(p->statements[0].conditions[1].vals[1], "2001:0db8:85a3:0000:0000:8a2e:0370:7334");
+  boost::optional<rgw::IAM::MaskedIP> convertedIPv6 = rgw::IAM::Condition::as_network(p->statements[0].conditions[1].vals[1]);
+  EXPECT_TRUE(convertedIPv6.is_initialized());
+  if (convertedIPv6.is_initialized()) {
+    EXPECT_EQ(*convertedIPv6, allowedIPv6);
+  }
+}
+
+TEST_F(IPPolicyTest, EvalIPAddress) {
+  auto allowp  = Policy(cct.get(), arbitrary_tenant,
+			bufferlist::static_from_string(ip_address_allow_example));
+  auto denyp  = Policy(cct.get(), arbitrary_tenant,
+		       bufferlist::static_from_string(ip_address_deny_example));
+  auto fullp  = Policy(cct.get(), arbitrary_tenant,
+		   bufferlist::static_from_string(ip_address_full_example));
+  Environment e;
+  Environment allowedIP, blacklistedIP, allowedIPv6, blacklistedIPv6;
+  allowedIP["aws:SourceIp"] = "192.168.1.2";
+  allowedIPv6["aws:SourceIp"] = "::1";
+  blacklistedIP["aws:SourceIp"] = "192.168.1.1";
+  blacklistedIPv6["aws:SourceIp"] = "2001:0db8:85a3:0000:0000:8a2e:0370:7334";
+
+  auto trueacct = FakeIdentity(
+    Principal::tenant("ACCOUNT-ID-WITHOUT-HYPHENS"));
+  // Without an IP address in the environment then evaluation will always pass
+  EXPECT_EQ(allowp.eval(e, trueacct, s3ListBucket,
+			ARN(Partition::aws, Service::s3,
+			    "", arbitrary_tenant, "example_bucket")),
+	    Effect::Pass);
+  EXPECT_EQ(fullp.eval(e, trueacct, s3ListBucket,
+		       ARN(Partition::aws, Service::s3,
+			   "", arbitrary_tenant, "example_bucket/myobject")),
+	    Effect::Pass);
+
+  EXPECT_EQ(allowp.eval(allowedIP, trueacct, s3ListBucket,
+			ARN(Partition::aws, Service::s3,
+			    "", arbitrary_tenant, "example_bucket")),
+	    Effect::Allow);
+  EXPECT_EQ(allowp.eval(blacklistedIPv6, trueacct, s3ListBucket,
+			ARN(Partition::aws, Service::s3,
+			    "", arbitrary_tenant, "example_bucket")),
+	    Effect::Pass);
+
+
+  EXPECT_EQ(denyp.eval(allowedIP, trueacct, s3ListBucket,
+		       ARN(Partition::aws, Service::s3,
+			   "", arbitrary_tenant, "example_bucket")),
+	    Effect::Deny);
+  EXPECT_EQ(denyp.eval(allowedIP, trueacct, s3ListBucket,
+		       ARN(Partition::aws, Service::s3,
+			   "", arbitrary_tenant, "example_bucket/myobject")),
+	    Effect::Deny);
+
+  EXPECT_EQ(denyp.eval(blacklistedIP, trueacct, s3ListBucket,
+		       ARN(Partition::aws, Service::s3,
+			   "", arbitrary_tenant, "example_bucket")),
+	    Effect::Pass);
+  EXPECT_EQ(denyp.eval(blacklistedIP, trueacct, s3ListBucket,
+		       ARN(Partition::aws, Service::s3,
+			   "", arbitrary_tenant, "example_bucket/myobject")),
+	    Effect::Pass);
+
+  EXPECT_EQ(denyp.eval(blacklistedIPv6, trueacct, s3ListBucket,
+		       ARN(Partition::aws, Service::s3,
+			   "", arbitrary_tenant, "example_bucket")),
+	    Effect::Pass);
+  EXPECT_EQ(denyp.eval(blacklistedIPv6, trueacct, s3ListBucket,
+		       ARN(Partition::aws, Service::s3,
+			   "", arbitrary_tenant, "example_bucket/myobject")),
+	    Effect::Pass);
+  EXPECT_EQ(denyp.eval(allowedIPv6, trueacct, s3ListBucket,
+		       ARN(Partition::aws, Service::s3,
+			   "", arbitrary_tenant, "example_bucket")),
+	    Effect::Deny);
+  EXPECT_EQ(denyp.eval(allowedIPv6, trueacct, s3ListBucket,
+		       ARN(Partition::aws, Service::s3,
+			   "", arbitrary_tenant, "example_bucket/myobject")),
+	    Effect::Deny);
+
+  EXPECT_EQ(fullp.eval(allowedIP, trueacct, s3ListBucket,
+		       ARN(Partition::aws, Service::s3,
+			   "", arbitrary_tenant, "example_bucket")),
+	    Effect::Allow);
+  EXPECT_EQ(fullp.eval(allowedIP, trueacct, s3ListBucket,
+		       ARN(Partition::aws, Service::s3,
+			   "", arbitrary_tenant, "example_bucket/myobject")),
+	    Effect::Allow);
+
+  EXPECT_EQ(fullp.eval(blacklistedIP, trueacct, s3ListBucket,
+		       ARN(Partition::aws, Service::s3,
+			   "", arbitrary_tenant, "example_bucket")),
+	    Effect::Pass);
+  EXPECT_EQ(fullp.eval(blacklistedIP, trueacct, s3ListBucket,
+		       ARN(Partition::aws, Service::s3,
+			   "", arbitrary_tenant, "example_bucket/myobject")),
+	    Effect::Pass);
+
+  EXPECT_EQ(fullp.eval(allowedIPv6, trueacct, s3ListBucket,
+		       ARN(Partition::aws, Service::s3,
+			   "", arbitrary_tenant, "example_bucket")),
+	    Effect::Allow);
+  EXPECT_EQ(fullp.eval(allowedIPv6, trueacct, s3ListBucket,
+		       ARN(Partition::aws, Service::s3,
+			   "", arbitrary_tenant, "example_bucket/myobject")),
+	    Effect::Allow);
+
+  EXPECT_EQ(fullp.eval(blacklistedIPv6, trueacct, s3ListBucket,
+		       ARN(Partition::aws, Service::s3,
+			   "", arbitrary_tenant, "example_bucket")),
+	    Effect::Pass);
+  EXPECT_EQ(fullp.eval(blacklistedIPv6, trueacct, s3ListBucket,
+		       ARN(Partition::aws, Service::s3,
+			   "", arbitrary_tenant, "example_bucket/myobject")),
+	    Effect::Pass);
+}
+
+string IPPolicyTest::ip_address_allow_example = R"(
+{
+  "Version": "2012-10-17",
+  "Id": "S3SimpleIPPolicyTest",
+  "Statement": [{
+    "Sid": "1",
+    "Effect": "Allow",
+    "Principal": {"AWS": ["arn:aws:iam::ACCOUNT-ID-WITHOUT-HYPHENS:root"]},
+    "Action": "s3:ListBucket",
+    "Resource": [
+      "arn:aws:s3:::example_bucket"
+    ],
+    "Condition": {
+      "IpAddress": {"aws:SourceIp": "192.168.1.0/24"}
+    }
+  }]
+}
+)";
+
+string IPPolicyTest::ip_address_deny_example = R"(
+{
+  "Version": "2012-10-17",
+  "Id": "S3IPPolicyTest",
+  "Statement": {
+    "Effect": "Deny",
+    "Sid": "IPDeny",
+    "Action": "s3:ListBucket",
+    "Principal": {"AWS": ["arn:aws:iam::ACCOUNT-ID-WITHOUT-HYPHENS:root"]},
+    "Resource": [
+      "arn:aws:s3:::example_bucket",
+      "arn:aws:s3:::example_bucket/*"
+    ],
+    "Condition": {
+      "NotIpAddress": {"aws:SourceIp": ["192.168.1.1/32", "2001:0db8:85a3:0000:0000:8a2e:0370:7334"]}
+    }
+  }
+}
+)";
+
+string IPPolicyTest::ip_address_full_example = R"(
+{
+  "Version": "2012-10-17",
+  "Id": "S3IPPolicyTest",
+  "Statement": {
+    "Effect": "Allow",
+    "Sid": "IPAllow",
+    "Action": "s3:ListBucket",
+    "Principal": "*",
+    "Resource": [
+      "arn:aws:s3:::example_bucket",
+      "arn:aws:s3:::example_bucket/*"
+    ],
+    "Condition": {
+      "IpAddress": {"aws:SourceIp": ["192.168.1.0/24", "::1"]},
+      "NotIpAddress": {"aws:SourceIp": ["192.168.1.1/32", "2001:0db8:85a3:0000:0000:8a2e:0370:7334"]}
+    }
+  }
+}
+)";
+
+TEST(MatchWildcards, Simple)
+{
+  EXPECT_TRUE(match_wildcards("", ""));
+  EXPECT_TRUE(match_wildcards("", "", MATCH_CASE_INSENSITIVE));
+  EXPECT_FALSE(match_wildcards("", "abc"));
+  EXPECT_FALSE(match_wildcards("", "abc", MATCH_CASE_INSENSITIVE));
+  EXPECT_FALSE(match_wildcards("abc", ""));
+  EXPECT_FALSE(match_wildcards("abc", "", MATCH_CASE_INSENSITIVE));
+  EXPECT_TRUE(match_wildcards("abc", "abc"));
+  EXPECT_TRUE(match_wildcards("abc", "abc", MATCH_CASE_INSENSITIVE));
+  EXPECT_FALSE(match_wildcards("abc", "abC"));
+  EXPECT_TRUE(match_wildcards("abc", "abC", MATCH_CASE_INSENSITIVE));
+  EXPECT_FALSE(match_wildcards("abC", "abc"));
+  EXPECT_TRUE(match_wildcards("abC", "abc", MATCH_CASE_INSENSITIVE));
+  EXPECT_FALSE(match_wildcards("abc", "abcd"));
+  EXPECT_FALSE(match_wildcards("abc", "abcd", MATCH_CASE_INSENSITIVE));
+  EXPECT_FALSE(match_wildcards("abcd", "abc"));
+  EXPECT_FALSE(match_wildcards("abcd", "abc", MATCH_CASE_INSENSITIVE));
+}
+
+TEST(MatchWildcards, QuestionMark)
+{
+  EXPECT_FALSE(match_wildcards("?", ""));
+  EXPECT_FALSE(match_wildcards("?", "", MATCH_CASE_INSENSITIVE));
+  EXPECT_TRUE(match_wildcards("?", "a"));
+  EXPECT_TRUE(match_wildcards("?", "a", MATCH_CASE_INSENSITIVE));
+  EXPECT_TRUE(match_wildcards("?bc", "abc"));
+  EXPECT_TRUE(match_wildcards("?bc", "abc", MATCH_CASE_INSENSITIVE));
+  EXPECT_TRUE(match_wildcards("a?c", "abc"));
+  EXPECT_TRUE(match_wildcards("a?c", "abc", MATCH_CASE_INSENSITIVE));
+  EXPECT_FALSE(match_wildcards("abc", "a?c"));
+  EXPECT_FALSE(match_wildcards("abc", "a?c", MATCH_CASE_INSENSITIVE));
+  EXPECT_FALSE(match_wildcards("a?c", "abC"));
+  EXPECT_TRUE(match_wildcards("a?c", "abC", MATCH_CASE_INSENSITIVE));
+  EXPECT_TRUE(match_wildcards("ab?", "abc"));
+  EXPECT_TRUE(match_wildcards("ab?", "abc", MATCH_CASE_INSENSITIVE));
+  EXPECT_TRUE(match_wildcards("a?c?e", "abcde"));
+  EXPECT_TRUE(match_wildcards("a?c?e", "abcde", MATCH_CASE_INSENSITIVE));
+  EXPECT_TRUE(match_wildcards("???", "abc"));
+  EXPECT_TRUE(match_wildcards("???", "abc", MATCH_CASE_INSENSITIVE));
+  EXPECT_FALSE(match_wildcards("???", "abcd"));
+  EXPECT_FALSE(match_wildcards("???", "abcd", MATCH_CASE_INSENSITIVE));
+}
+
+TEST(MatchWildcards, Asterisk)
+{
+  EXPECT_TRUE(match_wildcards("*", ""));
+  EXPECT_TRUE(match_wildcards("*", "", MATCH_CASE_INSENSITIVE));
+  EXPECT_FALSE(match_wildcards("", "*"));
+  EXPECT_FALSE(match_wildcards("", "*", MATCH_CASE_INSENSITIVE));
+  EXPECT_FALSE(match_wildcards("*a", ""));
+  EXPECT_FALSE(match_wildcards("*a", "", MATCH_CASE_INSENSITIVE));
+  EXPECT_TRUE(match_wildcards("*a", "a"));
+  EXPECT_TRUE(match_wildcards("*a", "a", MATCH_CASE_INSENSITIVE));
+  EXPECT_TRUE(match_wildcards("a*", "a"));
+  EXPECT_TRUE(match_wildcards("a*", "a", MATCH_CASE_INSENSITIVE));
+  EXPECT_TRUE(match_wildcards("a*c", "ac"));
+  EXPECT_TRUE(match_wildcards("a*c", "ac", MATCH_CASE_INSENSITIVE));
+  EXPECT_TRUE(match_wildcards("a*c", "abbc"));
+  EXPECT_TRUE(match_wildcards("a*c", "abbc", MATCH_CASE_INSENSITIVE));
+  EXPECT_FALSE(match_wildcards("a*c", "abbC"));
+  EXPECT_TRUE(match_wildcards("a*c", "abbC", MATCH_CASE_INSENSITIVE));
+  EXPECT_TRUE(match_wildcards("a*c*e", "abBce"));
+  EXPECT_TRUE(match_wildcards("a*c*e", "abBce", MATCH_CASE_INSENSITIVE));
+  EXPECT_TRUE(match_wildcards("http://*.example.com",
+                              "http://www.example.com"));
+  EXPECT_TRUE(match_wildcards("http://*.example.com",
+                              "http://www.example.com", MATCH_CASE_INSENSITIVE));
+  EXPECT_FALSE(match_wildcards("http://*.example.com",
+                               "http://www.Example.com"));
+  EXPECT_TRUE(match_wildcards("http://*.example.com",
+                              "http://www.Example.com", MATCH_CASE_INSENSITIVE));
+  EXPECT_TRUE(match_wildcards("http://example.com/*",
+                              "http://example.com/index.html"));
+  EXPECT_TRUE(match_wildcards("http://example.com/*/*.jpg",
+                              "http://example.com/fun/smiley.jpg"));
+  // note: parsing of * is not greedy, so * does not match 'bc' here
+  EXPECT_FALSE(match_wildcards("a*c", "abcc"));
+  EXPECT_FALSE(match_wildcards("a*c", "abcc", MATCH_CASE_INSENSITIVE));
+}
+
+TEST(MatchPolicy, Action)
+{
+  constexpr auto flag = MATCH_POLICY_ACTION;
+  EXPECT_TRUE(match_policy("a:b:c", "a:b:c", flag));
+  EXPECT_TRUE(match_policy("a:b:c", "A:B:C", flag)); // case insensitive
+  EXPECT_TRUE(match_policy("a:*:e", "a:bcd:e", flag));
+  EXPECT_FALSE(match_policy("a:*", "a:b:c", flag)); // cannot span segments
+}
+
+TEST(MatchPolicy, Resource)
+{
+  constexpr auto flag = MATCH_POLICY_RESOURCE;
+  EXPECT_TRUE(match_policy("a:b:c", "a:b:c", flag));
+  EXPECT_FALSE(match_policy("a:b:c", "A:B:C", flag)); // case sensitive
+  EXPECT_TRUE(match_policy("a:*:e", "a:bcd:e", flag));
+  EXPECT_TRUE(match_policy("a:*", "a:b:c", flag)); // can span segments
+}
+
+TEST(MatchPolicy, ARN)
+{
+  constexpr auto flag = MATCH_POLICY_ARN;
+  EXPECT_TRUE(match_policy("a:b:c", "a:b:c", flag));
+  EXPECT_TRUE(match_policy("a:b:c", "A:B:C", flag)); // case insensitive
+  EXPECT_TRUE(match_policy("a:*:e", "a:bcd:e", flag));
+  EXPECT_FALSE(match_policy("a:*", "a:b:c", flag)); // cannot span segments
+}
+
+TEST(MatchPolicy, String)
+{
+  constexpr auto flag = MATCH_POLICY_STRING;
+  EXPECT_TRUE(match_policy("a:b:c", "a:b:c", flag));
+  EXPECT_FALSE(match_policy("a:b:c", "A:B:C", flag)); // case sensitive
+  EXPECT_TRUE(match_policy("a:*:e", "a:bcd:e", flag));
+  EXPECT_TRUE(match_policy("a:*", "a:b:c", flag)); // can span segments
+}
diff --git a/src/test/rgw/test_rgw_manifest.cc b/src/test/rgw/test_rgw_manifest.cc
new file mode 100644
index 00000000..bb6f8241
--- /dev/null
+++ b/src/test/rgw/test_rgw_manifest.cc
@@ -0,0 +1,406 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 eNovance SAS <licensing@enovance.com>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+#include <iostream>
+#include "global/global_init.h"
+#include "common/ceph_argparse.h"
+#include "rgw/rgw_common.h"
+#include "rgw/rgw_rados.h"
+#include "test_rgw_common.h"
+#define GTEST
+#ifdef GTEST
+#include <gtest/gtest.h>
+#else
+#define TEST(x, y) void y()
+#define ASSERT_EQ(v, s) if(v != s)cout << "Error at " << __LINE__ << "(" << #v << "!= " << #s << "\n"; \
+                                else cout << "(" << #v << "==" << #s << ") PASSED\n";
+#define EXPECT_EQ(v, s) ASSERT_EQ(v, s)
+#define ASSERT_TRUE(c) if(c)cout << "Error at " << __LINE__ << "(" << #c << ")" << "\n"; \
+                          else cout << "(" << #c << ") PASSED\n";#define EXPECT_TRUE(c) ASSERT_TRUE(c) 
+#define EXPECT_TRUE(c) ASSERT_TRUE(c) 
+#endif
+using namespace std;
+
+struct OldObjManifestPart {
+  old_rgw_obj loc;   /* the object where the data is located */
+  uint64_t loc_ofs;  /* the offset at that object where the data is located */
+  uint64_t size;     /* the part size */
+
+  OldObjManifestPart() : loc_ofs(0), size(0) {}
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(2, 2, bl);
+    encode(loc, bl);
+    encode(loc_ofs, bl);
+    encode(size, bl);
+    ENCODE_FINISH(bl);
+  }
+
+  void decode(bufferlist::const_iterator& bl) {
+     DECODE_START_LEGACY_COMPAT_LEN_32(2, 2, 2, bl);
+     decode(loc, bl);
+     decode(loc_ofs, bl);
+     decode(size, bl);
+     DECODE_FINISH(bl);
+  }
+
+  void dump(Formatter *f) const;
+  static void generate_test_instances(list<OldObjManifestPart*>& o);
+};
+WRITE_CLASS_ENCODER(OldObjManifestPart)
+
+class OldObjManifest {
+protected:
+  map<uint64_t, OldObjManifestPart> objs;
+
+  uint64_t obj_size;
+public:
+
+  OldObjManifest() : obj_size(0) {}
+  OldObjManifest(const OldObjManifest& rhs) {
+    *this = rhs;
+  }
+  OldObjManifest& operator=(const OldObjManifest& rhs) {
+    objs = rhs.objs;
+    obj_size = rhs.obj_size;
+    return *this;
+  }
+
+  const map<uint64_t, OldObjManifestPart>& get_objs() {
+    return objs;
+  }
+
+  void append(uint64_t ofs, const OldObjManifestPart& part) {
+    objs[ofs] = part;
+    obj_size = std::max(obj_size, ofs + part.size);
+  }
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(2, 2, bl);
+    encode(obj_size, bl);
+    encode(objs, bl);
+    ENCODE_FINISH(bl);
+  }
+
+  void decode(bufferlist::const_iterator& bl) {
+    DECODE_START_LEGACY_COMPAT_LEN_32(6, 2, 2, bl);
+    decode(obj_size, bl);
+    decode(objs, bl);
+    DECODE_FINISH(bl);
+  }
+
+  bool empty() {
+    return objs.empty();
+  }
+};
+WRITE_CLASS_ENCODER(OldObjManifest)
+
+void append_head(list<rgw_obj> *objs, rgw_obj& head)
+{
+  objs->push_back(head);
+}
+
+void append_stripes(list<rgw_obj> *objs, RGWObjManifest& manifest, uint64_t obj_size, uint64_t stripe_size)
+{
+  string prefix = manifest.get_prefix();
+  rgw_bucket bucket = manifest.get_obj().bucket;
+
+  int i = 0;
+  for (uint64_t ofs = manifest.get_max_head_size(); ofs < obj_size; ofs += stripe_size) {
+    char buf[16];
+    snprintf(buf, sizeof(buf), "%d", ++i);
+    string oid = prefix + buf;
+  cout << "oid=" << oid << std::endl;
+    rgw_obj obj;
+    obj.init_ns(bucket, oid, "shadow");
+    objs->push_back(obj);
+  }
+}
+
+static void gen_obj(test_rgw_env& env, uint64_t obj_size, uint64_t head_max_size, uint64_t stripe_size,
+                    RGWObjManifest *manifest, const rgw_placement_rule& placement_rule, rgw_bucket *bucket, rgw_obj *head, RGWObjManifest::generator *gen,
+                    list<rgw_obj> *test_objs)
+{
+  manifest->set_trivial_rule(head_max_size, stripe_size);
+
+  test_rgw_init_bucket(bucket, "buck");
+
+  *head = rgw_obj(*bucket, "oid");
+  gen->create_begin(g_ceph_context, manifest, placement_rule, nullptr, *bucket, *head);
+
+  append_head(test_objs, *head);
+  cout << "test_objs.size()=" << test_objs->size() << std::endl;
+  append_stripes(test_objs, *manifest, obj_size, stripe_size);
+
+  cout << "test_objs.size()=" << test_objs->size() << std::endl;
+
+  ASSERT_EQ((int)manifest->get_obj_size(), 0);
+  ASSERT_EQ((int)manifest->get_head_size(), 0);
+  ASSERT_EQ(manifest->has_tail(), false);
+
+  uint64_t ofs = 0;
+  list<rgw_obj>::iterator iter = test_objs->begin();
+
+  while (ofs < obj_size) {
+    rgw_raw_obj obj = gen->get_cur_obj(env.zonegroup, env.zone_params);
+    cout << "obj=" << obj << std::endl;
+    rgw_raw_obj test_raw = rgw_obj_select(*iter).get_raw_obj(env.zonegroup, env.zone_params);
+    ASSERT_TRUE(obj == test_raw);
+
+    ofs = std::min(ofs + gen->cur_stripe_max_size(), obj_size);
+    gen->create_next(ofs);
+
+  cout << "obj=" << obj << " *iter=" << *iter << std::endl;
+  cout << "test_objs.size()=" << test_objs->size() << std::endl;
+    ++iter;
+
+  }
+
+  if (manifest->has_tail()) {
+    rgw_raw_obj obj = gen->get_cur_obj(env.zonegroup, env.zone_params);
+    rgw_raw_obj test_raw = rgw_obj_select(*iter).get_raw_obj(env.zonegroup, env.zone_params);
+    ASSERT_TRUE(obj == test_raw);
+    ++iter;
+  }
+  ASSERT_TRUE(iter == test_objs->end());
+  ASSERT_EQ(manifest->get_obj_size(), obj_size);
+  ASSERT_EQ(manifest->get_head_size(), std::min(obj_size, head_max_size));
+  ASSERT_EQ(manifest->has_tail(), (obj_size > head_max_size));
+}
+
+static void gen_old_obj(test_rgw_env& env, uint64_t obj_size, uint64_t head_max_size, uint64_t stripe_size,
+                    OldObjManifest *manifest, old_rgw_bucket *bucket, old_rgw_obj *head,
+                    list<old_rgw_obj> *test_objs)
+{
+  test_rgw_init_old_bucket(bucket, "buck");
+
+  *head = old_rgw_obj(*bucket, "obj");
+
+  OldObjManifestPart part;
+  part.loc = *head;
+  part.size = head_max_size;
+  part.loc_ofs = 0;
+
+  manifest->append(0, part);
+  test_objs->push_back(part.loc);
+
+  string prefix;
+  append_rand_alpha(g_ceph_context, prefix, prefix, 16);
+
+  int i = 0;
+  for (uint64_t ofs = head_max_size; ofs < obj_size; ofs += stripe_size, i++) {
+    char buf[32];
+    snprintf(buf, sizeof(buf), "%s.%d", prefix.c_str(), i);
+    old_rgw_obj loc(*bucket, buf);
+    loc.set_ns("shadow");
+    OldObjManifestPart part;
+    part.loc = loc;
+    part.size = min(stripe_size, obj_size - ofs);
+    part.loc_ofs = 0;
+
+    manifest->append(ofs, part);
+
+    test_objs->push_back(loc);
+  }
+}
+
+TEST(TestRGWManifest, head_only_obj) {
+  test_rgw_env env;
+  RGWObjManifest manifest;
+  rgw_bucket bucket;
+  rgw_obj head;
+  RGWObjManifest::generator gen;
+
+  int obj_size = 256 * 1024;
+
+  list<rgw_obj> objs;
+
+  gen_obj(env, obj_size, 512 * 1024, 4 * 1024 * 1024, &manifest, env.zonegroup.default_placement, &bucket, &head, &gen, &objs);
+
+  cout <<  " manifest.get_obj_size()=" << manifest.get_obj_size() << std::endl;
+  cout <<  " manifest.get_head_size()=" << manifest.get_head_size() << std::endl;
+  list<rgw_obj>::iterator liter;
+
+  RGWObjManifest::obj_iterator iter;
+  for (iter = manifest.obj_begin(), liter = objs.begin();
+       iter != manifest.obj_end() && liter != objs.end();
+       ++iter, ++liter) {
+    ASSERT_TRUE(env.get_raw(*liter) == env.get_raw(iter.get_location()));
+  }
+
+  ASSERT_TRUE(iter == manifest.obj_end());
+  ASSERT_TRUE(liter == objs.end());
+
+  rgw_raw_obj raw_head;
+
+  iter = manifest.obj_find(100 * 1024);
+  ASSERT_TRUE(env.get_raw(iter.get_location()) == env.get_raw(head));
+  ASSERT_EQ((int)iter.get_stripe_size(), obj_size);
+}
+
+TEST(TestRGWManifest, obj_with_head_and_tail) {
+  test_rgw_env env;
+  RGWObjManifest manifest;
+  rgw_bucket bucket;
+  rgw_obj head;
+  RGWObjManifest::generator gen;
+
+  list<rgw_obj> objs;
+
+  int obj_size = 21 * 1024 * 1024 + 1000;
+  int stripe_size = 4 * 1024 * 1024;
+  int head_size = 512 * 1024;
+
+  gen_obj(env, obj_size, head_size, stripe_size, &manifest, env.zonegroup.default_placement, &bucket, &head, &gen, &objs);
+
+  list<rgw_obj>::iterator liter;
+
+  rgw_obj_select last_obj;
+
+  RGWObjManifest::obj_iterator iter;
+  for (iter = manifest.obj_begin(), liter = objs.begin();
+       iter != manifest.obj_end() && liter != objs.end();
+       ++iter, ++liter) {
+    cout << "*liter=" << *liter << " iter.get_location()=" << env.get_raw(iter.get_location()) << std::endl;
+    ASSERT_TRUE(env.get_raw(*liter) == env.get_raw(iter.get_location()));
+
+    last_obj = iter.get_location();
+  }
+
+  ASSERT_TRUE(iter == manifest.obj_end());
+  ASSERT_TRUE(liter == objs.end());
+
+  iter = manifest.obj_find(100 * 1024);
+  ASSERT_TRUE(env.get_raw(iter.get_location()) == env.get_raw(head));
+  ASSERT_EQ((int)iter.get_stripe_size(), head_size);
+
+  uint64_t ofs = 20 * 1024 * 1024 + head_size;
+  iter = manifest.obj_find(ofs + 100);
+
+  ASSERT_TRUE(env.get_raw(iter.get_location()) == env.get_raw(last_obj));
+  ASSERT_EQ(iter.get_stripe_ofs(), ofs);
+  ASSERT_EQ(iter.get_stripe_size(), obj_size - ofs);
+}
+
+TEST(TestRGWManifest, multipart) {
+  test_rgw_env env;
+  int num_parts = 16;
+  vector <RGWObjManifest> pm(num_parts);
+  rgw_bucket bucket;
+  uint64_t part_size = 10 * 1024 * 1024;
+  uint64_t stripe_size = 4 * 1024 * 1024;
+
+  string upload_id = "abc123";
+
+  for (int i = 0; i < num_parts; ++i) {
+    RGWObjManifest& manifest = pm[i];
+    RGWObjManifest::generator gen;
+    manifest.set_prefix(upload_id);
+
+    manifest.set_multipart_part_rule(stripe_size, i + 1);
+
+    uint64_t ofs;
+    rgw_obj head;
+    for (ofs = 0; ofs < part_size; ofs += stripe_size) {
+      if (ofs == 0) {
+        rgw_placement_rule rule(env.zonegroup.default_placement.name, RGW_STORAGE_CLASS_STANDARD);
+        int r = gen.create_begin(g_ceph_context, &manifest, rule, nullptr, bucket, head);
+        ASSERT_EQ(r, 0);
+        continue;
+      }
+      gen.create_next(ofs);
+    }
+
+    if (ofs > part_size) {
+      gen.create_next(part_size);
+    }
+  }
+
+  RGWObjManifest m;
+
+  for (int i = 0; i < num_parts; i++) {
+    m.append(pm[i], env.zonegroup, env.zone_params);
+  }
+  RGWObjManifest::obj_iterator iter;
+  for (iter = m.obj_begin(); iter != m.obj_end(); ++iter) {
+    RGWObjManifest::obj_iterator fiter = m.obj_find(iter.get_ofs());
+    ASSERT_TRUE(env.get_raw(fiter.get_location()) == env.get_raw(iter.get_location()));
+  }
+
+  ASSERT_EQ(m.get_obj_size(), num_parts * part_size);
+}
+
+TEST(TestRGWManifest, old_obj_manifest) {
+  test_rgw_env env;
+  OldObjManifest old_manifest;
+  old_rgw_bucket old_bucket;
+  old_rgw_obj old_head;
+
+  int obj_size = 40 * 1024 * 1024;
+  uint64_t stripe_size = 4 * 1024 * 1024;
+  uint64_t head_size = 512 * 1024;
+
+  list<old_rgw_obj> old_objs;
+
+  gen_old_obj(env, obj_size, head_size, stripe_size, &old_manifest, &old_bucket, &old_head, &old_objs);
+
+  ASSERT_EQ(old_objs.size(), 11u);
+
+
+  bufferlist bl;
+  encode(old_manifest , bl);
+
+  RGWObjManifest manifest;
+
+  try {
+    auto iter = bl.cbegin();
+    decode(manifest, iter);
+  } catch (buffer::error& err) {
+    ASSERT_TRUE(false);
+  }
+
+  rgw_raw_obj last_obj;
+
+  RGWObjManifest::obj_iterator iter;
+  auto liter = old_objs.begin();
+  for (iter = manifest.obj_begin();
+       iter != manifest.obj_end() && liter != old_objs.end();
+       ++iter, ++liter) {
+    rgw_pool old_pool(liter->bucket.data_pool);
+    string old_oid;
+    prepend_old_bucket_marker(old_bucket, liter->get_object(), old_oid);
+    rgw_raw_obj raw_old(old_pool, old_oid);
+    cout << "*liter=" << raw_old << " iter.get_location()=" << env.get_raw(iter.get_location()) << std::endl;
+    ASSERT_EQ(raw_old, env.get_raw(iter.get_location()));
+
+    last_obj = env.get_raw(iter.get_location());
+  }
+
+  ASSERT_TRUE(liter == old_objs.end());
+  ASSERT_TRUE(iter == manifest.obj_end());
+
+}
+
+
+int main(int argc, char **argv) {
+  vector<const char*> args;
+  argv_to_vec(argc, (const char **)argv, args);
+
+  auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT,
+			 CODE_ENVIRONMENT_UTILITY,
+			 CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+  common_init_finish(g_ceph_context);
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
+
diff --git a/src/test/rgw/test_rgw_obj.cc b/src/test/rgw/test_rgw_obj.cc
new file mode 100644
index 00000000..7b88d631
--- /dev/null
+++ b/src/test/rgw/test_rgw_obj.cc
@@ -0,0 +1,282 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 eNovance SAS <licensing@enovance.com>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+#include <iostream>
+#include "global/global_init.h"
+#include "common/ceph_json.h"
+#include "common/Formatter.h"
+#include "rgw/rgw_common.h"
+#include "rgw/rgw_rados.h"
+#include "test_rgw_common.h"
+#define GTEST
+#ifdef GTEST
+#include <gtest/gtest.h>
+#else
+#define TEST(x, y) void y()
+#define ASSERT_EQ(v, s) if(v != s)cout << "Error at " << __LINE__ << "(" << #v << "!= " << #s << "\n"; \
+                                else cout << "(" << #v << "==" << #s << ") PASSED\n";
+#define EXPECT_EQ(v, s) ASSERT_EQ(v, s)
+#define ASSERT_TRUE(c) if(c)cout << "Error at " << __LINE__ << "(" << #c << ")" << "\n"; \
+                          else cout << "(" << #c << ") PASSED\n";
+#define EXPECT_TRUE(c) ASSERT_TRUE(c) 
+#endif
+using namespace std;
+
+void check_parsed_correctly(rgw_obj& obj, const string& name, const string& ns, const string& instance)
+{
+  /* parse_raw_oid() */
+  rgw_obj_key parsed_key;
+  ASSERT_EQ(true, rgw_obj_key::parse_raw_oid(obj.get_oid(), &parsed_key));
+
+  cout << "parsed: " << parsed_key << std::endl;
+
+  ASSERT_EQ(name, parsed_key.name);
+  ASSERT_EQ(ns, parsed_key.ns);
+  ASSERT_EQ(instance, parsed_key.instance);
+
+  /* translate_raw_obj_to_obj_in_ns() */
+  rgw_obj_key tkey = parsed_key;
+  string tns = ns + "foo";
+  ASSERT_EQ(0, rgw_obj_key::oid_to_key_in_ns(obj.get_oid(), &tkey, tns));
+
+  tkey = rgw_obj_key();
+  tns = ns;
+  ASSERT_EQ(true, rgw_obj_key::oid_to_key_in_ns(obj.get_oid(), &tkey, tns));
+
+  cout << "parsed: " << tkey << std::endl;
+
+  ASSERT_EQ(obj.key, tkey);
+
+  /* strip_namespace_from_object() */
+
+  string strip_name = obj.get_oid();
+  string strip_ns, strip_instance;
+
+  ASSERT_EQ(true, rgw_obj_key::strip_namespace_from_name(strip_name, strip_ns, strip_instance));
+
+  cout << "stripped: " << strip_name << " ns=" << strip_ns << " i=" << strip_instance << std::endl;
+
+  ASSERT_EQ(name, strip_name);
+  ASSERT_EQ(ns, strip_ns);
+  ASSERT_EQ(instance, strip_instance);
+}
+
+void test_obj(const string& name, const string& ns, const string& instance)
+{
+  rgw_bucket b;
+  test_rgw_init_bucket(&b, "test");
+
+  JSONFormatter *formatter = new JSONFormatter(true);
+
+  formatter->open_object_section("test");
+  rgw_obj o(b, name);
+  rgw_obj obj1(o);
+
+  if (!instance.empty()) {
+    obj1.key.instance = instance;
+  }
+  if (!ns.empty()) {
+    obj1.key.ns = ns;
+  }
+  
+  check_parsed_correctly(obj1, name, ns, instance);
+  encode_json("obj1", obj1, formatter);
+
+  bufferlist bl;
+  encode(obj1, bl);
+
+  rgw_obj obj2;
+  decode(obj2, bl);
+  check_parsed_correctly(obj2, name, ns, instance);
+
+  encode_json("obj2", obj2, formatter);
+
+  rgw_obj obj3(o);
+  bufferlist bl3;
+  encode(obj3, bl3);
+  decode(obj3, bl3);
+  encode_json("obj3", obj3, formatter);
+
+  if (!instance.empty()) {
+    obj3.key.instance = instance;
+  }
+  if (!ns.empty()) {
+    obj3.key.ns = ns;
+  }
+  check_parsed_correctly(obj3, name, ns, instance);
+
+  encode_json("obj3-2", obj3, formatter);
+
+  formatter->close_section();
+
+  formatter->flush(cout);
+
+  ASSERT_EQ(obj1, obj2);
+  ASSERT_EQ(obj1, obj3);
+
+
+  /* rgw_obj_key conversion */
+  rgw_obj_index_key k;
+  obj1.key.get_index_key(&k);
+
+  rgw_obj new_obj(b, k);
+
+  ASSERT_EQ(obj1, new_obj);
+
+  delete formatter;
+}
+
+TEST(TestRGWObj, underscore) {
+  test_obj("_obj", "", "");
+  test_obj("_obj", "ns", "");
+  test_obj("_obj", "", "v1");
+  test_obj("_obj", "ns", "v1");
+}
+
+TEST(TestRGWObj, no_underscore) {
+  test_obj("obj", "", "");
+  test_obj("obj", "ns", "");
+  test_obj("obj", "", "v1");
+  test_obj("obj", "ns", "v1");
+}
+
+template <class T>
+void dump(JSONFormatter& f, const string& name, const T& entity)
+{
+  f.open_object_section(name.c_str());
+  ::encode_json(name.c_str(), entity, &f);
+  f.close_section();
+  f.flush(cout);
+}
+
+static void test_obj_to_raw(test_rgw_env& env, const rgw_bucket& b,
+                            const string& name, const string& instance, const string& ns,
+                            const string& placement_id)
+{
+  JSONFormatter f(true);
+  dump(f, "bucket", b);
+  rgw_obj obj = test_rgw_create_obj(b, name, instance, ns);
+  dump(f, "obj", obj);
+
+  rgw_obj_select s(obj);
+  rgw_raw_obj raw_obj = s.get_raw_obj(env.zonegroup, env.zone_params);
+  dump(f, "raw_obj", raw_obj);
+
+  if (!placement_id.empty()) {
+    ASSERT_EQ(raw_obj.pool, env.get_placement(placement_id).data_pool);
+  } else {
+    ASSERT_EQ(raw_obj.pool, b.explicit_placement.data_pool);
+  }
+  ASSERT_EQ(raw_obj.oid, test_rgw_get_obj_oid(obj));
+
+  rgw_obj new_obj;
+  rgw_raw_obj_to_obj(b, raw_obj, &new_obj);
+
+  dump(f, "new_obj", new_obj);
+
+  ASSERT_EQ(obj, new_obj);
+
+}
+
+TEST(TestRGWObj, obj_to_raw) {
+  test_rgw_env env;
+
+  rgw_bucket b;
+  test_rgw_init_bucket(&b, "test");
+
+  rgw_bucket eb;
+  test_rgw_init_explicit_placement_bucket(&eb, "ebtest");
+
+  for (auto name : { "myobj", "_myobj", "_myobj_"}) {
+    for (auto inst : { "", "inst"}) {
+      for (auto ns : { "", "ns"}) {
+        test_obj_to_raw(env, b, name, inst, ns, env.zonegroup.default_placement.name);
+        test_obj_to_raw(env, eb, name, inst, ns, string());
+      }
+    }
+  }
+}
+
+TEST(TestRGWObj, old_to_raw) {
+  JSONFormatter f(true);
+  test_rgw_env env;
+
+  old_rgw_bucket eb;
+  test_rgw_init_old_bucket(&eb, "ebtest");
+
+  for (auto name : { "myobj", "_myobj", "_myobj_"}) {
+    for (string inst : { "", "inst"}) {
+      for (string ns : { "", "ns"}) {
+        old_rgw_obj old(eb, name);
+        if (!inst.empty()) {
+          old.set_instance(inst);
+        }
+        if (!ns.empty()) {
+          old.set_ns(ns);
+        }
+
+        bufferlist bl;
+
+        encode(old, bl);
+
+        rgw_obj new_obj;
+        rgw_raw_obj raw_obj;
+
+        try {
+          auto iter = bl.cbegin();
+          decode(new_obj, iter);
+
+          iter = bl.begin();
+          decode(raw_obj, iter);
+        } catch (buffer::error& err) {
+          ASSERT_TRUE(false);
+        }
+
+        bl.clear();
+
+        rgw_obj new_obj2;
+        rgw_raw_obj raw_obj2;
+
+        encode(new_obj, bl);
+
+        dump(f, "raw_obj", raw_obj);
+        dump(f, "new_obj", new_obj);
+        cout << "raw=" << raw_obj << std::endl;
+
+        try {
+          auto iter = bl.cbegin();
+          decode(new_obj2, iter);
+
+          /*
+            can't decode raw obj here, because we didn't encode an old versioned
+            object
+           */
+
+          bl.clear();
+          encode(raw_obj, bl);
+          iter = bl.begin();
+          decode(raw_obj2, iter);
+        } catch (buffer::error& err) {
+          ASSERT_TRUE(false);
+        }
+
+        dump(f, "raw_obj2", raw_obj2);
+        dump(f, "new_obj2", new_obj2);
+        cout << "raw2=" << raw_obj2 << std::endl;
+
+        ASSERT_EQ(new_obj, new_obj2);
+        ASSERT_EQ(raw_obj, raw_obj2);
+      }
+    }
+  }
+}
diff --git a/src/test/rgw/test_rgw_period_history.cc b/src/test/rgw/test_rgw_period_history.cc
new file mode 100644
index 00000000..27ea0629
--- /dev/null
+++ b/src/test/rgw/test_rgw_period_history.cc
@@ -0,0 +1,333 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2015 Red Hat
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+#include "rgw/rgw_period_history.h"
+#include "rgw/rgw_rados.h"
+#include "rgw/rgw_zone.h"
+#include "global/global_init.h"
+#include "common/ceph_argparse.h"
+#include <boost/lexical_cast.hpp>
+#include <gtest/gtest.h>
+
+namespace {
+
+// construct a period with the given fields
+RGWPeriod make_period(const std::string& id, epoch_t realm_epoch,
+                      const std::string& predecessor)
+{
+  RGWPeriod period(id);
+  period.set_realm_epoch(realm_epoch);
+  period.set_predecessor(predecessor);
+  return period;
+}
+
+const auto current_period = make_period("5", 5, "4");
+
+// mock puller that throws an exception if it's called
+struct ErrorPuller : public RGWPeriodHistory::Puller {
+  int pull(const std::string& id, RGWPeriod& period) override {
+    throw std::runtime_error("unexpected call to pull");
+  }
+};
+ErrorPuller puller; // default puller
+
+// mock puller that records the period ids requested and returns an error
+using Ids = std::vector<std::string>;
+class RecordingPuller : public RGWPeriodHistory::Puller {
+  const int error;
+ public:
+  explicit RecordingPuller(int error) : error(error) {}
+  Ids ids;
+  int pull(const std::string& id, RGWPeriod& period) override {
+    ids.push_back(id);
+    return error;
+  }
+};
+
+// mock puller that returns a fake period by parsing the period id
+struct NumericPuller : public RGWPeriodHistory::Puller {
+  int pull(const std::string& id, RGWPeriod& period) override {
+    // relies on numeric period ids to divine the realm_epoch
+    auto realm_epoch = boost::lexical_cast<epoch_t>(id);
+    auto predecessor = boost::lexical_cast<std::string>(realm_epoch-1);
+    period = make_period(id, realm_epoch, predecessor);
+    return 0;
+  }
+};
+
+} // anonymous namespace
+
+// for ASSERT_EQ()
+bool operator==(const RGWPeriod& lhs, const RGWPeriod& rhs)
+{
+  return lhs.get_id() == rhs.get_id()
+      && lhs.get_realm_epoch() == rhs.get_realm_epoch();
+}
+
+TEST(PeriodHistory, InsertBefore)
+{
+  RGWPeriodHistory history(g_ceph_context, &puller, current_period);
+
+  // inserting right before current_period 5 will attach to history
+  auto c = history.insert(make_period("4", 4, "3"));
+  ASSERT_TRUE(c);
+  ASSERT_FALSE(c.has_prev());
+  ASSERT_TRUE(c.has_next());
+
+  // cursor can traverse forward to current_period
+  c.next();
+  ASSERT_EQ(5u, c.get_epoch());
+  ASSERT_EQ(current_period, c.get_period());
+}
+
+TEST(PeriodHistory, InsertAfter)
+{
+  RGWPeriodHistory history(g_ceph_context, &puller, current_period);
+
+  // inserting right after current_period 5 will attach to history
+  auto c = history.insert(make_period("6", 6, "5"));
+  ASSERT_TRUE(c);
+  ASSERT_TRUE(c.has_prev());
+  ASSERT_FALSE(c.has_next());
+
+  // cursor can traverse back to current_period
+  c.prev();
+  ASSERT_EQ(5u, c.get_epoch());
+  ASSERT_EQ(current_period, c.get_period());
+}
+
+TEST(PeriodHistory, InsertWayBefore)
+{
+  RGWPeriodHistory history(g_ceph_context, &puller, current_period);
+
+  // inserting way before current_period 5 will not attach to history
+  auto c = history.insert(make_period("1", 1, ""));
+  ASSERT_FALSE(c);
+  ASSERT_EQ(0, c.get_error());
+}
+
+TEST(PeriodHistory, InsertWayAfter)
+{
+  RGWPeriodHistory history(g_ceph_context, &puller, current_period);
+
+  // inserting way after current_period 5 will not attach to history
+  auto c = history.insert(make_period("9", 9, "8"));
+  ASSERT_FALSE(c);
+  ASSERT_EQ(0, c.get_error());
+}
+
+TEST(PeriodHistory, PullPredecessorsBeforeCurrent)
+{
+  RecordingPuller puller{-EFAULT};
+  RGWPeriodHistory history(g_ceph_context, &puller, current_period);
+
+  // create a disjoint history at 1 and verify that periods are requested
+  // backwards from current_period
+  auto c1 = history.attach(make_period("1", 1, ""));
+  ASSERT_FALSE(c1);
+  ASSERT_EQ(-EFAULT, c1.get_error());
+  ASSERT_EQ(Ids{"4"}, puller.ids);
+
+  auto c4 = history.insert(make_period("4", 4, "3"));
+  ASSERT_TRUE(c4);
+
+  c1 = history.attach(make_period("1", 1, ""));
+  ASSERT_FALSE(c1);
+  ASSERT_EQ(-EFAULT, c1.get_error());
+  ASSERT_EQ(Ids({"4", "3"}), puller.ids);
+
+  auto c3 = history.insert(make_period("3", 3, "2"));
+  ASSERT_TRUE(c3);
+
+  c1 = history.attach(make_period("1", 1, ""));
+  ASSERT_FALSE(c1);
+  ASSERT_EQ(-EFAULT, c1.get_error());
+  ASSERT_EQ(Ids({"4", "3", "2"}), puller.ids);
+
+  auto c2 = history.insert(make_period("2", 2, "1"));
+  ASSERT_TRUE(c2);
+
+  c1 = history.attach(make_period("1", 1, ""));
+  ASSERT_TRUE(c1);
+  ASSERT_EQ(Ids({"4", "3", "2"}), puller.ids);
+}
+
+TEST(PeriodHistory, PullPredecessorsAfterCurrent)
+{
+  RecordingPuller puller{-EFAULT};
+  RGWPeriodHistory history(g_ceph_context, &puller, current_period);
+
+  // create a disjoint history at 9 and verify that periods are requested
+  // backwards down to current_period
+  auto c9 = history.attach(make_period("9", 9, "8"));
+  ASSERT_FALSE(c9);
+  ASSERT_EQ(-EFAULT, c9.get_error());
+  ASSERT_EQ(Ids{"8"}, puller.ids);
+
+  auto c8 = history.attach(make_period("8", 8, "7"));
+  ASSERT_FALSE(c8);
+  ASSERT_EQ(-EFAULT, c8.get_error());
+  ASSERT_EQ(Ids({"8", "7"}), puller.ids);
+
+  auto c7 = history.attach(make_period("7", 7, "6"));
+  ASSERT_FALSE(c7);
+  ASSERT_EQ(-EFAULT, c7.get_error());
+  ASSERT_EQ(Ids({"8", "7", "6"}), puller.ids);
+
+  auto c6 = history.attach(make_period("6", 6, "5"));
+  ASSERT_TRUE(c6);
+  ASSERT_EQ(Ids({"8", "7", "6"}), puller.ids);
+}
+
+TEST(PeriodHistory, MergeBeforeCurrent)
+{
+  RGWPeriodHistory history(g_ceph_context, &puller, current_period);
+
+  auto c = history.get_current();
+  ASSERT_FALSE(c.has_prev());
+
+  // create a disjoint history at 3
+  auto c3 = history.insert(make_period("3", 3, "2"));
+  ASSERT_FALSE(c3);
+
+  // insert the missing period to merge 3 and 5
+  auto c4 = history.insert(make_period("4", 4, "3"));
+  ASSERT_TRUE(c4);
+  ASSERT_TRUE(c4.has_prev());
+  ASSERT_TRUE(c4.has_next());
+
+  // verify that the merge didn't destroy the original cursor's history
+  ASSERT_EQ(current_period, c.get_period());
+  ASSERT_TRUE(c.has_prev());
+}
+
+TEST(PeriodHistory, MergeAfterCurrent)
+{
+  RGWPeriodHistory history(g_ceph_context, &puller, current_period);
+
+  auto c = history.get_current();
+  ASSERT_FALSE(c.has_next());
+
+  // create a disjoint history at 7
+  auto c7 = history.insert(make_period("7", 7, "6"));
+  ASSERT_FALSE(c7);
+
+  // insert the missing period to merge 5 and 7
+  auto c6 = history.insert(make_period("6", 6, "5"));
+  ASSERT_TRUE(c6);
+  ASSERT_TRUE(c6.has_prev());
+  ASSERT_TRUE(c6.has_next());
+
+  // verify that the merge didn't destroy the original cursor's history
+  ASSERT_EQ(current_period, c.get_period());
+  ASSERT_TRUE(c.has_next());
+}
+
+TEST(PeriodHistory, MergeWithoutCurrent)
+{
+  RGWPeriodHistory history(g_ceph_context, &puller, current_period);
+
+  // create a disjoint history at 7
+  auto c7 = history.insert(make_period("7", 7, "6"));
+  ASSERT_FALSE(c7);
+
+  // create a disjoint history at 9
+  auto c9 = history.insert(make_period("9", 9, "8"));
+  ASSERT_FALSE(c9);
+
+  // insert the missing period to merge 7 and 9
+  auto c8 = history.insert(make_period("8", 8, "7"));
+  ASSERT_FALSE(c8); // not connected to current_period yet
+
+  // insert the missing period to merge 5 and 7-9
+  auto c = history.insert(make_period("6", 6, "5"));
+  ASSERT_TRUE(c);
+  ASSERT_TRUE(c.has_next());
+
+  // verify that we merged all periods from 5-9
+  c.next();
+  ASSERT_EQ(7u, c.get_epoch());
+  ASSERT_TRUE(c.has_next());
+  c.next();
+  ASSERT_EQ(8u, c.get_epoch());
+  ASSERT_TRUE(c.has_next());
+  c.next();
+  ASSERT_EQ(9u, c.get_epoch());
+  ASSERT_FALSE(c.has_next());
+}
+
+TEST(PeriodHistory, AttachBefore)
+{
+  NumericPuller puller;
+  RGWPeriodHistory history(g_ceph_context, &puller, current_period);
+
+  auto c1 = history.attach(make_period("1", 1, ""));
+  ASSERT_TRUE(c1);
+
+  // verify that we pulled and merged all periods from 1-5
+  auto c = history.get_current();
+  ASSERT_TRUE(c);
+  ASSERT_TRUE(c.has_prev());
+  c.prev();
+  ASSERT_EQ(4u, c.get_epoch());
+  ASSERT_TRUE(c.has_prev());
+  c.prev();
+  ASSERT_EQ(3u, c.get_epoch());
+  ASSERT_TRUE(c.has_prev());
+  c.prev();
+  ASSERT_EQ(2u, c.get_epoch());
+  ASSERT_TRUE(c.has_prev());
+  c.prev();
+  ASSERT_EQ(1u, c.get_epoch());
+  ASSERT_FALSE(c.has_prev());
+}
+
+TEST(PeriodHistory, AttachAfter)
+{
+  NumericPuller puller;
+  RGWPeriodHistory history(g_ceph_context, &puller, current_period);
+
+  auto c9 = history.attach(make_period("9", 9, "8"));
+  ASSERT_TRUE(c9);
+
+  // verify that we pulled and merged all periods from 5-9
+  auto c = history.get_current();
+  ASSERT_TRUE(c);
+  ASSERT_TRUE(c.has_next());
+  c.next();
+  ASSERT_EQ(6u, c.get_epoch());
+  ASSERT_TRUE(c.has_next());
+  c.next();
+  ASSERT_EQ(7u, c.get_epoch());
+  ASSERT_TRUE(c.has_next());
+  c.next();
+  ASSERT_EQ(8u, c.get_epoch());
+  ASSERT_TRUE(c.has_next());
+  c.next();
+  ASSERT_EQ(9u, c.get_epoch());
+  ASSERT_FALSE(c.has_next());
+}
+
+int main(int argc, char** argv)
+{
+  vector<const char*> args;
+  argv_to_vec(argc, (const char **)argv, args);
+
+  auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT,
+			 CODE_ENVIRONMENT_UTILITY,
+			 CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+  common_init_finish(g_ceph_context);
+
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/src/test/rgw/test_rgw_putobj.cc b/src/test/rgw/test_rgw_putobj.cc
new file mode 100644
index 00000000..91661f45
--- /dev/null
+++ b/src/test/rgw/test_rgw_putobj.cc
@@ -0,0 +1,196 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2018 Red Hat, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#include "rgw/rgw_putobj.h"
+#include <gtest/gtest.h>
+
+inline bufferlist string_buf(const char* buf) {
+  bufferlist bl;
+  bl.append(buffer::create_static(strlen(buf), (char*)buf));
+  return bl;
+}
+
+struct Op {
+  std::string data;
+  uint64_t offset;
+};
+inline bool operator==(const Op& lhs, const Op& rhs) {
+  return lhs.data == rhs.data && lhs.offset == rhs.offset;
+}
+inline std::ostream& operator<<(std::ostream& out, const Op& op) {
+  return out << "{off=" << op.offset << " data='" << op.data << "'}";
+}
+
+struct MockProcessor : rgw::putobj::DataProcessor {
+  std::vector<Op> ops;
+
+  int process(bufferlist&& data, uint64_t offset) override {
+    ops.push_back({data.to_str(), offset});
+    return {};
+  }
+};
+
+TEST(PutObj_Chunk, FlushHalf)
+{
+  MockProcessor mock;
+  rgw::putobj::ChunkProcessor chunk(&mock, 4);
+
+  ASSERT_EQ(0, chunk.process(string_buf("22"), 0));
+  ASSERT_TRUE(mock.ops.empty()); // no writes
+
+  ASSERT_EQ(0, chunk.process({}, 2)); // flush
+  ASSERT_EQ(2u, mock.ops.size());
+  EXPECT_EQ(Op({"22", 0}), mock.ops[0]);
+  EXPECT_EQ(Op({"", 2}), mock.ops[1]);
+}
+
+TEST(PutObj_Chunk, One)
+{
+  MockProcessor mock;
+  rgw::putobj::ChunkProcessor chunk(&mock, 4);
+
+  ASSERT_EQ(0, chunk.process(string_buf("4444"), 0));
+  ASSERT_EQ(1u, mock.ops.size());
+  EXPECT_EQ(Op({"4444", 0}), mock.ops[0]);
+
+  ASSERT_EQ(0, chunk.process({}, 4)); // flush
+  ASSERT_EQ(2u, mock.ops.size());
+  EXPECT_EQ(Op({"", 4}), mock.ops[1]);
+}
+
+TEST(PutObj_Chunk, OneAndFlushHalf)
+{
+  MockProcessor mock;
+  rgw::putobj::ChunkProcessor chunk(&mock, 4);
+
+  ASSERT_EQ(0, chunk.process(string_buf("22"), 0));
+  ASSERT_TRUE(mock.ops.empty());
+
+  ASSERT_EQ(0, chunk.process(string_buf("4444"), 2));
+  ASSERT_EQ(1u, mock.ops.size());
+  EXPECT_EQ(Op({"2244", 0}), mock.ops[0]);
+
+  ASSERT_EQ(0, chunk.process({}, 6)); // flush
+  ASSERT_EQ(3u, mock.ops.size());
+  EXPECT_EQ(Op({"44", 4}), mock.ops[1]);
+  EXPECT_EQ(Op({"", 6}), mock.ops[2]);
+}
+
+TEST(PutObj_Chunk, Two)
+{
+  MockProcessor mock;
+  rgw::putobj::ChunkProcessor chunk(&mock, 4);
+
+  ASSERT_EQ(0, chunk.process(string_buf("88888888"), 0));
+  ASSERT_EQ(2u, mock.ops.size());
+  EXPECT_EQ(Op({"8888", 0}), mock.ops[0]);
+  EXPECT_EQ(Op({"8888", 4}), mock.ops[1]);
+
+  ASSERT_EQ(0, chunk.process({}, 8)); // flush
+  ASSERT_EQ(3u, mock.ops.size());
+  EXPECT_EQ(Op({"", 8}), mock.ops[2]);
+}
+
+TEST(PutObj_Chunk, TwoAndFlushHalf)
+{
+  MockProcessor mock;
+  rgw::putobj::ChunkProcessor chunk(&mock, 4);
+
+  ASSERT_EQ(0, chunk.process(string_buf("22"), 0));
+  ASSERT_TRUE(mock.ops.empty());
+
+  ASSERT_EQ(0, chunk.process(string_buf("88888888"), 2));
+  ASSERT_EQ(2u, mock.ops.size());
+  EXPECT_EQ(Op({"2288", 0}), mock.ops[0]);
+  EXPECT_EQ(Op({"8888", 4}), mock.ops[1]);
+
+  ASSERT_EQ(0, chunk.process({}, 10)); // flush
+  ASSERT_EQ(4u, mock.ops.size());
+  EXPECT_EQ(Op({"88", 8}), mock.ops[2]);
+  EXPECT_EQ(Op({"", 10}), mock.ops[3]);
+}
+
+
+using StripeMap = std::map<uint64_t, uint64_t>; // offset, stripe_size
+
+class StripeMapGen : public rgw::putobj::StripeGenerator {
+  const StripeMap& stripes;
+ public:
+  StripeMapGen(const StripeMap& stripes) : stripes(stripes) {}
+
+  int next(uint64_t offset, uint64_t *stripe_size) override {
+    auto i = stripes.find(offset);
+    if (i == stripes.end()) {
+      return -ENOENT;
+    }
+    *stripe_size = i->second;
+    return 0;
+  }
+};
+
+TEST(PutObj_Stripe, DifferentStripeSize)
+{
+  MockProcessor mock;
+  StripeMap stripes{
+    { 0, 4},
+    { 4, 6},
+    {10, 2}
+  };
+  StripeMapGen gen(stripes);
+  rgw::putobj::StripeProcessor processor(&mock, &gen, stripes.begin()->second);
+
+  ASSERT_EQ(0, processor.process(string_buf("22"), 0));
+  ASSERT_EQ(1u, mock.ops.size());
+  EXPECT_EQ(Op({"22", 0}), mock.ops[0]);
+
+  ASSERT_EQ(0, processor.process(string_buf("4444"), 2));
+  ASSERT_EQ(4u, mock.ops.size());
+  EXPECT_EQ(Op({"44", 2}), mock.ops[1]);
+  EXPECT_EQ(Op({"", 4}), mock.ops[2]); // flush
+  EXPECT_EQ(Op({"44", 0}), mock.ops[3]);
+
+  ASSERT_EQ(0, processor.process(string_buf("666666"), 6));
+  ASSERT_EQ(7u, mock.ops.size());
+  EXPECT_EQ(Op({"6666", 2}), mock.ops[4]);
+  EXPECT_EQ(Op({"", 6}), mock.ops[5]); // flush
+  EXPECT_EQ(Op({"66", 0}), mock.ops[6]);
+
+  ASSERT_EQ(0, processor.process({}, 12));
+  ASSERT_EQ(8u, mock.ops.size());
+  EXPECT_EQ(Op({"", 2}), mock.ops[7]); // flush
+
+  // gen returns an error past this
+  ASSERT_EQ(-ENOENT, processor.process(string_buf("1"), 12));
+}
+
+TEST(PutObj_Stripe, SkipFirstChunk)
+{
+  MockProcessor mock;
+  StripeMap stripes{
+    {0, 4},
+    {4, 4},
+  };
+  StripeMapGen gen(stripes);
+  rgw::putobj::StripeProcessor processor(&mock, &gen, stripes.begin()->second);
+
+  ASSERT_EQ(0, processor.process(string_buf("666666"), 2));
+  ASSERT_EQ(3u, mock.ops.size());
+  EXPECT_EQ(Op({"66", 2}), mock.ops[0]);
+  EXPECT_EQ(Op({"", 4}), mock.ops[1]); // flush
+  EXPECT_EQ(Op({"6666", 0}), mock.ops[2]);
+
+  ASSERT_EQ(0, processor.process({}, 8));
+  ASSERT_EQ(4u, mock.ops.size());
+  EXPECT_EQ(Op({"", 4}), mock.ops[3]); // flush
+}
diff --git a/src/test/rgw/test_rgw_reshard_wait.cc b/src/test/rgw/test_rgw_reshard_wait.cc
new file mode 100644
index 00000000..e63f066d
--- /dev/null
+++ b/src/test/rgw/test_rgw_reshard_wait.cc
@@ -0,0 +1,165 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2018 Red Hat, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#include "rgw/rgw_reshard.h"
+
+#include <gtest/gtest.h>
+
+using namespace std::chrono_literals;
+using Clock = RGWReshardWait::Clock;
+
+TEST(ReshardWait, wait_block)
+{
+  constexpr ceph::timespan wait_duration = 10ms;
+  RGWReshardWait waiter(wait_duration);
+
+  const auto start = Clock::now();
+  EXPECT_EQ(0, waiter.wait(null_yield));
+  const ceph::timespan elapsed = Clock::now() - start;
+
+  EXPECT_LE(wait_duration, elapsed); // waited at least 10ms
+  waiter.stop();
+}
+
+TEST(ReshardWait, stop_block)
+{
+  constexpr ceph::timespan short_duration = 10ms;
+  constexpr ceph::timespan long_duration = 10s;
+
+  RGWReshardWait long_waiter(long_duration);
+  RGWReshardWait short_waiter(short_duration);
+
+  const auto start = Clock::now();
+  std::thread thread([&long_waiter] {
+    EXPECT_EQ(-ECANCELED, long_waiter.wait(null_yield));
+  });
+
+  EXPECT_EQ(0, short_waiter.wait(null_yield));
+
+  long_waiter.stop(); // cancel long waiter
+
+  thread.join();
+  const ceph::timespan elapsed = Clock::now() - start;
+
+  EXPECT_LE(short_duration, elapsed); // waited at least 10ms
+  EXPECT_GT(long_duration, elapsed); // waited less than 10s
+  short_waiter.stop();
+}
+
+#ifdef HAVE_BOOST_CONTEXT
+TEST(ReshardWait, wait_yield)
+{
+  constexpr ceph::timespan wait_duration = 10ms;
+  RGWReshardWait waiter(wait_duration);
+
+  boost::asio::io_context context;
+  boost::asio::spawn(context, [&] (boost::asio::yield_context yield) {
+      EXPECT_EQ(0, waiter.wait(optional_yield{context, yield}));
+    });
+
+  const auto start = Clock::now();
+  EXPECT_EQ(1u, context.poll()); // spawn
+  EXPECT_FALSE(context.stopped());
+
+  EXPECT_EQ(1u, context.run_one()); // timeout
+  EXPECT_TRUE(context.stopped());
+  const ceph::timespan elapsed = Clock::now() - start;
+
+  EXPECT_LE(wait_duration, elapsed); // waited at least 10ms
+  waiter.stop();
+}
+
+TEST(ReshardWait, stop_yield)
+{
+  constexpr ceph::timespan short_duration = 10ms;
+  constexpr ceph::timespan long_duration = 10s;
+
+  RGWReshardWait long_waiter(long_duration);
+  RGWReshardWait short_waiter(short_duration);
+
+  boost::asio::io_context context;
+  boost::asio::spawn(context,
+    [&] (boost::asio::yield_context yield) {
+      EXPECT_EQ(-ECANCELED, long_waiter.wait(optional_yield{context, yield}));
+    });
+
+  const auto start = Clock::now();
+  EXPECT_EQ(1u, context.poll()); // spawn
+  EXPECT_FALSE(context.stopped());
+
+  EXPECT_EQ(0, short_waiter.wait(null_yield));
+
+  long_waiter.stop(); // cancel long waiter
+
+  EXPECT_EQ(1u, context.run_one_for(short_duration)); // timeout
+  EXPECT_TRUE(context.stopped());
+  const ceph::timespan elapsed = Clock::now() - start;
+
+  EXPECT_LE(short_duration, elapsed); // waited at least 10ms
+  EXPECT_GT(long_duration, elapsed); // waited less than 10s
+  short_waiter.stop();
+}
+
+TEST(ReshardWait, stop_multiple)
+{
+  constexpr ceph::timespan short_duration = 10ms;
+  constexpr ceph::timespan long_duration = 10s;
+
+  RGWReshardWait long_waiter(long_duration);
+  RGWReshardWait short_waiter(short_duration);
+
+  // spawn 4 threads
+  std::vector<std::thread> threads;
+  {
+    auto sync_waiter([&long_waiter] {
+      EXPECT_EQ(-ECANCELED, long_waiter.wait(null_yield));
+    });
+    threads.emplace_back(sync_waiter);
+    threads.emplace_back(sync_waiter);
+    threads.emplace_back(sync_waiter);
+    threads.emplace_back(sync_waiter);
+  }
+  // spawn 4 coroutines
+  boost::asio::io_context context;
+  {
+    auto async_waiter = [&] (boost::asio::yield_context yield) {
+        EXPECT_EQ(-ECANCELED, long_waiter.wait(optional_yield{context, yield}));
+      };
+    boost::asio::spawn(context, async_waiter);
+    boost::asio::spawn(context, async_waiter);
+    boost::asio::spawn(context, async_waiter);
+    boost::asio::spawn(context, async_waiter);
+  }
+
+  const auto start = Clock::now();
+  EXPECT_EQ(4u, context.poll()); // spawn
+  EXPECT_FALSE(context.stopped());
+
+  EXPECT_EQ(0, short_waiter.wait(null_yield));
+
+  long_waiter.stop(); // cancel long waiter
+
+  EXPECT_EQ(4u, context.run_for(short_duration)); // timeout
+  EXPECT_TRUE(context.stopped());
+
+  for (auto& thread : threads) {
+    thread.join();
+  }
+  const ceph::timespan elapsed = Clock::now() - start;
+
+  EXPECT_LE(short_duration, elapsed); // waited at least 10ms
+  EXPECT_GT(long_duration, elapsed); // waited less than 10s
+  short_waiter.stop();
+}
+#endif // HAVE_BOOST_CONTEXT
diff --git a/src/test/rgw/test_rgw_string.cc b/src/test/rgw/test_rgw_string.cc
new file mode 100644
index 00000000..96dc8a9b
--- /dev/null
+++ b/src/test/rgw/test_rgw_string.cc
@@ -0,0 +1,76 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2017 Red Hat
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#include "rgw/rgw_string.h"
+#include <gtest/gtest.h>
+
+const std::string abc{"abc"};
+const char *def{"def"}; // const char*
+char ghi_arr[] = {'g', 'h', 'i', '\0'};
+char *ghi{ghi_arr}; // char*
+constexpr boost::string_view jkl{"jkl", 3};
+#define mno "mno" // string literal (char[4])
+char pqr[] = {'p', 'q', 'r', '\0'};
+
+TEST(string_size, types)
+{
+  ASSERT_EQ(3u, string_size(abc));
+  ASSERT_EQ(3u, string_size(def));
+  ASSERT_EQ(3u, string_size(ghi));
+  ASSERT_EQ(3u, string_size(jkl));
+  ASSERT_EQ(3u, string_size(mno));
+  ASSERT_EQ(3u, string_size(pqr));
+
+  constexpr auto compile_time_string_view_size = string_size(jkl);
+  ASSERT_EQ(3u, compile_time_string_view_size);
+  constexpr auto compile_time_string_literal_size = string_size(mno);
+  ASSERT_EQ(3u, compile_time_string_literal_size);
+
+  char arr[] = {'a', 'b', 'c'}; // not null-terminated
+  ASSERT_THROW(string_size(arr), std::invalid_argument);
+}
+
+TEST(string_cat_reserve, types)
+{
+  ASSERT_EQ("abcdefghijklmnopqr",
+            string_cat_reserve(abc, def, ghi, jkl, mno, pqr));
+}
+
+TEST(string_cat_reserve, count)
+{
+  ASSERT_EQ("", string_cat_reserve());
+  ASSERT_EQ("abc", string_cat_reserve(abc));
+  ASSERT_EQ("abcdef", string_cat_reserve(abc, def));
+}
+
+TEST(string_join_reserve, types)
+{
+  ASSERT_EQ("abc, def, ghi, jkl, mno, pqr",
+            string_join_reserve(", ", abc, def, ghi, jkl, mno, pqr));
+}
+
+TEST(string_join_reserve, count)
+{
+  ASSERT_EQ("", string_join_reserve(", "));
+  ASSERT_EQ("abc", string_join_reserve(", ", abc));
+  ASSERT_EQ("abc, def", string_join_reserve(", ", abc, def));
+}
+
+TEST(string_join_reserve, delim)
+{
+  ASSERT_EQ("abcdef", string_join_reserve("", abc, def));
+  ASSERT_EQ("abc def", string_join_reserve(' ', abc, def));
+  ASSERT_EQ("abc\ndef", string_join_reserve('\n', abc, def));
+  ASSERT_EQ("abcfoodef", string_join_reserve(std::string{"foo"}, abc, def));
+}
diff --git a/src/test/rgw/test_rgw_throttle.cc b/src/test/rgw/test_rgw_throttle.cc
new file mode 100644
index 00000000..c1b97a04
--- /dev/null
+++ b/src/test/rgw/test_rgw_throttle.cc
@@ -0,0 +1,123 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2018 Red Hat, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#include "rgw/rgw_aio_throttle.h"
+#include "rgw/rgw_rados.h"
+
+#include "include/rados/librados.hpp"
+
+#include <gtest/gtest.h>
+
+struct RadosEnv : public ::testing::Environment {
+ public:
+  static constexpr auto poolname = "ceph_test_rgw_throttle";
+
+  static std::optional<RGWSI_RADOS> rados;
+
+  void SetUp() override {
+    rados.emplace(g_ceph_context);
+    ASSERT_EQ(0, rados->start());
+    int r = rados->pool({poolname}).create();
+    if (r == -EEXIST)
+      r = 0;
+    ASSERT_EQ(0, r);
+  }
+  void TearDown() {
+    rados.reset();
+  }
+};
+std::optional<RGWSI_RADOS> RadosEnv::rados;
+
+auto *const rados_env = ::testing::AddGlobalTestEnvironment(new RadosEnv);
+
+// test fixture for global setup/teardown
+class RadosFixture : public ::testing::Test {
+ protected:
+  RGWSI_RADOS::Obj make_obj(const std::string& oid) {
+    auto obj = RadosEnv::rados->obj({{RadosEnv::poolname}, oid});
+    ceph_assert_always(0 == obj.open());
+    return obj;
+  }
+};
+
+using Aio_Throttle = RadosFixture;
+
+namespace rgw {
+
+TEST_F(Aio_Throttle, NoThrottleUpToMax)
+{
+  AioThrottle throttle(4);
+  auto obj = make_obj(__PRETTY_FUNCTION__);
+  {
+    librados::ObjectWriteOperation op1;
+    auto c1 = throttle.submit(obj, &op1, 1, 0);
+    EXPECT_TRUE(c1.empty());
+    librados::ObjectWriteOperation op2;
+    auto c2 = throttle.submit(obj, &op2, 1, 0);
+    EXPECT_TRUE(c2.empty());
+    librados::ObjectWriteOperation op3;
+    auto c3 = throttle.submit(obj, &op3, 1, 0);
+    EXPECT_TRUE(c3.empty());
+    librados::ObjectWriteOperation op4;
+    auto c4 = throttle.submit(obj, &op4, 1, 0);
+    EXPECT_TRUE(c4.empty());
+    // no completions because no ops had to wait
+    auto c5 = throttle.poll();
+  }
+  auto completions = throttle.drain();
+  ASSERT_EQ(4u, completions.size());
+  for (auto& c : completions) {
+    EXPECT_EQ(-EINVAL, c.result);
+  }
+}
+
+TEST_F(Aio_Throttle, CostOverWindow)
+{
+  AioThrottle throttle(4);
+  auto obj = make_obj(__PRETTY_FUNCTION__);
+
+  librados::ObjectWriteOperation op;
+  auto c = throttle.submit(obj, &op, 8, 0);
+  ASSERT_EQ(1u, c.size());
+  EXPECT_EQ(-EDEADLK, c.front().result);
+}
+
+TEST_F(Aio_Throttle, ThrottleOverMax)
+{
+  constexpr uint64_t window = 4;
+  AioThrottle throttle(window);
+
+  auto obj = make_obj(__PRETTY_FUNCTION__);
+
+  // issue 32 writes, and verify that max_outstanding <= window
+  constexpr uint64_t total = 32;
+  uint64_t max_outstanding = 0;
+  uint64_t outstanding = 0;
+
+  for (uint64_t i = 0; i < total; i++) {
+    librados::ObjectWriteOperation op;
+    auto c = throttle.submit(obj, &op, 1, 0);
+    outstanding++;
+    outstanding -= c.size();
+    if (max_outstanding < outstanding) {
+      max_outstanding = outstanding;
+    }
+  }
+  auto c = throttle.drain();
+  outstanding -= c.size();
+  EXPECT_EQ(0u, outstanding);
+  EXPECT_EQ(window, max_outstanding);
+}
+
+} // namespace rgw
diff --git a/src/test/rgw/test_rgw_url.cc b/src/test/rgw/test_rgw_url.cc
new file mode 100644
index 00000000..d38457b8
--- /dev/null
+++ b/src/test/rgw/test_rgw_url.cc
@@ -0,0 +1,99 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "rgw/rgw_url.h"
+#include <string>
+#include <gtest/gtest.h>
+
+using namespace rgw;
+
+TEST(TestURL, SimpleAuthority)
+{
+    std::string host;
+    std::string user;
+    std::string password;
+    const std::string url = "http://example.com";
+    ASSERT_TRUE(parse_url_authority(url, host, user, password));
+    ASSERT_TRUE(user.empty());
+    ASSERT_TRUE(password.empty());
+    EXPECT_STREQ(host.c_str(), "example.com"); 
+}
+
+TEST(TestURL, IPAuthority)
+{
+    std::string host;
+    std::string user;
+    std::string password;
+    const std::string url = "http://1.2.3.4";
+    ASSERT_TRUE(parse_url_authority(url, host, user, password));
+    ASSERT_TRUE(user.empty());
+    ASSERT_TRUE(password.empty());
+    EXPECT_STREQ(host.c_str(), "1.2.3.4"); 
+}
+
+TEST(TestURL, IPv6Authority)
+{
+    std::string host;
+    std::string user;
+    std::string password;
+    const std::string url = "http://FE80:CD00:0000:0CDE:1257:0000:211E:729C";
+    ASSERT_TRUE(parse_url_authority(url, host, user, password));
+    ASSERT_TRUE(user.empty());
+    ASSERT_TRUE(password.empty());
+    EXPECT_STREQ(host.c_str(), "FE80:CD00:0000:0CDE:1257:0000:211E:729C"); 
+}
+
+TEST(TestURL, AuthorityWithUserinfo)
+{
+    std::string host;
+    std::string user;
+    std::string password;
+    const std::string url = "https://user:password@example.com";
+    ASSERT_TRUE(parse_url_authority(url, host, user, password));
+    EXPECT_STREQ(host.c_str(), "example.com"); 
+    EXPECT_STREQ(user.c_str(), "user"); 
+    EXPECT_STREQ(password.c_str(), "password"); 
+}
+
+TEST(TestURL, AuthorityWithPort)
+{
+    std::string host;
+    std::string user;
+    std::string password;
+    const std::string url = "http://user:password@example.com:1234";
+    ASSERT_TRUE(parse_url_authority(url, host, user, password));
+    EXPECT_STREQ(host.c_str(), "example.com:1234"); 
+    EXPECT_STREQ(user.c_str(), "user"); 
+    EXPECT_STREQ(password.c_str(), "password"); 
+}
+
+TEST(TestURL, DifferentSchema)
+{
+    std::string host;
+    std::string user;
+    std::string password;
+    const std::string url = "kafka://example.com";
+    ASSERT_TRUE(parse_url_authority(url, host, user, password));
+    ASSERT_TRUE(user.empty());
+    ASSERT_TRUE(password.empty());
+    EXPECT_STREQ(host.c_str(), "example.com"); 
+}
+
+TEST(TestURL, InvalidHost)
+{
+    std::string host;
+    std::string user;
+    std::string password;
+    const std::string url = "http://exa_mple.com";
+    ASSERT_FALSE(parse_url_authority(url, host, user, password));
+}
+
+TEST(TestURL, WithPath)
+{
+    std::string host;
+    std::string user;
+    std::string password;
+    const std::string url = "amqps://www.example.com:1234/vhost_name";
+    ASSERT_TRUE(parse_url_authority(url, host, user, password));
+}
+
diff --git a/src/test/rgw/test_rgw_xml.cc b/src/test/rgw/test_rgw_xml.cc
new file mode 100644
index 00000000..50c42c7a
--- /dev/null
+++ b/src/test/rgw/test_rgw_xml.cc
@@ -0,0 +1,463 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "rgw/rgw_xml.h"
+#include <gtest/gtest.h>
+#include <list>
+#include <stdexcept>
+
+struct NameAndStatus {
+  // these are sub-tags
+  std::string name;
+  bool status;
+  
+  // intrusive XML decoding API
+  bool decode_xml(XMLObj *obj) {
+    if (!RGWXMLDecoder::decode_xml("Name", name, obj, true)) {
+      // name is mandatory
+      return false;
+    }
+    if (!RGWXMLDecoder::decode_xml("Status", status, obj, false)) {
+      // status is optional and defaults to True
+      status = true;
+    }
+    return true;
+  }
+};
+
+struct Item {
+  // these are sub-tags
+  NameAndStatus name_and_status;
+  int value;
+  int extra_value;
+
+  // these are attributes
+  std::string date;
+  std::string comment;
+ 
+  // intrusive XML decoding API
+  bool decode_xml(XMLObj *obj) {
+    if (!RGWXMLDecoder::decode_xml("NameAndStatus", name_and_status, obj, true)) {
+      // name amd status are mandatory
+      return false;
+    }
+    if (!RGWXMLDecoder::decode_xml("Value", value, obj, true)) {
+      // value is mandatory
+      return false;
+    }
+    if (!RGWXMLDecoder::decode_xml("ExtraValue", extra_value, obj, false)) {
+      // extra value is optional and defaults to zero
+      extra_value = 0;
+    }
+
+    // date attribute is optional
+    if (!obj->get_attr("Date", date)) {
+      date = "no date";
+    }
+    // comment attribute is optional
+    if (!obj->get_attr("Comment", comment)) {
+      comment = "no comment";
+    }
+
+    return true;
+  }
+};
+
+struct Items {
+  // these are sub-tags
+  std::list<Item> item_list;
+  
+  // intrusive XML decoding API
+  bool decode_xml(XMLObj *obj) {
+    do_decode_xml_obj(item_list, "Item", obj);
+    return true;
+  }
+};
+
+// in case of non-intrusive decoding class
+// hierarchy should reflect the XML hierarchy
+
+class NameXMLObj: public XMLObj {
+protected:
+  void xml_handle_data(const char *s, int len) override {
+    // no need to set "data", setting "name" directly
+    value.append(s, len);
+  }
+
+public:
+  std::string value;
+  ~NameXMLObj() override = default;
+};
+
+class StatusXMLObj: public XMLObj {
+protected:
+  void xml_handle_data(const char *s, int len) override {
+    std::istringstream is(std::string(s, len));
+    is >> std::boolalpha >> value;
+  }
+
+public:
+  bool value;
+  ~StatusXMLObj() override = default;
+};
+
+class NameAndStatusXMLObj: public NameAndStatus, public XMLObj {
+public:
+  ~NameAndStatusXMLObj() override = default;
+
+  bool xml_end(const char *el) override {
+    XMLObjIter iter = find("Name");
+    NameXMLObj* _name = static_cast<NameXMLObj*>(iter.get_next());
+    if (!_name) {
+      // name is mandatory
+      return false;
+    }
+    name = _name->value;
+    iter = find("Status");
+    StatusXMLObj* _status = static_cast<StatusXMLObj*>(iter.get_next());
+    if (!_status) {
+      // status is optional and defaults to True
+      status = true;
+    } else {
+      status = _status->value;
+    }
+    return true;
+  }
+};
+
+class ItemXMLObj: public Item, public XMLObj {
+public:
+  ~ItemXMLObj() override = default;
+  
+  bool xml_end(const char *el) override {
+    XMLObjIter iter = find("NameAndStatus");
+    NameAndStatusXMLObj* _name_and_status = static_cast<NameAndStatusXMLObj*>(iter.get_next());
+    if (!_name_and_status) {
+      // name and status are mandatory
+      return false;
+    }
+    name_and_status = *static_cast<NameAndStatus*>(_name_and_status);
+    iter = find("Value");
+    XMLObj* _value = iter.get_next();
+    if (!_value) {
+      // value is mandatory
+      return false;
+    }
+    try {
+      value = std::stoi(_value->get_data());
+    } catch (const std::exception& e) {
+      return false;
+    }
+    iter = find("ExtraValue");
+    XMLObj* _extra_value = iter.get_next();
+    if (_extra_value) {
+      // extra value is optional but cannot contain garbage
+      try {
+        extra_value = std::stoi(_extra_value->get_data());
+      } catch (const std::exception& e) {
+        return false;
+      }
+    } else {
+      // if not set, it defaults to zero
+      extra_value = 0;
+    }
+
+    // date attribute is optional
+    if (!get_attr("Date", date)) {
+      date = "no date";
+    }
+    // comment attribute is optional
+    if (!get_attr("Comment", comment)) {
+      comment = "no comment";
+    }
+
+    return true;
+  }
+};
+
+class ItemsXMLObj: public Items, public XMLObj {
+public:
+  ~ItemsXMLObj() override = default;
+
+  bool xml_end(const char *el) override {
+    XMLObjIter iter = find("Item");
+    ItemXMLObj* item_ptr = static_cast<ItemXMLObj*>(iter.get_next());
+    // mandatory to have at least one item
+    bool item_found = false;
+    while (item_ptr) {
+      item_list.push_back(*static_cast<Item*>(item_ptr));
+      item_ptr = static_cast<ItemXMLObj*>(iter.get_next());
+      item_found = true;
+    }
+    return item_found;
+  }
+};
+
+class ItemsXMLParser: public RGWXMLParser {
+  static const int MAX_NAME_LEN = 16;
+public:
+  XMLObj *alloc_obj(const char *el) override {
+    if (strncmp(el, "Items", MAX_NAME_LEN) == 0) {
+      items = new ItemsXMLObj;
+      return items;
+    } else if (strncmp(el, "Item", MAX_NAME_LEN) == 0) {
+      return new ItemXMLObj;
+    } else if (strncmp(el, "NameAndStatus", MAX_NAME_LEN) == 0) {
+      return new NameAndStatusXMLObj;
+    } else if (strncmp(el, "Name", MAX_NAME_LEN) == 0) {
+      return new NameXMLObj;
+    } else if (strncmp(el, "Status", MAX_NAME_LEN) == 0) {
+      return new StatusXMLObj;
+    }
+    return nullptr;
+  }
+  // this is a pointer to the parsed results
+  ItemsXMLObj* items;
+};
+
+static const char* good_input = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
+                           "<Items>"
+                             "<Item><NameAndStatus><Name>hello</Name></NameAndStatus><Value>1</Value></Item>"
+                             "<Item><ExtraValue>99</ExtraValue><NameAndStatus><Name>world</Name></NameAndStatus><Value>2</Value></Item>"
+                             "<Item><Value>3</Value><NameAndStatus><Name>foo</Name></NameAndStatus></Item>"
+                             "<Item><Value>4</Value><ExtraValue>42</ExtraValue><NameAndStatus><Name>bar</Name><Status>False</Status></NameAndStatus></Item>"
+                           "</Items>";
+
+static const char* expected_output = "((hello,1),1,0),((world,1),2,99),((foo,1),3,0),((bar,0),4,42),";
+
+std::string to_string(const Items& items) {
+  std::stringstream ss;
+  for (const auto& item : items.item_list) {
+    ss << "((" << item.name_and_status.name << "," << item.name_and_status.status << ")," << item.value << "," << item.extra_value << ")" << ",";
+  }
+  return ss.str();
+}
+
+std::string to_string_with_attributes(const Items& items) {
+  std::stringstream ss;
+  for (const auto& item : items.item_list) {
+    ss << "(" << item.date << "," << item.comment << ",(" << item.name_and_status.name << "," << item.name_and_status.status << ")," 
+      << item.value << "," << item.extra_value << ")" << ",";
+  }
+  return ss.str();
+}
+
+TEST(TestParser, BasicParsing)
+{  
+  ItemsXMLParser parser;
+  ASSERT_TRUE(parser.init());
+  ASSERT_TRUE(parser.parse(good_input, strlen(good_input), 1));
+  ASSERT_EQ(parser.items->item_list.size(), 4U);
+  ASSERT_STREQ(to_string(*parser.items).c_str(), expected_output);
+}
+
+static const char* malformed_input = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
+                           "<Items>"
+                             "<Item><NameAndStatus><Name>hello</Name></NameAndStatus><Value>1</Value><Item>"
+                             "<Item><ExtraValue>99</ExtraValue><NameAndStatus><Name>world</Name></NameAndStatus><Value>2</Value></Item>"
+                             "<Item><Value>3</Value><NameAndStatus><Name>foo</Name></NameAndStatus></Item>"
+                             "<Item><Value>4</Value><ExtraValue>42</ExtraValue><NameAndStatus><Name>bar</Name><Status>False</Status></NameAndStatus></Item>"
+                           "</Items>";
+
+TEST(TestParser, MalformedInput)
+{  
+  ItemsXMLParser parser;
+  ASSERT_TRUE(parser.init());
+  ASSERT_FALSE(parser.parse(good_input, strlen(malformed_input), 1));
+}
+
+static const char* missing_value_input = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
+                           "<Items>"
+                             "<Item><NameAndStatus><Name>hello</Name></NameAndStatus><Value>1</Value></Item>"
+                             "<Item><ExtraValue>99</ExtraValue><NameAndStatus><Name>world</Name></NameAndStatus><Value>2</Value></Item>"
+                             "<Item><Value>3</Value><NameAndStatus><Name>foo</Name></NameAndStatus></Item>"
+                             "<Item><ExtraValue>42</ExtraValue><NameAndStatus><Name>bar</Name><Status>False</Status></NameAndStatus></Item>"
+                           "</Items>";
+
+TEST(TestParser, MissingMandatoryTag)
+{  
+  ItemsXMLParser parser;
+  ASSERT_TRUE(parser.init());
+  ASSERT_FALSE(parser.parse(missing_value_input, strlen(missing_value_input), 1));
+}
+
+static const char* unknown_tag_input = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
+                           "<Items>"
+                             "<Item><NameAndStatus><Name>hello</Name></NameAndStatus><Value>1</Value></Item>"
+                             "<Item><ExtraValue>99</ExtraValue><NameAndStatus><Name>world</Name></NameAndStatus><Value>2</Value></Item>"
+                             "<Item><Value>3</Value><NameAndStatus><Name>foo</Name></NameAndStatus><Kaboom>0</Kaboom></Item>"
+                             "<Item><Value>4</Value><ExtraValue>42</ExtraValue><NameAndStatus><Name>bar</Name><Status>False</Status></NameAndStatus></Item>"
+                             "<Kaboom>0</Kaboom>"
+                           "</Items>";
+
+TEST(TestParser, UnknownTag)
+{  
+  ItemsXMLParser parser;
+  ASSERT_TRUE(parser.init());
+  ASSERT_TRUE(parser.parse(unknown_tag_input, strlen(unknown_tag_input), 1));
+  ASSERT_EQ(parser.items->item_list.size(), 4U);
+  ASSERT_STREQ(to_string(*parser.items).c_str(), expected_output);
+}
+
+static const char* invalid_value_input = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
+                           "<Items>"
+                             "<Item><NameAndStatus><Name>hello</Name></NameAndStatus><Value>1</Value></Item>"
+                             "<Item><ExtraValue>kaboom</ExtraValue><NameAndStatus><Name>world</Name></NameAndStatus><Value>2</Value></Item>"
+                             "<Item><Value>3</Value><NameAndStatus><Name>foo</Name></NameAndStatus></Item>"
+                             "<Item><Value>4</Value><ExtraValue>42</ExtraValue><NameAndStatus><Name>bar</Name><Status>False</Status></NameAndStatus></Item>"
+                           "</Items>";
+
+TEST(TestParser, InvalidValue)
+{  
+  ItemsXMLParser parser;
+  ASSERT_TRUE(parser.init());
+  ASSERT_FALSE(parser.parse(invalid_value_input, strlen(invalid_value_input), 1));
+}
+
+static const char* good_input1 = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
+                           "<Items>"
+                             "<Item><NameAndStatus><Name>hello</Name></NameAndStatus><Value>1</Value></Item>"
+                             "<Item><ExtraValue>99</ExtraValue><NameAndStatus><Name>world</Name>";
+
+static const char* good_input2 = "</NameAndStatus><Value>2</Value></Item>"
+                             "<Item><Value>3</Value><NameAndStatus><Name>foo</Name></NameAndStatus></Item>"
+                             "<Item><Value>4</Value><ExtraValue>42</ExtraValue><NameAndStatus><Name>bar</Name><Status>False</Status></NameAndStatus></Item>"
+                           "</Items>";
+
+TEST(TestParser, MultipleChunks)
+{  
+  ItemsXMLParser parser;
+  ASSERT_TRUE(parser.init());
+  ASSERT_TRUE(parser.parse(good_input1, strlen(good_input1), 0));
+  ASSERT_TRUE(parser.parse(good_input2, strlen(good_input2), 1));
+  ASSERT_EQ(parser.items->item_list.size(), 4U);
+  ASSERT_STREQ(to_string(*parser.items).c_str(), expected_output);
+}
+
+static const char* input_with_attributes = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
+                           "<Items>"
+                             "<Item Date=\"Tue Dec 27 17:21:29 2011\" Kaboom=\"just ignore\">"
+                               "<NameAndStatus><Name>hello</Name></NameAndStatus><Value>1</Value>"
+                             "</Item>"
+                             "<Item Comment=\"hello world\">"
+                               "<ExtraValue>99</ExtraValue><NameAndStatus><Name>world</Name></NameAndStatus><Value>2</Value>"
+                             "</Item>"
+                             "<Item><Value>3</Value><NameAndStatus><Name>foo</Name></NameAndStatus></Item>"
+                             "<Item Comment=\"goodbye\" Date=\"Thu Feb 28 10:00:18 UTC 2019 \">"
+                               "<Value>4</Value><ExtraValue>42</ExtraValue><NameAndStatus><Name>bar</Name><Status>False</Status></NameAndStatus>"
+                             "</Item>"
+                           "</Items>";
+
+static const char* expected_output_with_attributes = "(Tue Dec 27 17:21:29 2011,no comment,(hello,1),1,0),"
+                                                     "(no date,hello world,(world,1),2,99),"
+                                                     "(no date,no comment,(foo,1),3,0),"
+                                                     "(Thu Feb 28 10:00:18 UTC 2019 ,goodbye,(bar,0),4,42),";
+
+TEST(TestParser, Attributes)
+{  
+  ItemsXMLParser parser;
+  ASSERT_TRUE(parser.init());
+  ASSERT_TRUE(parser.parse(input_with_attributes, strlen(input_with_attributes), 1));
+  ASSERT_EQ(parser.items->item_list.size(), 4U);
+  ASSERT_STREQ(to_string_with_attributes(*parser.items).c_str(), 
+      expected_output_with_attributes);
+}
+
+TEST(TestDecoder, BasicParsing)
+{
+  RGWXMLDecoder::XMLParser parser;
+  ASSERT_TRUE(parser.init());
+  ASSERT_TRUE(parser.parse(good_input, strlen(good_input), 1));
+  Items result;
+  ASSERT_NO_THROW({
+    ASSERT_TRUE(RGWXMLDecoder::decode_xml("Items", result, &parser, true));
+  });
+  ASSERT_EQ(result.item_list.size(), 4U);
+  ASSERT_STREQ(to_string(result).c_str(), expected_output);
+}
+
+TEST(TestDecoder, MalfomedInput)
+{
+  RGWXMLDecoder::XMLParser parser;
+  ASSERT_TRUE(parser.init());
+  ASSERT_FALSE(parser.parse(good_input, strlen(malformed_input), 1));
+}
+
+TEST(TestDecoder, MissingMandatoryTag)
+{
+  RGWXMLDecoder::XMLParser parser;
+  ASSERT_TRUE(parser.init());
+  ASSERT_TRUE(parser.parse(missing_value_input, strlen(missing_value_input), 1));
+  Items result;
+  ASSERT_ANY_THROW({
+    ASSERT_TRUE(RGWXMLDecoder::decode_xml("Items", result, &parser, true));
+  });
+}
+
+TEST(TestDecoder, InvalidValue)
+{
+  RGWXMLDecoder::XMLParser parser;
+  ASSERT_TRUE(parser.init());
+  ASSERT_TRUE(parser.parse(invalid_value_input, strlen(invalid_value_input), 1));
+  Items result;
+  ASSERT_ANY_THROW({
+    ASSERT_TRUE(RGWXMLDecoder::decode_xml("Items", result, &parser, true));
+  });
+}
+
+TEST(TestDecoder, MultipleChunks)
+{  
+  RGWXMLDecoder::XMLParser parser;
+  ASSERT_TRUE(parser.init());
+  ASSERT_TRUE(parser.parse(good_input1, strlen(good_input1), 0));
+  ASSERT_TRUE(parser.parse(good_input2, strlen(good_input2), 1));
+  Items result;
+  ASSERT_NO_THROW({
+    ASSERT_TRUE(RGWXMLDecoder::decode_xml("Items", result, &parser, true));
+  });
+  ASSERT_EQ(result.item_list.size(), 4U);
+  ASSERT_STREQ(to_string(result).c_str(), expected_output);
+}
+
+TEST(TestDecoder, Attributes)
+{  
+  RGWXMLDecoder::XMLParser parser;
+  ASSERT_TRUE(parser.init());
+  ASSERT_TRUE(parser.parse(input_with_attributes, strlen(input_with_attributes), 1));
+  Items result;
+  ASSERT_NO_THROW({
+    ASSERT_TRUE(RGWXMLDecoder::decode_xml("Items", result, &parser, true));
+  });
+  ASSERT_EQ(result.item_list.size(), 4U);
+  ASSERT_STREQ(to_string_with_attributes(result).c_str(), 
+      expected_output_with_attributes);
+}
+
+static const char* expected_xml_output = "<Items xmlns=\"https://www.ceph.com/doc/\">"
+                             "<Item Order=\"0\"><NameAndStatus><Name>hello</Name><Status>True</Status></NameAndStatus><Value>0</Value></Item>"
+                             "<Item Order=\"1\"><NameAndStatus><Name>hello</Name><Status>False</Status></NameAndStatus><Value>1</Value></Item>"
+                             "<Item Order=\"2\"><NameAndStatus><Name>hello</Name><Status>True</Status></NameAndStatus><Value>2</Value></Item>"
+                             "<Item Order=\"3\"><NameAndStatus><Name>hello</Name><Status>False</Status></NameAndStatus><Value>3</Value></Item>"
+                             "<Item Order=\"4\"><NameAndStatus><Name>hello</Name><Status>True</Status></NameAndStatus><Value>4</Value></Item>"
+                           "</Items>";
+TEST(TestEncoder, ListWithAttrsAndNS)
+{
+  XMLFormatter f;
+  const auto array_size = 5;
+  f.open_array_section_in_ns("Items", "https://www.ceph.com/doc/");
+  for (auto i = 0; i < array_size; ++i) {
+    FormatterAttrs item_attrs("Order", std::to_string(i).c_str(), NULL);
+    f.open_object_section_with_attrs("Item", item_attrs);
+    f.open_object_section("NameAndStatus");
+    encode_xml("Name", "hello", &f);
+    encode_xml("Status", (i%2 == 0), &f);
+    f.close_section();
+    encode_xml("Value", i, &f);
+    f.close_section();
+  }
+  f.close_section();
+  std::stringstream ss;
+  f.flush(ss);
+  ASSERT_STREQ(ss.str().c_str(), expected_xml_output);
+}
+