Adding upstream version 16.2.11+ds.upstream/16.2.11+dsupstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

22 files changed, 2134 insertions, 0 deletions

diff --git a/src/test/system/CMakeLists.txt b/src/test/system/CMakeLists.txt
new file mode 100644
index 000000000..f2b69fb3e
--- /dev/null
+++ b/src/test/system/CMakeLists.txt
@@ -0,0 +1,50 @@
+## System tests
+
+set(libsystest_srcs 
+  cross_process_sem.cc
+  systest_runnable.cc
+  systest_settings.cc
+  st_rados_create_pool.cc
+  st_rados_delete_pool.cc
+  st_rados_list_objects.cc)
+add_library(systest STATIC ${libsystest_srcs})
+
+if(NOT WIN32)
+  set(RT_LIB rt)
+endif()
+
+# test_rados_list_parallel
+add_executable(ceph_test_rados_list_parallel
+  rados_list_parallel.cc
+  )
+target_link_libraries(ceph_test_rados_list_parallel librados systest global pthread
+  ${RT_LIB} ${BLKID_LIBRARIES} ${CMAKE_DL_LIBS})
+
+# test_rados_open_pools_parallel
+set(test_rados_open_pools_parallel_srcs 
+  rados_open_pools_parallel.cc)
+add_executable(ceph_test_rados_open_pools_parallel
+  ${test_rados_open_pools_parallel_srcs}
+  )
+target_link_libraries(ceph_test_rados_open_pools_parallel
+  librados systest global
+  pthread ${RT_LIB} ${BLKID_LIBRARIES} ${CMAKE_DL_LIBS})
+
+# test_rados_delete_pools_parallel
+set(test_rados_delete_pools_parallel_srcs
+  rados_delete_pools_parallel.cc
+  st_rados_create_pool.cc
+  st_rados_delete_pool.cc
+  st_rados_list_objects.cc
+  )
+add_executable(ceph_test_rados_delete_pools_parallel
+  ${test_rados_delete_pools_parallel_srcs}
+  )
+target_link_libraries(ceph_test_rados_delete_pools_parallel librados systest global
+  pthread ${RT_LIB} ${BLKID_LIBRARIES} ${CMAKE_DL_LIBS})
+
+install(TARGETS
+  ceph_test_rados_delete_pools_parallel
+  ceph_test_rados_list_parallel
+  ceph_test_rados_open_pools_parallel
+  DESTINATION ${CMAKE_INSTALL_BINDIR})
diff --git a/src/test/system/cross_process_sem.cc b/src/test/system/cross_process_sem.cc
new file mode 100644
index 000000000..d73259d25
--- /dev/null
+++ b/src/test/system/cross_process_sem.cc
@@ -0,0 +1,122 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2011 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation.  See file COPYING.
+*
+*/
+
+#include "cross_process_sem.h"
+
+#include <errno.h>
+#include <semaphore.h>
+#include <stdlib.h>
+#ifndef _WIN32
+#include <sys/mman.h>
+#endif
+
+#include "include/ceph_assert.h"
+
+/* We put our cross-process semaphore into a page of memory mapped with mmap. */
+struct cross_process_sem_data_t
+{
+  sem_t sem;
+};
+
+/* A factory function is a good choice here because we want to be able to
+ * return an error code. It does force heap allocation, but that is the
+ * easiest way to use synchronization primitives anyway. Most programmers don't
+ * care about destroying semaphores before the process finishes. It's pretty
+ * difficult to get it right and there is usually no benefit.
+ */
+int CrossProcessSem::
+create(int initial_val, CrossProcessSem** res)
+{
+  #ifndef _WIN32
+  struct cross_process_sem_data_t *data = static_cast < cross_process_sem_data_t*> (
+    mmap(NULL, sizeof(struct cross_process_sem_data_t),
+       PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0));
+  if (data == MAP_FAILED) {
+    int err = errno;
+    return err;
+  }
+  int ret = sem_init(&data->sem, 1, initial_val);
+  #else
+  // We can't use multiple processes on Windows for the time being.
+  struct cross_process_sem_data_t *data = (cross_process_sem_data_t*)malloc(
+    sizeof(cross_process_sem_data_t));
+  int ret = sem_init(&data->sem, 0, initial_val);
+  #endif /* _WIN32 */
+  if (ret) {
+    return ret;
+  }
+  *res = new CrossProcessSem(data);
+  return 0;
+}
+
+CrossProcessSem::
+~CrossProcessSem()
+{
+  #ifndef _WIN32
+  munmap(m_data, sizeof(struct cross_process_sem_data_t));
+  #else
+  free(m_data);
+  #endif
+  m_data = NULL;
+}
+
+void CrossProcessSem::
+wait()
+{
+  while(true) {
+    int ret = sem_wait(&m_data->sem);
+    if (ret == 0)
+      return;
+    int err = errno;
+    if (err == -EINTR)
+      continue;
+    ceph_abort();
+  }
+}
+
+void CrossProcessSem::
+post()
+{
+  int ret = sem_post(&m_data->sem);
+  if (ret == -1) {
+    ceph_abort();
+  }
+}
+
+int CrossProcessSem::
+reinit(int dval)
+{
+  if (dval < 0)
+    return -EINVAL;
+  int cval;
+  if (sem_getvalue(&m_data->sem, &cval) == -1)
+    return errno;
+  if (cval < dval) {
+    int diff = dval - cval;
+    for (int i = 0; i < diff; ++i)
+      sem_post(&m_data->sem);
+  }
+  else {
+    int diff = cval - dval;
+    for (int i = 0; i < diff; ++i)
+      sem_wait(&m_data->sem);
+  }
+  return 0;
+}
+
+CrossProcessSem::
+CrossProcessSem(struct cross_process_sem_data_t *data)
+  : m_data(data)
+{
+}
diff --git a/src/test/system/cross_process_sem.h b/src/test/system/cross_process_sem.h
new file mode 100644
index 000000000..0cbedf475
--- /dev/null
+++ b/src/test/system/cross_process_sem.h
@@ -0,0 +1,40 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2011 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation.  See file COPYING.
+*
+*/
+
+struct cross_process_sem_data_t;
+
+class CrossProcessSem
+{
+public:
+  static int create(int initial_val, CrossProcessSem** ret);
+  ~CrossProcessSem();
+
+  /* Initialize the semaphore. Must be called before any operations */
+  int init();
+
+  /* Semaphore wait */
+  void wait();
+
+  /* Semaphore post */
+  void post();
+
+  /* Reinitialize the semaphore to the desired value.
+   * NOT thread-safe if it is in use at the time!
+   */ 
+  int reinit(int dval);
+
+private:
+  explicit CrossProcessSem(struct cross_process_sem_data_t *data);
+  struct cross_process_sem_data_t *m_data;
+};
diff --git a/src/test/system/rados_delete_pools_parallel.cc b/src/test/system/rados_delete_pools_parallel.cc
new file mode 100644
index 000000000..47d878c7c
--- /dev/null
+++ b/src/test/system/rados_delete_pools_parallel.cc
@@ -0,0 +1,110 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2011 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation.  See file COPYING.
+*
+*/
+
+#include "cross_process_sem.h"
+#include "include/rados/librados.h"
+#include "st_rados_create_pool.h"
+#include "st_rados_delete_pool.h"
+#include "st_rados_list_objects.h"
+#include "systest_runnable.h"
+#include "systest_settings.h"
+
+#include <errno.h>
+#include <pthread.h>
+#include <semaphore.h>
+#include <sstream>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string>
+#include <time.h>
+#include <vector>
+
+using std::ostringstream;
+using std::string;
+using std::vector;
+
+static int g_num_objects = 50;
+
+/*
+ * rados_delete_pools_parallel
+ *
+ * This tests creation and deletion races.
+ *
+ * EXPECT:            * can delete a pool while another user is using it
+ *                    * operations on pools return error codes after the pools
+ *                      are deleted
+ *
+ * DO NOT EXPECT      * hangs, crashes
+ */
+
+const char *get_id_str()
+{
+  return "main";
+}
+
+int main(int argc, const char **argv)
+{
+  const char *num_objects = getenv("NUM_OBJECTS");
+  const std::string pool = get_temp_pool_name(argv[0]);
+  if (num_objects) {
+    g_num_objects = atoi(num_objects); 
+    if (g_num_objects == 0)
+      return 100;
+  }
+
+  CrossProcessSem *pool_setup_sem = NULL;
+  RETURN1_IF_NONZERO(CrossProcessSem::create(0, &pool_setup_sem));
+  CrossProcessSem *delete_pool_sem = NULL;
+  RETURN1_IF_NONZERO(CrossProcessSem::create(0, &delete_pool_sem));
+
+  // first test: create a pool, then delete that pool
+  {
+    StRadosCreatePool r1(argc, argv, NULL, pool_setup_sem, NULL,
+			 pool, 50, ".obj");
+    StRadosDeletePool r2(argc, argv, pool_setup_sem, NULL, pool);
+    vector < SysTestRunnable* > vec;
+    vec.push_back(&r1);
+    vec.push_back(&r2);
+    std::string error = SysTestRunnable::run_until_finished(vec);
+    if (!error.empty()) {
+      printf("test1: got error: %s\n", error.c_str());
+      return EXIT_FAILURE;
+    }
+  }
+
+  // second test: create a pool, the list objects in that pool while it's
+  // being deleted.
+  RETURN1_IF_NONZERO(pool_setup_sem->reinit(0));
+  RETURN1_IF_NONZERO(delete_pool_sem->reinit(0));
+  {
+    StRadosCreatePool r1(argc, argv, NULL, pool_setup_sem, NULL,
+			 pool, g_num_objects, ".obj");
+    StRadosDeletePool r2(argc, argv, delete_pool_sem, NULL, pool);
+    StRadosListObjects r3(argc, argv, pool, true, g_num_objects / 2,
+			  pool_setup_sem, NULL, delete_pool_sem);
+    vector < SysTestRunnable* > vec;
+    vec.push_back(&r1);
+    vec.push_back(&r2);
+    vec.push_back(&r3);
+    std::string error = SysTestRunnable::run_until_finished(vec);
+    if (!error.empty()) {
+      printf("test2: got error: %s\n", error.c_str());
+      return EXIT_FAILURE;
+    }
+  }
+
+  printf("******* SUCCESS **********\n"); 
+  return EXIT_SUCCESS;
+}
diff --git a/src/test/system/rados_list_parallel.cc b/src/test/system/rados_list_parallel.cc
new file mode 100644
index 000000000..d6b3dfc05
--- /dev/null
+++ b/src/test/system/rados_list_parallel.cc
@@ -0,0 +1,348 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2011 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation.  See file COPYING.
+*
+*/
+
+#include "cross_process_sem.h"
+#include "include/rados/librados.h"
+#include "include/stringify.h"
+#include "st_rados_create_pool.h"
+#include "st_rados_list_objects.h"
+#include "systest_runnable.h"
+#include "systest_settings.h"
+
+#include <errno.h>
+#include <map>
+#include <pthread.h>
+#include <semaphore.h>
+#include <sstream>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string>
+#include <time.h>
+#include <vector>
+#include <sys/types.h>
+#include <unistd.h>
+
+using std::ostringstream;
+using std::string;
+using std::vector;
+
+static int g_num_objects = 50;
+
+static CrossProcessSem *pool_setup_sem = NULL;
+static CrossProcessSem *modify_sem = NULL;
+
+class RadosDeleteObjectsR : public SysTestRunnable
+{
+public:
+  RadosDeleteObjectsR(int argc, const char **argv,
+		      const std::string &pool_name)
+    : SysTestRunnable(argc, argv), m_pool_name(pool_name)
+  {
+  }
+
+  ~RadosDeleteObjectsR() override
+  {
+  }
+
+  int run(void) override
+  {
+    int ret_val = 0;
+    rados_t cl;
+    RETURN1_IF_NONZERO(rados_create(&cl, NULL));
+    rados_conf_parse_argv(cl, m_argc, m_argv);
+    RETURN1_IF_NONZERO(rados_conf_read_file(cl, NULL));
+    rados_conf_parse_env(cl, NULL);
+    std::string log_name = SysTestSettings::inst().get_log_name(get_id_str());
+    if (!log_name.empty())
+      rados_conf_set(cl, "log_file", log_name.c_str());
+    RETURN1_IF_NONZERO(rados_connect(cl));
+    pool_setup_sem->wait();
+    pool_setup_sem->post();
+
+    rados_ioctx_t io_ctx;
+    rados_pool_create(cl, m_pool_name.c_str());
+    RETURN1_IF_NONZERO(rados_ioctx_create(cl, m_pool_name.c_str(), &io_ctx));
+
+    std::map <int, std::string> to_delete;
+    for (int i = 0; i < g_num_objects; ++i) {
+      char oid[128];
+      snprintf(oid, sizeof(oid), "%d.obj", i);
+      to_delete[i] = oid;
+    }
+
+    int removed = 0;
+    while (true) {
+      if (to_delete.empty())
+	break;
+      int r = rand() % to_delete.size();
+      std::map <int, std::string>::iterator d = to_delete.begin();
+      for (int i = 0; i < r; ++i)
+	++d;
+      if (d == to_delete.end()) {
+	ret_val = -EDOM;
+	goto out;
+      }
+      std::string oid(d->second);
+      to_delete.erase(d);
+      int ret = rados_remove(io_ctx, oid.c_str());
+      if (ret != 0) {
+	printf("%s: rados_remove(%s) failed with error %d\n",
+	       get_id_str(), oid.c_str(), ret);
+	ret_val = ret;
+	goto out;
+      }
+      ++removed;
+      if ((removed % 25) == 0) {
+	printf("%s: removed %d objects...\n", get_id_str(), removed);
+      }
+      if (removed == g_num_objects / 2) {
+	printf("%s: removed half of the objects\n", get_id_str());
+	modify_sem->post();
+      }
+    }
+
+    printf("%s: removed %d objects\n", get_id_str(), removed);
+
+out:
+    rados_ioctx_destroy(io_ctx);
+    rados_shutdown(cl);
+
+    return ret_val;
+  }
+private:
+  std::string m_pool_name;
+};
+
+class RadosAddObjectsR : public SysTestRunnable
+{
+public:
+  RadosAddObjectsR(int argc, const char **argv,
+		   const std::string &pool_name,
+		   const std::string &suffix)
+    : SysTestRunnable(argc, argv),
+      m_pool_name(pool_name),
+      m_suffix(suffix)
+  {
+  }
+
+  ~RadosAddObjectsR() override
+  {
+  }
+
+  int run(void) override
+  {
+    int ret_val = 0;
+    rados_t cl;
+    RETURN1_IF_NONZERO(rados_create(&cl, NULL));
+    rados_conf_parse_argv(cl, m_argc, m_argv);
+    RETURN1_IF_NONZERO(rados_conf_read_file(cl, NULL));
+    rados_conf_parse_env(cl, NULL);
+    std::string log_name = SysTestSettings::inst().get_log_name(get_id_str());
+    if (!log_name.empty())
+      rados_conf_set(cl, "log_file", log_name.c_str());
+    RETURN1_IF_NONZERO(rados_connect(cl));
+    pool_setup_sem->wait();
+    pool_setup_sem->post();
+
+    rados_ioctx_t io_ctx;
+    rados_pool_create(cl, m_pool_name.c_str());
+    RETURN1_IF_NONZERO(rados_ioctx_create(cl, m_pool_name.c_str(), &io_ctx));
+
+    std::map <int, std::string> to_add;
+    for (int i = 0; i < g_num_objects; ++i) {
+      char oid[128];
+      snprintf(oid, sizeof(oid), "%d%s", i, m_suffix.c_str());
+      to_add[i] = oid;
+    }
+
+    int added = 0;
+    while (true) {
+      if (to_add.empty())
+	break;
+      int r = rand() % to_add.size();
+      std::map <int, std::string>::iterator d = to_add.begin();
+      for (int i = 0; i < r; ++i)
+	++d;
+      if (d == to_add.end()) {
+	ret_val = -EDOM;
+	goto out;
+      }
+      std::string oid(d->second);
+      to_add.erase(d);
+
+      std::string buf(StRadosCreatePool::get_random_buf(256));
+      int ret = rados_write(io_ctx, oid.c_str(), buf.c_str(), buf.size(), 0);
+      if (ret != 0) {
+	printf("%s: rados_write(%s) failed with error %d\n",
+	       get_id_str(), oid.c_str(), ret);
+	ret_val = ret;
+	goto out;
+      }
+      ++added;
+      if ((added % 25) == 0) {
+	printf("%s: added %d objects...\n", get_id_str(), added);
+      }
+      if (added == g_num_objects / 2) {
+	printf("%s: added half of the objects\n", get_id_str());
+	modify_sem->post();
+      }
+    }
+
+    printf("%s: added %d objects\n", get_id_str(), added);
+
+  out:
+    rados_ioctx_destroy(io_ctx);
+    rados_shutdown(cl);
+
+    return ret_val;
+  }
+private:
+  std::string m_pool_name;
+  std::string m_suffix;
+};
+
+const char *get_id_str()
+{
+  return "main";
+}
+
+int main(int argc, const char **argv)
+{
+  const char *num_objects = getenv("NUM_OBJECTS");
+  const std::string pool = get_temp_pool_name(argv[0]);
+  if (num_objects) {
+    g_num_objects = atoi(num_objects);
+    if (g_num_objects == 0)
+      return 100;
+  }
+
+  RETURN1_IF_NONZERO(CrossProcessSem::create(0, &pool_setup_sem));
+  RETURN1_IF_NONZERO(CrossProcessSem::create(1, &modify_sem));
+
+  std::string error;
+
+  // Test 1... list objects
+  {
+    StRadosCreatePool r1(argc, argv, NULL, pool_setup_sem, NULL,
+			 pool, g_num_objects, ".obj");
+    StRadosListObjects r2(argc, argv, pool, false, g_num_objects,
+			  pool_setup_sem, modify_sem, NULL);
+    vector < SysTestRunnable* > vec;
+    vec.push_back(&r1);
+    vec.push_back(&r2);
+    error = SysTestRunnable::run_until_finished(vec);
+    if (!error.empty()) {
+      printf("got error: %s\n", error.c_str());
+      return EXIT_FAILURE;
+    }
+  }
+
+  // Test 2... list objects while they're being deleted
+  RETURN1_IF_NONZERO(pool_setup_sem->reinit(0));
+  RETURN1_IF_NONZERO(modify_sem->reinit(0));
+  {
+    StRadosCreatePool r1(argc, argv, NULL, pool_setup_sem, NULL,
+			 pool, g_num_objects, ".obj");
+    StRadosListObjects r2(argc, argv, pool, false, g_num_objects / 2,
+			  pool_setup_sem, modify_sem, NULL);
+    RadosDeleteObjectsR r3(argc, argv, pool);
+    vector < SysTestRunnable* > vec;
+    vec.push_back(&r1);
+    vec.push_back(&r2);
+    vec.push_back(&r3);
+    error = SysTestRunnable::run_until_finished(vec);
+    if (!error.empty()) {
+      printf("got error: %s\n", error.c_str());
+      return EXIT_FAILURE;
+    }
+  }
+
+  // Test 3... list objects while others are being added
+  RETURN1_IF_NONZERO(pool_setup_sem->reinit(0));
+  RETURN1_IF_NONZERO(modify_sem->reinit(0));
+  {
+    StRadosCreatePool r1(argc, argv, NULL, pool_setup_sem, NULL,
+			 pool, g_num_objects, ".obj");
+    StRadosListObjects r2(argc, argv, pool, false, g_num_objects / 2,
+			  pool_setup_sem, modify_sem, NULL);
+    RadosAddObjectsR r3(argc, argv, pool, ".obj2");
+    vector < SysTestRunnable* > vec;
+    vec.push_back(&r1);
+    vec.push_back(&r2);
+    vec.push_back(&r3);
+    error = SysTestRunnable::run_until_finished(vec);
+    if (!error.empty()) {
+      printf("got error: %s\n", error.c_str());
+      return EXIT_FAILURE;
+    }
+  }
+
+  // Test 4... list objects while others are being added and deleted
+  RETURN1_IF_NONZERO(pool_setup_sem->reinit(0));
+  RETURN1_IF_NONZERO(modify_sem->reinit(0));
+  {
+    StRadosCreatePool r1(argc, argv, NULL, pool_setup_sem, NULL,
+			 pool, g_num_objects, ".obj");
+    StRadosListObjects r2(argc, argv, pool, false, g_num_objects / 2,
+			  pool_setup_sem, modify_sem, NULL);
+    RadosAddObjectsR r3(argc, argv, pool, ".obj2");
+    RadosAddObjectsR r4(argc, argv, pool, ".obj3");
+    RadosDeleteObjectsR r5(argc, argv, pool);
+    vector < SysTestRunnable* > vec;
+    vec.push_back(&r1);
+    vec.push_back(&r2);
+    vec.push_back(&r3);
+    vec.push_back(&r4);
+    vec.push_back(&r5);
+    error = SysTestRunnable::run_until_finished(vec);
+    if (!error.empty()) {
+      printf("got error: %s\n", error.c_str());
+      return EXIT_FAILURE;
+    }
+  }
+
+  // Test 5... list objects while they are being modified
+  RETURN1_IF_NONZERO(pool_setup_sem->reinit(0));
+  RETURN1_IF_NONZERO(modify_sem->reinit(0));
+  {
+    StRadosCreatePool r1(argc, argv, NULL, pool_setup_sem, NULL,
+			 pool, g_num_objects, ".obj");
+    StRadosListObjects r2(argc, argv, pool, false, g_num_objects / 2,
+			  pool_setup_sem, modify_sem, NULL);
+    // AddObjects with the same 'suffix' as used in StRadosCreatePool
+    RadosAddObjectsR r3(argc, argv, pool, ".obj");
+    vector < SysTestRunnable* > vec;
+    vec.push_back(&r1);
+    vec.push_back(&r2);
+    vec.push_back(&r3);
+    error = SysTestRunnable::run_until_finished(vec);
+    if (!error.empty()) {
+      printf("got error: %s\n", error.c_str());
+      return EXIT_FAILURE;
+    }
+  }
+
+  rados_t cl;
+  rados_create(&cl, NULL);
+  rados_conf_parse_argv(cl, argc, argv);
+  rados_conf_parse_argv(cl, argc, argv);
+  rados_conf_read_file(cl, NULL);
+  rados_conf_parse_env(cl, NULL);
+  rados_connect(cl);
+  rados_pool_delete(cl, pool.c_str());
+
+  printf("******* SUCCESS **********\n");
+  return EXIT_SUCCESS;
+}
diff --git a/src/test/system/rados_open_pools_parallel.cc b/src/test/system/rados_open_pools_parallel.cc
new file mode 100644
index 000000000..6f99c4270
--- /dev/null
+++ b/src/test/system/rados_open_pools_parallel.cc
@@ -0,0 +1,143 @@
+
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2011 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation.  See file COPYING.
+*
+*/
+
+#include "cross_process_sem.h"
+#include "include/rados/librados.h"
+#include "st_rados_create_pool.h"
+#include "systest_runnable.h"
+#include "systest_settings.h"
+
+#include <errno.h>
+#include <pthread.h>
+#include <semaphore.h>
+#include <sstream>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string>
+#include <time.h>
+#include <vector>
+
+using std::ostringstream;
+using std::string;
+using std::vector;
+
+/*
+ * rados_open_pools_parallel
+ *
+ * This tests creating a pool in one Runnable, and then opening an io context
+ * based on that pool in another.
+ *
+ * EXPECT:            * can't create the same pool twice
+ *                    * one Runnable can use the pool after the other one creates it
+ *
+ * DO NOT EXPECT      * hangs, crashes
+ */
+class StRadosOpenPool : public SysTestRunnable
+{
+public:
+  StRadosOpenPool(int argc, const char **argv,
+                  CrossProcessSem *pool_setup_sem,
+                  CrossProcessSem *open_pool_sem,
+                  const std::string& pool_name)
+    : SysTestRunnable(argc, argv),
+      m_pool_setup_sem(pool_setup_sem),
+      m_open_pool_sem(open_pool_sem),
+      m_pool_name(pool_name)
+  {
+  }
+
+  ~StRadosOpenPool() override
+  {
+  }
+
+  int run() override
+  {
+    rados_t cl;
+    RETURN1_IF_NONZERO(rados_create(&cl, NULL));
+    rados_conf_parse_argv(cl, m_argc, m_argv);
+    std::string log_name = SysTestSettings::inst().get_log_name(get_id_str());
+    if (!log_name.empty())
+      rados_conf_set(cl, "log_file", log_name.c_str());
+    RETURN1_IF_NONZERO(rados_conf_read_file(cl, NULL));
+    rados_conf_parse_env(cl, NULL);
+    RETURN1_IF_NONZERO(rados_connect(cl));
+    if (m_pool_setup_sem)
+      m_pool_setup_sem->wait();
+
+    printf("%s: rados_pool_create.\n", get_id_str());
+    rados_pool_create(cl, m_pool_name.c_str());
+    rados_ioctx_t io_ctx;
+    printf("%s: rados_ioctx_create.\n", get_id_str());
+    RETURN1_IF_NONZERO(rados_ioctx_create(cl, m_pool_name.c_str(), &io_ctx));
+    if (m_open_pool_sem)
+      m_open_pool_sem->post();
+    rados_ioctx_destroy(io_ctx);
+    rados_pool_delete(cl, m_pool_name.c_str());
+    rados_shutdown(cl);
+    return 0;
+  }
+
+private:
+  CrossProcessSem *m_pool_setup_sem;
+  CrossProcessSem *m_open_pool_sem;
+  std::string m_pool_name;
+};
+
+const char *get_id_str()
+{
+  return "main";
+}
+
+int main(int argc, const char **argv)
+{
+  const std::string pool = get_temp_pool_name(argv[0]);
+  // first test: create a pool, shut down the client, access that 
+  // pool in a different process.
+  CrossProcessSem *pool_setup_sem = NULL;
+  RETURN1_IF_NONZERO(CrossProcessSem::create(0, &pool_setup_sem));
+  StRadosCreatePool r1(argc, argv, NULL, pool_setup_sem, NULL,
+					   pool, 50, ".obj");
+  StRadosOpenPool r2(argc, argv, pool_setup_sem, NULL, pool);
+  vector < SysTestRunnable* > vec;
+  vec.push_back(&r1);
+  vec.push_back(&r2);
+  std::string error = SysTestRunnable::run_until_finished(vec);
+  if (!error.empty()) {
+    printf("test1: got error: %s\n", error.c_str());
+    return EXIT_FAILURE;
+  }
+
+  // second test: create a pool, access that 
+  // pool in a different process, THEN shut down the first client.
+  CrossProcessSem *pool_setup_sem2 = NULL;
+  RETURN1_IF_NONZERO(CrossProcessSem::create(0, &pool_setup_sem2));
+  CrossProcessSem *open_pool_sem2 = NULL;
+  RETURN1_IF_NONZERO(CrossProcessSem::create(0, &open_pool_sem2));
+  StRadosCreatePool r3(argc, argv, NULL, pool_setup_sem2, open_pool_sem2,
+					   pool, 50, ".obj");
+  StRadosOpenPool r4(argc, argv, pool_setup_sem2, open_pool_sem2, pool);
+  vector < SysTestRunnable* > vec2;
+  vec2.push_back(&r3);
+  vec2.push_back(&r4);
+  error = SysTestRunnable::run_until_finished(vec2);
+  if (!error.empty()) {
+    printf("test2: got error: %s\n", error.c_str());
+    return EXIT_FAILURE;
+  }
+
+  printf("******* SUCCESS **********\n"); 
+  return EXIT_SUCCESS;
+}
diff --git a/src/test/system/rados_watch_notify.cc b/src/test/system/rados_watch_notify.cc
new file mode 100644
index 000000000..e69e932ec
--- /dev/null
+++ b/src/test/system/rados_watch_notify.cc
@@ -0,0 +1,195 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2011 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation.  See file COPYING.
+*
+*/
+
+#include "cross_process_sem.h"
+#include "include/rados/librados.h"
+#include "st_rados_create_pool.h"
+#include "st_rados_delete_pool.h"
+#include "st_rados_delete_objs.h"
+#include "st_rados_watch.h"
+#include "st_rados_notify.h"
+#include "systest_runnable.h"
+#include "systest_settings.h"
+#include "include/stringify.h"
+
+#include <errno.h>
+#include <pthread.h>
+#include <semaphore.h>
+#include <sstream>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string>
+#include <time.h>
+#include <vector>
+#include <sys/types.h>
+#include <unistd.h>
+
+using std::ostringstream;
+using std::string;
+using std::vector;
+
+/*
+ * rados_watch_notify
+ *
+ * This tests watch/notify with pool and object deletion.
+ *
+ * EXPECT:            * notifies to a deleted object or pool are not received
+ *                    * notifies to existing objects are received
+ *
+ * DO NOT EXPECT      * hangs, crashes
+ */
+
+const char *get_id_str()
+{
+  return "main";
+}
+
+int main(int argc, const char **argv)
+{
+  std::string pool = "foo." + stringify(getpid());
+  CrossProcessSem *setup_sem = NULL;
+  RETURN1_IF_NONZERO(CrossProcessSem::create(0, &setup_sem));
+  CrossProcessSem *watch_sem = NULL;
+  RETURN1_IF_NONZERO(CrossProcessSem::create(0, &watch_sem));
+  CrossProcessSem *notify_sem = NULL;
+  RETURN1_IF_NONZERO(CrossProcessSem::create(0, &notify_sem));
+
+  // create a pool and an object, watch the object, notify.
+  {
+    StRadosCreatePool r1(argc, argv, NULL, setup_sem, NULL, pool, 1, ".obj");
+    StRadosWatch r2(argc, argv, setup_sem, watch_sem, notify_sem,
+		    1, 0, pool, "0.obj");
+    StRadosNotify r3(argc, argv, setup_sem, watch_sem, notify_sem,
+		     0, pool, "0.obj");
+    StRadosDeletePool r4(argc, argv, notify_sem, NULL, pool);
+    vector<SysTestRunnable*> vec;
+    vec.push_back(&r1);
+    vec.push_back(&r2);
+    vec.push_back(&r3);
+    vec.push_back(&r4);
+    std::string error = SysTestRunnable::run_until_finished(vec);
+    if (!error.empty()) {
+      printf("test1: got error: %s\n", error.c_str());
+      return EXIT_FAILURE;
+    }
+  }
+
+  RETURN1_IF_NONZERO(setup_sem->reinit(0));
+  RETURN1_IF_NONZERO(watch_sem->reinit(0));
+  RETURN1_IF_NONZERO(notify_sem->reinit(0));
+
+  // create a pool and an object, watch a non-existent object,
+  // notify non-existent object.watch
+  pool += ".";
+  {
+    StRadosCreatePool r1(argc, argv, NULL, setup_sem, NULL, pool, 0, ".obj");
+    StRadosWatch r2(argc, argv, setup_sem, watch_sem, notify_sem,
+		    0, -ENOENT, pool, "0.obj");
+    StRadosNotify r3(argc, argv, setup_sem, watch_sem, notify_sem,
+		     -ENOENT, pool, "0.obj");
+    StRadosDeletePool r4(argc, argv, notify_sem, NULL, pool);
+    vector<SysTestRunnable*> vec;
+    vec.push_back(&r1);
+    vec.push_back(&r2);
+    vec.push_back(&r3);
+    vec.push_back(&r4);
+    std::string error = SysTestRunnable::run_until_finished(vec);
+    if (!error.empty()) {
+      printf("test2: got error: %s\n", error.c_str());
+      return EXIT_FAILURE;
+    }
+  }
+
+  RETURN1_IF_NONZERO(setup_sem->reinit(0));
+  RETURN1_IF_NONZERO(watch_sem->reinit(0));
+  RETURN1_IF_NONZERO(notify_sem->reinit(0));
+
+  CrossProcessSem *finished_notifies_sem = NULL;
+  RETURN1_IF_NONZERO(CrossProcessSem::create(0, &finished_notifies_sem));
+  CrossProcessSem *deleted_sem = NULL;
+  RETURN1_IF_NONZERO(CrossProcessSem::create(0, &deleted_sem));
+  CrossProcessSem *second_pool_sem = NULL;
+  RETURN1_IF_NONZERO(CrossProcessSem::create(0, &second_pool_sem));
+
+  // create a pool and an object, watch the object, notify,
+  // then delete the pool.
+  // Create a new pool and write to it to make the osd get the updated map,
+  // then try notifying on the deleted pool.
+  pool += ".";
+  {
+    StRadosCreatePool r1(argc, argv, NULL, setup_sem, NULL, pool, 1, ".obj");
+    StRadosWatch r2(argc, argv, setup_sem, watch_sem, finished_notifies_sem,
+		    1, 0, pool, "0.obj");
+    StRadosNotify r3(argc, argv, setup_sem, watch_sem, notify_sem,
+		     0, pool, "0.obj");
+    StRadosDeletePool r4(argc, argv, notify_sem, deleted_sem, pool);
+    StRadosCreatePool r5(argc, argv, deleted_sem, second_pool_sem, NULL,
+			 "bar", 1, ".obj");
+    StRadosNotify r6(argc, argv, second_pool_sem, NULL, finished_notifies_sem,
+		     0, "bar", "0.obj");
+    StRadosDeletePool r7(argc, argv, finished_notifies_sem, NULL, "bar");
+    vector<SysTestRunnable*> vec;
+    vec.push_back(&r1);
+    vec.push_back(&r2);
+    vec.push_back(&r3);
+    vec.push_back(&r4);
+    vec.push_back(&r5);
+    vec.push_back(&r6);
+    vec.push_back(&r7);
+    std::string error = SysTestRunnable::run_until_finished(vec);
+    if (!error.empty()) {
+      printf("test3: got error: %s\n", error.c_str());
+      return EXIT_FAILURE;
+    }
+  }
+
+  RETURN1_IF_NONZERO(setup_sem->reinit(0));
+  RETURN1_IF_NONZERO(watch_sem->reinit(0));
+  RETURN1_IF_NONZERO(notify_sem->reinit(0));
+  RETURN1_IF_NONZERO(finished_notifies_sem->reinit(0));
+  RETURN1_IF_NONZERO(deleted_sem->reinit(0));
+
+  // create a pool and an object, watch the object, notify,
+  // then delete the object, notify
+  // this test is enabled for the resolution of bug #2339.
+  pool += ".";
+  {
+    StRadosCreatePool r1(argc, argv, NULL, setup_sem, NULL, pool, 1, ".obj");
+    StRadosWatch r2(argc, argv, setup_sem, watch_sem, finished_notifies_sem,
+		    1, 0, pool, "0.obj");
+    StRadosNotify r3(argc, argv, setup_sem, watch_sem, notify_sem,
+		     0, pool, "0.obj");
+    StRadosDeleteObjs r4(argc, argv, notify_sem, deleted_sem, 1, pool, ".obj");
+    StRadosNotify r5(argc, argv, setup_sem, deleted_sem, finished_notifies_sem,
+		     -ENOENT, pool, "0.obj");
+    StRadosDeletePool r6(argc, argv, finished_notifies_sem, NULL, pool);
+
+    vector<SysTestRunnable*> vec;
+    vec.push_back(&r1);
+    vec.push_back(&r2);
+    vec.push_back(&r3);
+    vec.push_back(&r4);
+    vec.push_back(&r5);
+    vec.push_back(&r6);
+    std::string error = SysTestRunnable::run_until_finished(vec);
+    if (!error.empty()) {
+      printf("test4: got error: %s\n", error.c_str());
+      return EXIT_FAILURE;
+    }
+  }
+
+  printf("******* SUCCESS **********\n");
+  return EXIT_SUCCESS;
+}
diff --git a/src/test/system/rerun.sh b/src/test/system/rerun.sh
new file mode 100755
index 000000000..5d7d4635b
--- /dev/null
+++ b/src/test/system/rerun.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+[ -z $ITERATIONS ] && ITERATIONS=10
+
+TMPDIR=`mktemp -d -t rerun_logs.XXXXXXXXXX` || exit 1
+
+rm -rf $TMPDIR/logs
+mkdir $TMPDIR/logs
+
+for i in `seq 1 $ITERATIONS`; do
+    echo "********************* iteration $i *********************"
+    LOG_FILE_BASE=$TMPDIR/logs $EXE "$@"
+    if [ $? -ne 0 ]; then
+        die "failed! logs are in $TMPDIR/logs"
+    fi
+done
+
+echo "********************* success *********************"
+rm -rf $TMPDIR
+exit 0
diff --git a/src/test/system/st_rados_create_pool.cc b/src/test/system/st_rados_create_pool.cc
new file mode 100644
index 000000000..a802d9da4
--- /dev/null
+++ b/src/test/system/st_rados_create_pool.cc
@@ -0,0 +1,132 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2011 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation.  See file COPYING.
+*
+*/
+
+#include "cross_process_sem.h"
+#include "include/ceph_assert.h"
+#include "include/rados/librados.h"
+#include "st_rados_create_pool.h"
+#include "systest_runnable.h"
+#include "systest_settings.h"
+
+#include <errno.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <sstream>
+#include <string>
+
+using std::ostringstream;
+
+std::string StRadosCreatePool::
+get_random_buf(int sz)
+{
+  ostringstream oss;
+  int size = rand() % sz; // yep, it's not very random
+  for (int i = 0; i < size; ++i) {
+    oss << ".";
+  }
+  return oss.str();
+}
+
+StRadosCreatePool::
+StRadosCreatePool(int argc, const char **argv,
+		  CrossProcessSem *setup_sem,
+		  CrossProcessSem *pool_setup_sem,
+		  CrossProcessSem *close_create_pool,
+		  const std::string &pool_name,
+		  int num_objects,
+		  const std::string &suffix)
+  : SysTestRunnable(argc, argv),
+    m_setup_sem(setup_sem),
+    m_pool_setup_sem(pool_setup_sem),
+    m_close_create_pool(close_create_pool),
+    m_pool_name(pool_name),
+    m_num_objects(num_objects),
+    m_suffix(suffix)
+{
+}
+
+StRadosCreatePool::
+~StRadosCreatePool()
+{
+}
+
+int StRadosCreatePool::
+run()
+{
+  int ret_val = 0;
+  rados_t cl;
+  RETURN1_IF_NONZERO(rados_create(&cl, NULL));
+  rados_conf_parse_argv(cl, m_argc, m_argv);
+  rados_conf_parse_argv(cl, m_argc, m_argv);
+  RETURN1_IF_NONZERO(rados_conf_read_file(cl, NULL));
+  std::string log_name = SysTestSettings::inst().get_log_name(get_id_str());
+  if (!log_name.empty())
+    rados_conf_set(cl, "log_file", log_name.c_str());
+  rados_conf_parse_env(cl, NULL);
+
+  if (m_setup_sem) {
+    m_setup_sem->wait();
+    m_setup_sem->post();
+  }
+
+  RETURN1_IF_NONZERO(rados_connect(cl));
+
+  printf("%s: creating pool %s\n", get_id_str(), m_pool_name.c_str());
+  rados_pool_create(cl, m_pool_name.c_str());
+  rados_ioctx_t io_ctx;
+  RETURN1_IF_NONZERO(rados_ioctx_create(cl, m_pool_name.c_str(), &io_ctx));
+
+  for (int i = 0; i < m_num_objects; ++i) {
+    char oid[128];
+    snprintf(oid, sizeof(oid), "%d%s", i, m_suffix.c_str());
+    std::string buf(get_random_buf(256));
+    int ret = rados_write(io_ctx, oid, buf.c_str(), buf.size(), 0);
+    if (ret != 0) {
+      printf("%s: rados_write(%s) failed with error: %d\n",
+	     get_id_str(), oid, ret);
+      ret_val = ret;
+      goto out;
+    }
+    if (((i % 25) == 0) || (i == m_num_objects - 1)) {
+      printf("%s: created object %d...\n", get_id_str(), i);
+    }
+  }
+
+out:
+  printf("%s: finishing.\n", get_id_str());
+  if (m_pool_setup_sem)
+    m_pool_setup_sem->post();
+  if (m_close_create_pool)
+    m_close_create_pool->wait();
+  rados_ioctx_destroy(io_ctx);
+  rados_shutdown(cl);
+  return ret_val;
+}
+
+std::string get_temp_pool_name(const char* prefix)
+{
+  ceph_assert(prefix);
+  char hostname[80];
+  int ret = 0;
+  ret = gethostname(hostname, sizeof(hostname));
+  ceph_assert(!ret);
+  char poolname[256];
+  ret = snprintf(poolname, sizeof(poolname),
+                 "%s.%s-%d", prefix, hostname, getpid());
+  ceph_assert(ret > 0);
+  ceph_assert((unsigned int)ret < sizeof(poolname));
+  return poolname;
+}
diff --git a/src/test/system/st_rados_create_pool.h b/src/test/system/st_rados_create_pool.h
new file mode 100644
index 000000000..ee65b22c6
--- /dev/null
+++ b/src/test/system/st_rados_create_pool.h
@@ -0,0 +1,53 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2011 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation.  See file COPYING.
+*
+*/
+
+#ifndef TEST_SYSTEM_ST_RADOS_CREATE_POOL_H
+#define TEST_SYSTEM_ST_RADOS_CREATE_POOL_H
+
+#include "systest_runnable.h"
+
+class CrossProcessSem;
+
+/*
+ * st_rados_create_pool
+ *
+ * Waits, then posts to setup_sem.
+ * Creates a pool and populates it with some objects.
+ * Then, calls pool_setup_sem->post()
+ */
+class StRadosCreatePool : public SysTestRunnable
+{
+public:
+  static std::string get_random_buf(int sz);
+  StRadosCreatePool(int argc, const char **argv,
+		    CrossProcessSem *setup_sem,
+		    CrossProcessSem *pool_setup_sem,
+		    CrossProcessSem *close_create_pool_sem,
+		    const std::string &pool_name,
+		    int num_objects,
+		    const std::string &suffix);
+  ~StRadosCreatePool() override;
+  int run() override;
+private:
+  CrossProcessSem *m_setup_sem;
+  CrossProcessSem *m_pool_setup_sem;
+  CrossProcessSem *m_close_create_pool;
+  std::string m_pool_name;
+  int m_num_objects;
+  std::string m_suffix;
+};
+
+std::string get_temp_pool_name(const char* prefix);
+
+#endif
diff --git a/src/test/system/st_rados_delete_objs.cc b/src/test/system/st_rados_delete_objs.cc
new file mode 100644
index 000000000..b43ffdda6
--- /dev/null
+++ b/src/test/system/st_rados_delete_objs.cc
@@ -0,0 +1,71 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2011 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation.  See file COPYING.
+*
+*/
+
+#include "cross_process_sem.h"
+#include "include/rados/librados.h"
+#include "st_rados_delete_objs.h"
+#include "systest_runnable.h"
+#include "systest_settings.h"
+
+#include <errno.h>
+
+StRadosDeleteObjs::StRadosDeleteObjs(int argc, const char **argv,
+				     CrossProcessSem *setup_sem,
+				     CrossProcessSem *deleted_sem,
+				     int num_objs,
+				     const std::string &pool_name,
+				     const std::string &suffix)
+  : SysTestRunnable(argc, argv),
+    m_setup_sem(setup_sem),
+    m_deleted_sem(deleted_sem),
+    m_num_objs(num_objs),
+    m_pool_name(pool_name),
+    m_suffix(suffix)
+{
+}
+
+StRadosDeleteObjs::~StRadosDeleteObjs()
+{
+}
+
+int StRadosDeleteObjs::run()
+{
+  rados_t cl;
+  RETURN1_IF_NONZERO(rados_create(&cl, NULL));
+  rados_conf_parse_argv(cl, m_argc, m_argv);
+  RETURN1_IF_NONZERO(rados_conf_read_file(cl, NULL));
+  rados_conf_parse_env(cl, NULL);
+  RETURN1_IF_NONZERO(rados_connect(cl));
+  m_setup_sem->wait();
+  m_setup_sem->post();
+
+  rados_ioctx_t io_ctx;
+  rados_pool_create(cl, m_pool_name.c_str());
+  RETURN1_IF_NONZERO(rados_ioctx_create(cl, m_pool_name.c_str(), &io_ctx));
+
+  for (int i = 0; i < m_num_objs; ++i) {
+    char oid[128];
+    snprintf(oid, sizeof(oid), "%d%s", i, m_suffix.c_str());
+    RETURN1_IF_NONZERO(rados_remove(io_ctx, oid));
+    if (((i % 25) == 0) || (i == m_num_objs - 1)) {
+      printf("%s: deleted object %d...\n", get_id_str(), i);
+    }
+  }
+
+  rados_ioctx_destroy(io_ctx);
+  if (m_deleted_sem)
+    m_deleted_sem->post();
+  rados_shutdown(cl);
+  return 0;
+}
diff --git a/src/test/system/st_rados_delete_objs.h b/src/test/system/st_rados_delete_objs.h
new file mode 100644
index 000000000..40771ee45
--- /dev/null
+++ b/src/test/system/st_rados_delete_objs.h
@@ -0,0 +1,48 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2011 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation.  See file COPYING.
+*
+*/
+
+#ifndef TEST_SYSTEM_ST_RADOS_DELETE_OBJS_H
+#define TEST_SYSTEM_ST_RADOS_DELETE_OBJS_H
+
+#include "systest_runnable.h"
+
+class CrossProcessSem;
+
+/*
+ * st_rados_delete_objs
+ *
+ * Waits on setup_sem, posts to it,
+ * deletes num_objs objects from the pool,
+ * and posts to deleted_sem.
+ */
+class StRadosDeleteObjs : public SysTestRunnable
+{
+public:
+  StRadosDeleteObjs(int argc, const char **argv,
+		    CrossProcessSem *setup_sem,
+		    CrossProcessSem *deleted_sem,
+		    int num_objs,
+		    const std::string &pool_name,
+		    const std::string &suffix);
+  ~StRadosDeleteObjs() override;
+  int run() override;
+private:
+  CrossProcessSem *m_setup_sem;
+  CrossProcessSem *m_deleted_sem;
+  int m_num_objs;
+  std::string m_pool_name;
+  std::string m_suffix;
+};
+
+#endif
diff --git a/src/test/system/st_rados_delete_pool.cc b/src/test/system/st_rados_delete_pool.cc
new file mode 100644
index 000000000..de553e98e
--- /dev/null
+++ b/src/test/system/st_rados_delete_pool.cc
@@ -0,0 +1,59 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2011 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation.  See file COPYING.
+*
+*/
+
+#include "cross_process_sem.h"
+#include "include/rados/librados.h"
+#include "st_rados_delete_pool.h"
+#include "systest_runnable.h"
+#include "systest_settings.h"
+
+#include <errno.h>
+
+StRadosDeletePool::StRadosDeletePool(int argc, const char **argv,
+				     CrossProcessSem *pool_setup_sem,
+				     CrossProcessSem *delete_pool_sem,
+				     const std::string &pool_name)
+    : SysTestRunnable(argc, argv),
+      m_pool_setup_sem(pool_setup_sem),
+      m_delete_pool_sem(delete_pool_sem),
+      m_pool_name(pool_name)
+{
+}
+
+StRadosDeletePool::~StRadosDeletePool()
+{
+}
+
+int StRadosDeletePool::run()
+{
+  rados_t cl;
+  RETURN1_IF_NONZERO(rados_create(&cl, NULL));
+  rados_conf_parse_argv(cl, m_argc, m_argv);
+  RETURN1_IF_NONZERO(rados_conf_read_file(cl, NULL));
+  rados_conf_parse_env(cl, NULL);
+  RETURN1_IF_NONZERO(rados_connect(cl));
+  m_pool_setup_sem->wait();
+  m_pool_setup_sem->post();
+
+  rados_ioctx_t io_ctx;
+  rados_pool_create(cl, m_pool_name.c_str());
+  RETURN1_IF_NONZERO(rados_ioctx_create(cl, m_pool_name.c_str(), &io_ctx));
+  rados_ioctx_destroy(io_ctx);
+  printf("%s: deleting pool %s\n", get_id_str(), m_pool_name.c_str());
+  RETURN1_IF_NONZERO(rados_pool_delete(cl, m_pool_name.c_str()));
+  if (m_delete_pool_sem)
+    m_delete_pool_sem->post();
+  rados_shutdown(cl);
+  return 0;
+}
diff --git a/src/test/system/st_rados_delete_pool.h b/src/test/system/st_rados_delete_pool.h
new file mode 100644
index 000000000..cc48af9b9
--- /dev/null
+++ b/src/test/system/st_rados_delete_pool.h
@@ -0,0 +1,43 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2011 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation.  See file COPYING.
+*
+*/
+
+#ifndef TEST_SYSTEM_ST_RADOS_DELETE_POOL_H
+#define TEST_SYSTEM_ST_RADOS_DELETE_POOL_H
+
+#include "systest_runnable.h"
+
+class CrossProcessSem;
+
+/*
+ * st_rados_delete_pool
+ *
+ * Waits on pool_setup_sem, posts to it,
+ * deletes a pool, and posts to delete_pool_sem.
+ */
+class StRadosDeletePool : public SysTestRunnable
+{
+public:
+  StRadosDeletePool(int argc, const char **argv,
+		    CrossProcessSem *pool_setup_sem,
+		    CrossProcessSem *delete_pool_sem,
+		    const std::string &pool_name);
+  ~StRadosDeletePool() override;
+  int run() override;
+private:
+  CrossProcessSem *m_pool_setup_sem;
+  CrossProcessSem *m_delete_pool_sem;
+  std::string m_pool_name;
+};
+
+#endif
diff --git a/src/test/system/st_rados_list_objects.cc b/src/test/system/st_rados_list_objects.cc
new file mode 100644
index 000000000..514dafe65
--- /dev/null
+++ b/src/test/system/st_rados_list_objects.cc
@@ -0,0 +1,107 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2011 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation.  See file COPYING.
+*
+*/
+
+#include "cross_process_sem.h"
+#include "include/rados/librados.h"
+#include "st_rados_list_objects.h"
+#include "systest_runnable.h"
+#include "systest_settings.h"
+
+#include <errno.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sstream>
+#include <string>
+
+using std::ostringstream;
+
+StRadosListObjects::
+StRadosListObjects(int argc, const char **argv,
+		   const std::string &pool_name,
+		   bool accept_list_errors,
+		   int midway_cnt,
+		   CrossProcessSem *pool_setup_sem,
+		   CrossProcessSem *midway_sem_wait,
+		   CrossProcessSem *midway_sem_post)
+  : SysTestRunnable(argc, argv),
+    m_pool_name(pool_name),
+    m_accept_list_errors(accept_list_errors),
+    m_midway_cnt(midway_cnt),
+    m_pool_setup_sem(pool_setup_sem),
+    m_midway_sem_wait(midway_sem_wait),
+    m_midway_sem_post(midway_sem_post)
+{
+}
+
+StRadosListObjects::
+~StRadosListObjects()
+{
+}
+
+int StRadosListObjects::
+run()
+{
+  int retval = 0;
+  rados_t cl;
+  RETURN1_IF_NONZERO(rados_create(&cl, NULL));
+  rados_conf_parse_argv(cl, m_argc, m_argv);
+  RETURN1_IF_NONZERO(rados_conf_read_file(cl, NULL));
+  rados_conf_parse_env(cl, NULL);
+  RETURN1_IF_NONZERO(rados_connect(cl));
+  m_pool_setup_sem->wait();
+  m_pool_setup_sem->post();
+
+  rados_ioctx_t io_ctx;
+  rados_pool_create(cl, m_pool_name.c_str());
+  RETURN1_IF_NONZERO(rados_ioctx_create(cl, m_pool_name.c_str(), &io_ctx));
+
+  int saw = 0;
+  const char *obj_name;
+  rados_list_ctx_t h;
+  printf("%s: listing objects.\n", get_id_str());
+  RETURN1_IF_NONZERO(rados_nobjects_list_open(io_ctx, &h));
+  while (true) {
+    int ret = rados_nobjects_list_next(h, &obj_name, NULL, NULL);
+    if (ret == -ENOENT) {
+      break;
+    }
+    else if (ret != 0) {
+      if (m_accept_list_errors && (!m_midway_sem_post || saw > m_midway_cnt))
+	break;
+      printf("%s: rados_objects_list_next error: %d\n", get_id_str(), ret);
+      retval = ret;
+      goto out;
+    }
+    if ((saw % 25) == 0) {
+      printf("%s: listed object %d...\n", get_id_str(), saw);
+    }
+    ++saw;
+    if (saw == m_midway_cnt) {
+      if (m_midway_sem_wait)
+	m_midway_sem_wait->wait();
+      if (m_midway_sem_post)
+	m_midway_sem_post->post();
+    }
+  }
+
+  printf("%s: saw %d objects\n", get_id_str(), saw);
+
+out:
+  rados_nobjects_list_close(h);
+  rados_ioctx_destroy(io_ctx);
+  rados_shutdown(cl);
+
+  return retval;
+}
diff --git a/src/test/system/st_rados_list_objects.h b/src/test/system/st_rados_list_objects.h
new file mode 100644
index 000000000..b26d51844
--- /dev/null
+++ b/src/test/system/st_rados_list_objects.h
@@ -0,0 +1,53 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2011 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation.  See file COPYING.
+*
+*/
+
+#ifndef TEST_SYSTEM_ST_RADOS_LIST_OBJECTS_H
+#define TEST_SYSTEM_ST_RADOS_LIST_OBJECTS_H
+
+#include "systest_runnable.h"
+
+class CrossProcessSem;
+
+/*
+ * st_rados_list_objects
+ *
+ * 1. calls pool_setup_sem->wait()
+ * 2. calls pool_setup_sem->post()
+ * 3. list some objects
+ * 4. modify_sem->wait()
+ * 5. list some objects
+ */
+class StRadosListObjects : public SysTestRunnable
+{
+public:
+  static std::string get_random_buf(int sz);
+  StRadosListObjects(int argc, const char **argv,
+		     const std::string &pool_name,
+		     bool accept_list_errors,
+		     int midway_cnt,
+		     CrossProcessSem *pool_setup_sem,
+		     CrossProcessSem *midway_sem_wait,
+		     CrossProcessSem *midway_sem_post);
+  ~StRadosListObjects() override;
+  int run() override;
+private:
+  std::string m_pool_name;
+  bool m_accept_list_errors;
+  int m_midway_cnt;
+  CrossProcessSem *m_pool_setup_sem;
+  CrossProcessSem *m_midway_sem_wait;
+  CrossProcessSem *m_midway_sem_post;
+};
+
+#endif
diff --git a/src/test/system/st_rados_notify.h b/src/test/system/st_rados_notify.h
new file mode 100644
index 000000000..1d61b4d12
--- /dev/null
+++ b/src/test/system/st_rados_notify.h
@@ -0,0 +1,52 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2011 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation.  See file COPYING.
+*
+*/
+
+#ifndef TEST_SYSTEM_ST_RADOS_NOTIFY_H
+#define TEST_SYSTEM_ST_RADOS_NOTIFY_H
+
+#include "systest_runnable.h"
+
+class CrossProcessSem;
+
+/*
+ * st_rados_notify
+ *
+ * 1. waits on and then posts to setup_sem
+ * 2. connects and opens the pool
+ * 3. waits on and then posts to notify_sem
+ * 4. notifies on the object
+ * 5. posts to notified_sem
+ */
+class StRadosNotify : public SysTestRunnable
+{
+public:
+  StRadosNotify(int argc, const char **argv,
+		CrossProcessSem *setup_sem,
+		CrossProcessSem *notify_sem,
+		CrossProcessSem *notified_sem,
+		int notify_retcode,
+		const std::string &pool_name,
+		const std::string &obj_name);
+  ~StRadosNotify() override;
+  int run() override;
+private:
+  CrossProcessSem *m_setup_sem;
+  CrossProcessSem *m_notify_sem;
+  CrossProcessSem *m_notified_sem;
+  int m_notify_retcode;
+  std::string m_pool_name;
+  std::string m_obj_name;
+};
+
+#endif
diff --git a/src/test/system/st_rados_watch.h b/src/test/system/st_rados_watch.h
new file mode 100644
index 000000000..366be3f87
--- /dev/null
+++ b/src/test/system/st_rados_watch.h
@@ -0,0 +1,56 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2011 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation.  See file COPYING.
+*
+*/
+
+#ifndef TEST_SYSTEM_ST_RADOS_WATCH_H
+#define TEST_SYSTEM_ST_RADOS_WATCH_H
+
+#include "systest_runnable.h"
+
+class CrossProcessSem;
+
+/*
+ * st_rados_watch
+ *
+ * 1. waits on setup_sem
+ * 2. posts to setup_sem
+ * 3. watches an object
+ * 4. posts to watch_sem
+ * 5. waits on notify_sem
+ * 6. posts to notify_sem
+ * 7. checks that the correct number of notifies were received
+ */
+class StRadosWatch : public SysTestRunnable
+{
+public:
+  StRadosWatch(int argc, const char **argv,
+	       CrossProcessSem *setup_sem,
+	       CrossProcessSem *watch_sem,
+	       CrossProcessSem *notify_sem,
+	       int num_notifies,
+	       int watch_retcode,
+	       const std::string &pool_name,
+	       const std::string &obj_name);
+  ~StRadosWatch() override;
+  int run() override;
+private:
+  CrossProcessSem *m_setup_sem;
+  CrossProcessSem *m_watch_sem;
+  CrossProcessSem *m_notify_sem;
+  int m_num_notifies;
+  int m_watch_retcode;
+  std::string m_pool_name;
+  std::string m_obj_name;
+};
+
+#endif
diff --git a/src/test/system/systest_runnable.cc b/src/test/system/systest_runnable.cc
new file mode 100644
index 000000000..f7342aa7d
--- /dev/null
+++ b/src/test/system/systest_runnable.cc
@@ -0,0 +1,231 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "include/compat.h"
+#include "common/errno.h"
+#include "systest_runnable.h"
+#include "systest_settings.h"
+
+#include <errno.h>
+#include <pthread.h>
+#include <sstream>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <string>
+#ifndef _WIN32
+#include <sys/syscall.h>
+#include <sys/wait.h>
+#endif
+#include <sys/types.h>
+#include <unistd.h>
+#include <atomic>
+#include <limits>
+#include <vector>
+
+using std::ostringstream;
+using std::string;
+
+static pid_t do_gettid(void)
+{
+#if defined(__linux__)
+  return static_cast < pid_t >(syscall(SYS_gettid));
+#elif defined(_WIN32)
+  return static_cast < pid_t >(GetCurrentThreadId());
+#else
+  return static_cast < pid_t >(pthread_getthreadid_np());
+#endif
+}
+
+std::atomic<unsigned> m_highest_id = { 0 };
+
+SysTestRunnable::
+SysTestRunnable(int argc, const char **argv)
+  : m_argc(0),
+    m_argv(NULL),
+    m_argv_orig(NULL)
+{
+  m_started = false;
+  m_id = ++m_highest_id;
+  memset(&m_pthread, 0, sizeof(m_pthread));
+  update_id_str(false);
+  set_argv(argc, argv);
+}
+
+SysTestRunnable::
+~SysTestRunnable()
+{
+  set_argv(0, NULL);
+}
+
+const char* SysTestRunnable::
+get_id_str(void) const
+{
+  return m_id_str;
+}
+
+int SysTestRunnable::
+start()
+{
+  if (m_started) {
+    return -EDOM;
+  }
+  int ret;
+  bool use_threads = SysTestSettings::inst().use_threads();
+  if (use_threads) {
+    ret = pthread_create(&m_pthread, NULL, systest_runnable_pthread_helper,
+			     static_cast<void*>(this));
+    if (ret)
+      return ret;
+    m_started = true;
+  } else {
+    #ifdef _WIN32
+    printf("Using separate processes is not supported on Windows.\n");
+    return -1;
+    #else
+    std::string err_msg;
+    ret = preforker.prefork(err_msg);
+    if (ret < 0)
+      preforker.exit(ret);
+
+    if (preforker.is_child()) {
+      m_started = true;
+      void *retptr = systest_runnable_pthread_helper(static_cast<void*>(this));
+      preforker.exit((int)(uintptr_t)retptr);
+    } else {
+      m_started = true;
+    }
+    #endif
+  }
+  return 0;
+}
+
+std::string SysTestRunnable::
+join()
+{
+  if (!m_started) {
+    return "SysTestRunnable was never started.";
+  }
+  int ret;
+  bool use_threads = SysTestSettings::inst().use_threads();
+  if (use_threads) {
+    void *ptrretval;
+    ret = pthread_join(m_pthread, &ptrretval);
+    if (ret) {
+      ostringstream oss;
+      oss << "pthread_join failed with error " << ret;
+      return oss.str();
+    }
+    int retval = (int)(uintptr_t)ptrretval;
+    if (retval != 0) {
+      ostringstream oss;
+      oss << "ERROR " << retval;
+      return oss.str();
+    }
+    return "";
+  } else {
+    #ifdef _WIN32
+    return "Using separate processes is not supported on Windows.\n";
+    #else
+    std::string err_msg;
+    ret = preforker.parent_wait(err_msg);
+    return err_msg;
+    #endif
+  }
+}
+
+std::string SysTestRunnable::
+run_until_finished(std::vector < SysTestRunnable * > &runnables)
+{
+  int index = 0;
+  for (std::vector < SysTestRunnable * >::const_iterator r = runnables.begin();
+      r != runnables.end(); ++r) {
+    int ret = (*r)->start();
+    if (ret) {
+      ostringstream oss;
+      oss << "run_until_finished: got error " << ret
+	  << " when starting runnable " << index;
+      return oss.str();
+    }
+    ++index;
+  }
+
+  for (std::vector < SysTestRunnable * >::const_iterator r = runnables.begin();
+      r != runnables.end(); ++r) {
+    std::string rstr = (*r)->join();
+    if (!rstr.empty()) {
+      ostringstream oss;
+      oss << "run_until_finished: runnable " << (*r)->get_id_str() 
+	  << ": got error: " << rstr;
+      return oss.str();
+    }
+  }
+  printf("*******************************\n");
+  return "";
+}
+
+void *systest_runnable_pthread_helper(void *arg)
+{
+  SysTestRunnable *st = static_cast < SysTestRunnable * >(arg);
+  st->update_id_str(true);
+  printf("%s: starting.\n", st->get_id_str());
+  int ret = st->run();
+  printf("%s: shutting down.\n", st->get_id_str());
+  return (void*)(uintptr_t)ret;
+}
+
+void SysTestRunnable::
+update_id_str(bool started)
+{
+  bool use_threads = SysTestSettings::inst().use_threads();
+  char extra[std::numeric_limits<int>::digits10 + 1];
+  extra[0] = '\0';
+
+  if (started) {
+    if (use_threads)
+      snprintf(extra, sizeof(extra), "_[%d]", do_gettid());
+    else
+      snprintf(extra, sizeof(extra), "_[%d]", getpid());
+  }
+  if (use_threads)
+    snprintf(m_id_str, SysTestRunnable::ID_STR_SZ, "thread_%d%s", m_id, extra);
+  else
+    snprintf(m_id_str, SysTestRunnable::ID_STR_SZ, "process_%d%s", m_id, extra);
+}
+
+// Copy argv so that if some fiend decides to modify it, it's ok.
+void SysTestRunnable::
+set_argv(int argc, const char **argv)
+{
+  if (m_argv_orig != NULL) {
+    for (int i = 0; i < m_argc; ++i)
+      free((void*)(m_argv_orig[i]));
+    delete[] m_argv_orig;
+    m_argv_orig = NULL;
+    delete[] m_argv;
+    m_argv = NULL;
+    m_argc = 0;
+  }
+  if (argv == NULL)
+    return;
+  m_argc = argc;
+  m_argv_orig = new const char*[m_argc+1];
+  for (int i = 0; i < m_argc; ++i)
+    m_argv_orig[i] = strdup(argv[i]);
+  m_argv_orig[argc] = NULL;
+  m_argv = new const char*[m_argc+1];
+  for (int i = 0; i <= m_argc; ++i)
+    m_argv[i] = m_argv_orig[i];
+}
diff --git a/src/test/system/systest_runnable.h b/src/test/system/systest_runnable.h
new file mode 100644
index 000000000..bfa9130ed
--- /dev/null
+++ b/src/test/system/systest_runnable.h
@@ -0,0 +1,94 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef CEPH_SYSTEM_TEST_H
+#define CEPH_SYSTEM_TEST_H
+
+#include <pthread.h>
+#include <stdio.h>
+#include <string>
+#include <vector>
+
+#ifndef _WIN32
+#include "common/Preforker.h"
+#endif
+
+#define RETURN1_IF_NOT_VAL(expected, expr) \
+  do {\
+    int _rinv_ret = expr;\
+    if (_rinv_ret != expected) {\
+      printf("%s: file %s, line %d: expected %d, got %d\n",\
+	     get_id_str(), __FILE__, __LINE__, expected, _rinv_ret);\
+      return 1; \
+    }\
+  } while(0);
+
+#define RETURN1_IF_NONZERO(expr) \
+  RETURN1_IF_NOT_VAL(0, expr)
+
+extern void* systest_runnable_pthread_helper(void *arg);
+std::string get_temp_pool_name(const char* prefix);
+/* Represents a single test thread / process.
+ *
+ * Inherit from this class and implement the test body in run().
+*/
+class SysTestRunnable
+{
+public:
+  static const int ID_STR_SZ = 196;
+
+  SysTestRunnable(int argc, const char **argv);
+  virtual ~SysTestRunnable();
+
+  /* Returns 0 on success; error code otherwise. */
+  virtual int run() = 0;
+
+  /* Return a string identifying the runnable. */ 
+  const char* get_id_str(void) const;
+
+  /* Start the Runnable */
+  int start();
+
+  /* Wait until the Runnable is finished. Returns an error string on failure. */
+  std::string join();
+
+  /* Starts a bunch of SystemTestRunnables and waits until they're done.
+   *
+   * Returns an error string on failure. */
+  static std::string run_until_finished(std::vector < SysTestRunnable * >&
+					runnables);
+
+protected:
+  int m_argc;
+  const char **m_argv;
+
+private:
+  explicit SysTestRunnable(const SysTestRunnable &rhs);
+  SysTestRunnable& operator=(const SysTestRunnable &rhs);
+  void update_id_str(bool started);
+  void set_argv(int argc, const char **argv);
+
+  friend void* systest_runnable_pthread_helper(void *arg);
+
+  #ifndef _WIN32
+  Preforker preforker;
+  #endif
+  const char **m_argv_orig;
+  bool m_started;
+  int m_id;
+  pthread_t m_pthread;
+  char m_id_str[ID_STR_SZ];
+};
+
+#endif
diff --git a/src/test/system/systest_settings.cc b/src/test/system/systest_settings.cc
new file mode 100644
index 000000000..787d763fe
--- /dev/null
+++ b/src/test/system/systest_settings.cc
@@ -0,0 +1,71 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "systest_settings.h"
+
+#include <pthread.h>
+#include <sstream>
+#include <stdlib.h>
+
+pthread_mutex_t g_system_test_settings_lock = PTHREAD_MUTEX_INITIALIZER;
+
+SysTestSettings& SysTestSettings::
+inst()
+{
+  pthread_mutex_lock(&g_system_test_settings_lock);
+  if (!m_inst)
+    m_inst = new SysTestSettings();
+  pthread_mutex_unlock(&g_system_test_settings_lock);
+  return *m_inst;
+}
+
+bool SysTestSettings::
+use_threads() const
+{
+  #ifdef _WIN32
+  // We can't use multiple processes on Windows for the time being.
+  // We'd need a mechanism for spawning those procecesses and also handle
+  // the inter-process communication.
+  return true;
+  #else
+  return m_use_threads;
+  #endif
+}
+
+std::string SysTestSettings::
+get_log_name(const std::string &suffix) const
+{
+  if (m_log_file_base.empty())
+    return "";
+  std::ostringstream oss;
+  oss << m_log_file_base << "." << suffix;
+  return oss.str();
+}
+
+SysTestSettings* SysTestSettings::
+m_inst = NULL;
+
+SysTestSettings::
+SysTestSettings()
+{
+  m_use_threads = !!getenv("USE_THREADS");
+  const char *lfb = getenv("LOG_FILE_BASE");
+  if (lfb)
+    m_log_file_base.assign(lfb);
+}
+
+SysTestSettings::
+~SysTestSettings()
+{
+}
diff --git a/src/test/system/systest_settings.h b/src/test/system/systest_settings.h
new file mode 100644
index 000000000..4c32143f7
--- /dev/null
+++ b/src/test/system/systest_settings.h
@@ -0,0 +1,36 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef CEPH_SYSTEM_TEST_SETTINGS_H
+#define CEPH_SYSTEM_TEST_SETTINGS_H
+
+#include <string>
+
+/* Singleton with settings grabbed from environment variables */
+class SysTestSettings
+{
+public:
+  static SysTestSettings& inst();
+  bool use_threads() const;
+  std::string get_log_name(const std::string &suffix) const;
+private:
+  static SysTestSettings* m_inst;
+  SysTestSettings();
+  ~SysTestSettings();
+
+  bool m_use_threads;
+  std::string m_log_file_base;
+};
+
+#endif