Adding upstream version 18.2.2.upstream/18.2.2

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 436 insertions, 0 deletions

diff --git a/src/test/librgw_file_aw.cc b/src/test/librgw_file_aw.cc
new file mode 100644
index 000000000..11c81ad7f
--- /dev/null
+++ b/src/test/librgw_file_aw.cc
@@ -0,0 +1,436 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2015 Red Hat, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include <stdint.h>
+#include <tuple>
+#include <iostream>
+#include <vector>
+#include <map>
+#include <random>
+#include "xxhash.h"
+
+#include "include/rados/librgw.h"
+#include "include/rados/rgw_file.h"
+
+#include "gtest/gtest.h"
+#include "common/ceph_argparse.h"
+#include "common/debug.h"
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_rgw
+
+using namespace std;
+
+namespace {
+  librgw_t rgw = nullptr;
+  string userid("testuser");
+  string access_key("");
+  string secret_key("");
+  struct rgw_fs *fs = nullptr;
+
+  uint32_t owner_uid = 867;
+  uint32_t owner_gid = 5309;
+  uint32_t create_mask = RGW_SETATTR_UID | RGW_SETATTR_GID | RGW_SETATTR_MODE;
+
+  bool do_create = false;
+  bool do_delete = false;
+  bool do_large = false;
+  bool do_verify = false;
+  bool do_hexdump = false;
+
+  string bucket_name = "sorrydave";
+  string object_name = "jocaml";
+
+  struct rgw_file_handle *bucket_fh = nullptr;
+  struct rgw_file_handle *object_fh = nullptr;
+
+  typedef std::tuple<string,uint64_t, struct rgw_file_handle*> fid_type;
+  std::vector<fid_type> fids;
+
+  std::uniform_int_distribution<uint8_t> uint_dist;
+  std::mt19937 rng;
+
+  constexpr int iovcnt = 4;
+  constexpr int page_size = /* 65536 */ 4 * 1024*1024;
+  constexpr int seed = 8675309;
+
+  struct ZPage
+  {
+    char data[page_size];
+    uint64_t cksum;
+  }; /* ZPage */
+  
+  struct ZPageSet
+  {
+    std::vector<ZPage*> pages;
+    struct iovec* iovs;
+
+    explicit ZPageSet(int n) {
+      pages.reserve(n);
+      iovs = (struct iovec*) calloc(n, sizeof(struct iovec));
+      for (int page_ix = 0; page_ix < n; ++page_ix) {
+	ZPage* p = new ZPage();
+	for (int data_ix = 0; data_ix < page_size; ++data_ix) {
+	  p->data[data_ix] = uint_dist(rng);
+	} // data_ix
+	p->cksum = XXH64(p->data, page_size, seed);
+	pages.emplace_back(p);
+	// and iovs
+	struct iovec* iov = &iovs[page_ix];
+	iov->iov_base = p->data;
+	iov->iov_len = page_size;
+      } // page_ix
+    }
+
+    int size() { return pages.size(); }
+
+    struct iovec* get_iovs() { return iovs; }
+
+    bool operator==(const ZPageSet& rhs) {
+      int n = size();
+      for (int page_ix = 0; page_ix < n; ++page_ix) {
+	ZPage* p1 = pages[page_ix];
+	ZPage* p2 = rhs.pages[page_ix];
+	if (p1->cksum != p2->cksum)
+	  return false;
+      }
+      return true;
+    }
+
+    bool operator==(const rgw_uio* uio) {
+      uint64_t cksum;
+      int vix = 0, off = 0;
+      rgw_vio* vio = &uio->uio_vio[vix];
+      int vio_len = vio->vio_len;
+      char *data;
+
+      for (int ix = 0; ix < iovcnt; ++ix) {
+	ZPage* p1 = pages[ix];
+	data = static_cast<char*>(vio->vio_base) + off;
+	cksum = XXH64(data, page_size, seed);
+
+	if (p1->cksum != cksum) {
+	  int r = memcmp(data, p1->data, page_size);
+	  std::cout << "problem at ix " << ix << " r " << r<< std::endl;
+	  return false;
+	}
+
+	off += page_size;
+	if (off >= vio_len) {
+	  vio = &uio->uio_vio[++vix];
+	  vio_len = vio->vio_len;
+	  off = 0;
+	}
+      }
+      return true;
+    }
+    
+    void cksum() {
+      int n = size();
+      for (int page_ix = 0; page_ix < n; ++page_ix) {
+	ZPage* p = pages[page_ix];
+	p->cksum = XXH64(p->data, page_size, seed);
+      }
+    }
+
+    void reset_iovs() { // VOP_READ and VOP_WRITE update
+      int n = size();
+      for (int page_ix = 0; page_ix < n; ++page_ix) {
+	ZPage* p = pages[page_ix];
+	struct iovec* iov = &iovs[page_ix];
+	iov->iov_base = p->data;
+	iov->iov_len = page_size;
+      }
+    }
+
+    ~ZPageSet() {
+      for (unsigned int ix = 0; ix < pages.size(); ++ix)
+	delete pages[ix];
+      free(iovs);
+    }
+  }; /* ZPageSet */
+
+  ZPageSet zp_set1{iovcnt}; // random data
+  ZPageSet zp_set2{iovcnt}; // random data in 64K pages
+
+  struct {
+    int argc;
+    char **argv;
+  } saved_args;
+}
+
+TEST(LibRGW, INIT) {
+  int ret = librgw_create(&rgw, saved_args.argc, saved_args.argv);
+  ASSERT_EQ(ret, 0);
+  ASSERT_NE(rgw, nullptr);
+}
+
+TEST(LibRGW, MOUNT) {
+  int ret = rgw_mount2(rgw, userid.c_str(), access_key.c_str(),
+                       secret_key.c_str(), "/", &fs, RGW_MOUNT_FLAG_NONE);
+  ASSERT_EQ(ret, 0);
+  ASSERT_NE(fs, nullptr);
+}
+
+TEST(LibRGW, CREATE_BUCKET) {
+  if (do_create) {
+    struct stat st;
+    struct rgw_file_handle *fh;
+
+    st.st_uid = owner_uid;
+    st.st_gid = owner_gid;
+    st.st_mode = 755;
+
+    int ret = rgw_mkdir(fs, fs->root_fh, bucket_name.c_str(), &st, create_mask,
+			&fh, RGW_MKDIR_FLAG_NONE);
+    ASSERT_EQ(ret, 0);
+  }
+}
+
+TEST(LibRGW, LOOKUP_BUCKET) {
+  int ret = rgw_lookup(fs, fs->root_fh, bucket_name.c_str(), &bucket_fh,
+		       nullptr, 0, RGW_LOOKUP_FLAG_NONE);
+  ASSERT_EQ(ret, 0);
+}
+
+TEST(LibRGW, LOOKUP_OBJECT) {
+  int ret = rgw_lookup(fs, bucket_fh, object_name.c_str(), &object_fh,
+		       nullptr, 0, RGW_LOOKUP_FLAG_CREATE);
+  ASSERT_EQ(ret, 0);
+}
+
+#if 0
+TEST(LibRGW, OPEN1) {
+  int ret = rgw_open(fs, object_fh, 0 /* posix flags */, RGW_OPEN_FLAG_NONE);
+  ASSERT_EQ(ret, 0);
+}
+
+TEST(LibRGW, PUT_OBJECT) {
+  size_t nbytes;
+  struct iovec *iovs = zp_set1.get_iovs();
+  off_t offset = 0;
+  for (int ix : {0, 1}) {
+    struct iovec *iov = &iovs[ix];
+    // quick check
+    sprintf(static_cast<char*>(iov->iov_base), "::hi mom (%d)", ix);
+    iov->iov_len = 14;
+    int ret = rgw_write(fs, object_fh, offset, iov->iov_len, &nbytes,
+			iov->iov_base, RGW_WRITE_FLAG_NONE);
+    offset += iov->iov_len;
+    ASSERT_EQ(ret, 0);
+    ASSERT_EQ(nbytes, iov->iov_len);
+  }
+}
+
+TEST(LibRGW, CLOSE1) {
+  int ret = rgw_close(fs, object_fh, RGW_CLOSE_FLAG_NONE);
+  ASSERT_EQ(ret, 0);
+}
+
+TEST(LibRGW, OPEN2) {
+  int ret = rgw_open(fs, object_fh, 0 /* posix flags */, RGW_OPEN_FLAG_NONE);
+  ASSERT_EQ(ret, 0);
+}
+
+TEST(LibRGW, GET_OBJECT) {
+  size_t nread;
+  off_t offset = 0;
+  struct iovec *iovs = zp_set1.get_iovs();
+  for (int ix : {2 , 3}) {
+    struct iovec *iov = &iovs[ix];
+    int ret = rgw_read(fs, object_fh, offset, iovs[ix-2].iov_len, &nread,
+		       iov->iov_base, RGW_READ_FLAG_NONE);
+    iov->iov_len = nread;
+    offset += iov->iov_len;
+    ASSERT_EQ(ret, 0);
+    ASSERT_EQ(nread, iovs[ix-2].iov_len);
+    std::cout << "read: " << static_cast<char*>(iov->iov_base) << std::endl;
+  }
+}
+
+TEST(LibRGW, CLOSE2) {
+  int ret = rgw_close(fs, object_fh, RGW_CLOSE_FLAG_NONE);
+  ASSERT_EQ(ret, 0);
+}
+#endif
+
+TEST (LibRGW, LARGE1) {
+  if (do_large) {
+    int ret;
+
+    ret = rgw_open(fs, object_fh, 0 /* posix flags */, RGW_OPEN_FLAG_NONE);
+    ASSERT_EQ(ret, 0);
+
+    size_t nbytes;
+    struct iovec *iovs = zp_set1.get_iovs();
+    off_t offset = 0;
+
+    for (int ix = 0; ix < iovcnt; ++ix) {
+      struct iovec *iov = &iovs[ix];
+      // write iov->iov_len
+      ret = rgw_write(fs, object_fh, offset, iov->iov_len, &nbytes,
+		      iov->iov_base, RGW_WRITE_FLAG_NONE);
+      offset += iov->iov_len;
+      ASSERT_EQ(ret, 0);
+      ASSERT_EQ(nbytes, iov->iov_len);
+    }
+
+    ret = rgw_close(fs, object_fh, RGW_CLOSE_FLAG_NONE);
+    ASSERT_EQ(ret, 0);
+  } /* do_large */
+}
+
+TEST (LibRGW, LARGE2) {
+  if (do_large) {
+    int ret;
+    if (do_verify) {
+      ret = rgw_open(fs, object_fh, 0 /* posix flags */, RGW_OPEN_FLAG_NONE);
+      ASSERT_EQ(ret, 0);
+
+      size_t nread;
+      off_t offset2 = 0;
+      struct iovec *iovs2 = zp_set2.get_iovs();
+      for (int ix = 0; ix < iovcnt; ++ix) {
+	struct iovec *iov2 = &iovs2[ix];
+	ret = rgw_read(fs, object_fh, offset2, iov2->iov_len, &nread,
+		       iov2->iov_base, RGW_READ_FLAG_NONE);
+	iov2->iov_len = nread;
+	offset2 += iov2->iov_len;
+	ASSERT_EQ(ret, 0);
+      }
+      zp_set1.cksum();
+      zp_set2.cksum();
+      ASSERT_TRUE(zp_set1 == zp_set2);
+
+      ret = rgw_close(fs, object_fh, RGW_CLOSE_FLAG_NONE);
+      ASSERT_EQ(ret, 0);
+    }
+  } /* do_large */
+}
+
+TEST(LibRGW, STAT_OBJECT) {
+  struct stat st;
+  int ret = rgw_getattr(fs, object_fh, &st, RGW_GETATTR_FLAG_NONE);
+  ASSERT_EQ(ret, 0);
+  dout(15) << "rgw_getattr on " << object_name << " size = "
+	   << st.st_size << dendl;
+}
+
+TEST(LibRGW, DELETE_OBJECT) {
+  if (do_delete) {
+    int ret = rgw_unlink(fs, bucket_fh, object_name.c_str(),
+			 RGW_UNLINK_FLAG_NONE);
+    ASSERT_EQ(ret, 0);
+  }
+}
+
+TEST(LibRGW, DELETE_BUCKET) {
+  if (do_delete) {
+    int ret = rgw_unlink(fs, fs->root_fh, bucket_name.c_str(),
+			 RGW_UNLINK_FLAG_NONE);
+    ASSERT_EQ(ret, 0);
+  }
+}
+
+TEST(LibRGW, CLEANUP) {
+  int ret;
+  if (object_fh) {
+    ret = rgw_fh_rele(fs, object_fh, RGW_FH_RELE_FLAG_NONE);
+    ASSERT_EQ(ret, 0);
+  }
+  ret = rgw_fh_rele(fs, bucket_fh, 0 /* flags */);
+  ASSERT_EQ(ret, 0);
+}
+
+TEST(LibRGW, UMOUNT) {
+  if (! fs)
+    return;
+
+  int ret = rgw_umount(fs, RGW_UMOUNT_FLAG_NONE);
+  ASSERT_EQ(ret, 0);
+}
+
+TEST(LibRGW, SHUTDOWN) {
+  librgw_shutdown(rgw);
+}
+
+int main(int argc, char *argv[])
+{
+  auto args = argv_to_vec(argc, argv);
+  env_to_vec(args);
+
+  char* v = getenv("AWS_ACCESS_KEY_ID");
+  if (v) {
+    access_key = v;
+  }
+
+  v = getenv("AWS_SECRET_ACCESS_KEY");
+  if (v) {
+    secret_key = v;
+  }
+
+  string val;
+
+  for (auto arg_iter = args.begin(); arg_iter != args.end();) {
+    if (ceph_argparse_witharg(args, arg_iter, &val, "--access",
+			      (char*) nullptr)) {
+      access_key = val;
+    } else if (ceph_argparse_witharg(args, arg_iter, &val, "--secret",
+				     (char*) nullptr)) {
+      secret_key = val;
+    } else if (ceph_argparse_witharg(args, arg_iter, &val, "--userid",
+				     (char*) nullptr)) {
+      userid = val;
+    } else if (ceph_argparse_witharg(args, arg_iter, &val, "--bn",
+				     (char*) nullptr)) {
+      bucket_name = val;
+    } else if (ceph_argparse_witharg(args, arg_iter, &val, "--uid",
+				     (char*) nullptr)) {
+      owner_uid = std::stoi(val);
+    } else if (ceph_argparse_witharg(args, arg_iter, &val, "--gid",
+				     (char*) nullptr)) {
+      owner_gid = std::stoi(val);
+    } else if (ceph_argparse_flag(args, arg_iter, "--verify",
+					    (char*) nullptr)) {
+      do_verify = true;
+    } else if (ceph_argparse_flag(args, arg_iter, "--create",
+					    (char*) nullptr)) {
+      do_create = true;
+    } else if (ceph_argparse_flag(args, arg_iter, "--delete",
+					    (char*) nullptr)) {
+      do_delete = true;
+    } else if (ceph_argparse_flag(args, arg_iter, "--large",
+					    (char*) nullptr)) {
+      do_large = true;
+    } else if (ceph_argparse_flag(args, arg_iter, "--hexdump",
+					    (char*) nullptr)) {
+      do_hexdump = true;
+    } else {
+      ++arg_iter;
+    }
+  }
+
+  /* don't accidentally run as anonymous */
+  if ((access_key == "") ||
+      (secret_key == "")) {
+    std::cout << argv[0] << " no AWS credentials, exiting" << std::endl;
+    return EPERM;
+  }
+
+  saved_args.argc = argc;
+  saved_args.argv = argv;
+
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}