summaryrefslogtreecommitdiffstats
path: root/src/test/librgw_file_gp.cc
diff options
context:
space:
mode:
Diffstat (limited to 'src/test/librgw_file_gp.cc')
-rw-r--r--src/test/librgw_file_gp.cc519
1 files changed, 519 insertions, 0 deletions
diff --git a/src/test/librgw_file_gp.cc b/src/test/librgw_file_gp.cc
new file mode 100644
index 000000000..4c4cfaa61
--- /dev/null
+++ b/src/test/librgw_file_gp.cc
@@ -0,0 +1,519 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2015 Red Hat, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#include <stdint.h>
+#include <tuple>
+#include <iostream>
+#include <vector>
+#include <map>
+#include <random>
+#include "xxhash.h"
+
+#include "include/rados/librgw.h"
+#include "include/rados/rgw_file.h"
+
+#include "gtest/gtest.h"
+#include "common/ceph_argparse.h"
+#include "common/debug.h"
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_rgw
+
+using namespace std;
+
+namespace {
+ librgw_t rgw = nullptr;
+ string uid("testuser");
+ string access_key("");
+ string secret_key("");
+ struct rgw_fs *fs = nullptr;
+
+ bool do_pre_list = false;
+ bool do_put = false;
+ bool do_bulk = false;
+ bool do_writev = false;
+ bool do_readv = false;
+ bool do_verify = false;
+ bool do_get = false;
+ bool do_create = false;
+ bool do_delete = false;
+ bool do_stat = false; // stat objects (not buckets)
+ bool do_hexdump = false;
+
+ bool object_open = false;
+
+ uint32_t owner_uid = 867;
+ uint32_t owner_gid = 5309;
+ uint32_t create_mask = RGW_SETATTR_UID | RGW_SETATTR_GID | RGW_SETATTR_MODE;
+
+ string bucket_name = "blastoff";
+ string object_name = "jocaml";
+
+ struct rgw_file_handle *bucket_fh = nullptr;
+ struct rgw_file_handle *object_fh = nullptr;
+
+ typedef std::tuple<string,uint64_t, struct rgw_file_handle*> fid_type;
+ std::vector<fid_type> fids;
+
+ std::uniform_int_distribution<uint8_t> uint_dist;
+ std::mt19937 rng;
+
+ constexpr int iovcnt = 16;
+ constexpr int page_size = 65536;
+ constexpr int seed = 8675309;
+
+ struct ZPage
+ {
+ char data[page_size];
+ uint64_t cksum;
+ }; /* ZPage */
+
+ struct ZPageSet
+ {
+ std::vector<ZPage*> pages;
+ struct iovec* iovs;
+
+ explicit ZPageSet(int n) {
+ pages.reserve(n);
+ iovs = (struct iovec*) calloc(n, sizeof(struct iovec));
+ for (int page_ix = 0; page_ix < n; ++page_ix) {
+ ZPage* p = new ZPage();
+ for (int data_ix = 0; data_ix < page_size; ++data_ix) {
+ p->data[data_ix] = uint_dist(rng);
+ } // data_ix
+ p->cksum = XXH64(p->data, page_size, seed);
+ pages.emplace_back(p);
+ // and iovs
+ struct iovec* iov = &iovs[page_ix];
+ iov->iov_base = p->data;
+ iov->iov_len = page_size;
+ } // page_ix
+ }
+
+ int size() { return pages.size(); }
+
+ struct iovec* get_iovs() { return iovs; }
+
+ bool operator==(const ZPageSet& rhs) {
+ int n = size();
+ for (int page_ix = 0; page_ix < n; ++page_ix) {
+ ZPage* p1 = pages[page_ix];
+ ZPage* p2 = rhs.pages[page_ix];
+ if (p1->cksum != p2->cksum)
+ return false;
+ }
+ return true;
+ }
+
+ bool operator==(const rgw_uio* uio) {
+ uint64_t cksum;
+ int vix = 0, off = 0;
+ rgw_vio* vio = &uio->uio_vio[vix];
+ int vio_len = vio->vio_len;
+ char *data;
+
+ for (int ix = 0; ix < iovcnt; ++ix) {
+ ZPage* p1 = pages[ix];
+ data = static_cast<char*>(vio->vio_base) + off;
+ cksum = XXH64(data, page_size, seed);
+
+ if (p1->cksum != cksum) {
+ int r = memcmp(data, p1->data, page_size);
+ std::cout << "problem at ix " << ix << " r " << r<< std::endl;
+ return false;
+ }
+
+ off += page_size;
+ if (off >= vio_len) {
+ vio = &uio->uio_vio[++vix];
+ vio_len = vio->vio_len;
+ off = 0;
+ }
+ }
+ return true;
+ }
+
+ void cksum() {
+ int n = size();
+ for (int page_ix = 0; page_ix < n; ++page_ix) {
+ ZPage* p = pages[page_ix];
+ p->cksum = XXH64(p->data, page_size, seed);
+ }
+ }
+
+ void reset_iovs() { // VOP_READ and VOP_WRITE update
+ int n = size();
+ for (int page_ix = 0; page_ix < n; ++page_ix) {
+ ZPage* p = pages[page_ix];
+ struct iovec* iov = &iovs[page_ix];
+ iov->iov_base = p->data;
+ iov->iov_len = page_size;
+ }
+ }
+
+ ~ZPageSet() {
+ for (unsigned int ix = 0; ix < pages.size(); ++ix)
+ delete pages[ix];
+ free(iovs);
+ }
+ }; /* ZPageSet */
+
+ rgw_uio uio[1];
+ ZPageSet zp_set1{iovcnt}; // 1M random data in 16 64K pages
+
+ struct {
+ int argc;
+ char **argv;
+ } saved_args;
+}
+
+TEST(LibRGW, INIT) {
+ int ret = librgw_create(&rgw, saved_args.argc, saved_args.argv);
+ ASSERT_EQ(ret, 0);
+ ASSERT_NE(rgw, nullptr);
+}
+
+TEST(LibRGW, MOUNT) {
+ int ret = rgw_mount2(rgw, uid.c_str(), access_key.c_str(), secret_key.c_str(),
+ "/", &fs, RGW_MOUNT_FLAG_NONE);
+ ASSERT_EQ(ret, 0);
+ ASSERT_NE(fs, nullptr);
+}
+
+TEST(LibRGW, CREATE_BUCKET) {
+ if (do_create) {
+ struct stat st;
+ struct rgw_file_handle *fh;
+
+ st.st_uid = owner_uid;
+ st.st_gid = owner_gid;
+ st.st_mode = 755;
+
+ int ret = rgw_mkdir(fs, fs->root_fh, bucket_name.c_str(), &st, create_mask,
+ &fh, RGW_MKDIR_FLAG_NONE);
+ ASSERT_EQ(ret, 0);
+ }
+}
+
+TEST(LibRGW, LOOKUP_BUCKET) {
+ int ret = rgw_lookup(fs, fs->root_fh, bucket_name.c_str(), &bucket_fh,
+ nullptr, 0, RGW_LOOKUP_FLAG_NONE);
+ ASSERT_EQ(ret, 0);
+}
+
+extern "C" {
+ static int r2_cb(const char* name, void *arg, uint64_t offset,
+ struct stat *st, uint32_t st_mask,
+ uint32_t flags) {
+ // don't need arg--it would point to fids
+ fids.push_back(fid_type(name, offset, nullptr));
+ return true; /* XXX ? */
+ }
+}
+
+TEST(LibRGW, LIST_OBJECTS) {
+ if (do_pre_list) {
+ /* list objects via readdir, bucketwise */
+ using std::get;
+
+ ldout(g_ceph_context, 0) << __func__ << " readdir on bucket "
+ << bucket_name << dendl;
+ bool eof = false;
+ uint64_t offset = 0;
+ int ret = rgw_readdir(fs, bucket_fh, &offset, r2_cb, &fids,
+ &eof, RGW_READDIR_FLAG_NONE);
+ for (auto& fid : fids) {
+ std::cout << "fname: " << get<0>(fid) << " fid: " << get<1>(fid)
+ << std::endl;
+ }
+ ASSERT_EQ(ret, 0);
+ }
+}
+
+TEST(LibRGW, LOOKUP_OBJECT) {
+ if (do_get || do_stat || do_put || do_bulk || do_readv || do_writev) {
+ int ret = rgw_lookup(fs, bucket_fh, object_name.c_str(), &object_fh,
+ nullptr, 0, RGW_LOOKUP_FLAG_CREATE);
+ ASSERT_EQ(ret, 0);
+ }
+}
+
+TEST(LibRGW, OBJ_OPEN) {
+ if (do_get || do_put || do_readv || do_writev) {
+ int ret = rgw_open(fs, object_fh, 0 /* posix flags */, 0 /* flags */);
+ ASSERT_EQ(ret, 0);
+ object_open = true;
+ }
+}
+
+TEST(LibRGW, PUT_OBJECT) {
+ if (do_put) {
+ size_t nbytes;
+ string data = "hi mom"; // fix this
+ int ret = rgw_write(fs, object_fh, 0, data.length(), &nbytes,
+ (void*) data.c_str(), RGW_WRITE_FLAG_NONE);
+ ASSERT_EQ(ret, 0);
+ ASSERT_EQ(nbytes, data.length());
+ /* commit write transaction */
+ ret = rgw_close(fs, object_fh, 0 /* flags */);
+ ASSERT_EQ(ret, 0);
+ }
+}
+
+TEST(LibRGW, GET_OBJECT) {
+ if (do_get) {
+ char sbuf[512];
+ memset(sbuf, 0, 512);
+ size_t nread;
+ int ret = rgw_read(fs, object_fh, 0 /* off */, 512 /* len */, &nread, sbuf,
+ RGW_READ_FLAG_NONE);
+ ASSERT_EQ(ret, 0);
+ buffer::list bl;
+ bl.push_back(buffer::create_static(nread, sbuf));
+ if (do_hexdump) {
+ dout(15) << "";
+ bl.hexdump(*_dout);
+ *_dout << dendl;
+ }
+ }
+}
+
+TEST(LibRGW, STAT_OBJECT) {
+ if (do_stat) {
+ struct stat st;
+ int ret = rgw_getattr(fs, object_fh, &st, RGW_GETATTR_FLAG_NONE);
+ ASSERT_EQ(ret, 0);
+ dout(15) << "rgw_getattr on " << object_name << " size = "
+ << st.st_size << dendl;
+ }
+}
+
+TEST(LibRGW, WRITE_READ_VERIFY)
+{
+ if (do_bulk && do_put) {
+ ZPageSet zp_set1{iovcnt}; // 1M random data in 16 64K pages
+ struct iovec *iovs = zp_set1.get_iovs();
+
+ /* read after write POSIX-style */
+ size_t nbytes, off = 0;
+ for (int ix = 0; ix < 16; ++ix, off += page_size) {
+ struct iovec *iov = &iovs[ix];
+ int ret = rgw_write(fs, object_fh, off, page_size, &nbytes,
+ iov->iov_base, RGW_WRITE_FLAG_NONE);
+ ASSERT_EQ(ret, 0);
+ ASSERT_EQ(nbytes, size_t(page_size));
+ }
+ zp_set1.reset_iovs();
+ }
+}
+
+/* "functions that call alloca are not inlined"
+ * --alexandre oliva
+ * http://gcc.gnu.org/ml/gcc-help/2004-04/msg00158.html
+ */
+#define alloca_uio() \
+ do {\
+ int uiosz = sizeof(rgw_uio) + iovcnt*sizeof(rgw_vio); \
+ uio = static_cast<rgw_uio*>(alloca(uiosz)); \
+ memset(uio, 0, uiosz); \
+ uio->uio_vio = reinterpret_cast<rgw_vio*>(uio+sizeof(rgw_uio)); \
+ } while (0); \
+
+TEST(LibRGW, WRITEV)
+{
+ if (do_writev) {
+ rgw_uio* uio;
+ struct iovec *iovs = zp_set1.get_iovs();
+ alloca_uio();
+ ASSERT_NE(uio, nullptr);
+
+ for (int ix = 0; ix < iovcnt; ++ix) {
+ struct iovec *iov = &iovs[ix];
+ rgw_vio *vio = &(uio->uio_vio[ix]);
+ vio->vio_base = iov->iov_base;
+ vio->vio_len = iov->iov_len;
+ vio->vio_u1 = iov; // private data
+ }
+ uio->uio_cnt = iovcnt;
+ uio->uio_offset = iovcnt * page_size;
+
+ int ret = rgw_writev(fs, object_fh, uio, RGW_WRITE_FLAG_NONE);
+ ASSERT_EQ(ret, 0);
+ }
+}
+
+TEST(LibRGW, READV)
+{
+ if (do_readv) {
+ memset(uio, 0, sizeof(rgw_uio));
+ uio->uio_offset = 0; // ok, it was already 0
+ uio->uio_resid = UINT64_MAX;
+ int ret = rgw_readv(fs, object_fh, uio, RGW_READ_FLAG_NONE);
+ ASSERT_EQ(ret, 0);
+ buffer::list bl;
+ for (unsigned int ix = 0; ix < uio->uio_cnt; ++ix) {
+ rgw_vio *vio = &(uio->uio_vio[ix]);
+ bl.push_back(
+ buffer::create_static(vio->vio_len,
+ static_cast<char*>(vio->vio_base)));
+ }
+
+ /* length check */
+ ASSERT_EQ(uint32_t{bl.length()}, uint32_t{iovcnt*page_size});
+
+ if (do_hexdump) {
+ dout(15) << "";
+ bl.hexdump(*_dout);
+ *_dout << dendl;
+ }
+ }
+}
+
+TEST(LibRGW, READV_AFTER_WRITEV)
+{
+ /* checksum data */
+ if (do_readv && do_writev && do_verify) {
+ ASSERT_TRUE(zp_set1 == uio);
+ }
+}
+
+TEST(LibRGW, DELETE_OBJECT) {
+ if (do_delete) {
+ int ret = rgw_unlink(fs, bucket_fh, object_name.c_str(),
+ RGW_UNLINK_FLAG_NONE);
+ ASSERT_EQ(ret, 0);
+ }
+}
+
+TEST(LibRGW, DELETE_BUCKET) {
+ if (do_delete) {
+ int ret = rgw_unlink(fs, fs->root_fh, bucket_name.c_str(),
+ RGW_UNLINK_FLAG_NONE);
+ ASSERT_EQ(ret, 0);
+ }
+}
+
+TEST(LibRGW, CLEANUP) {
+ if (do_readv) {
+ // release resources
+ ASSERT_NE(uio->uio_rele, nullptr);
+ if (uio->uio_rele) {
+ uio->uio_rele(uio, RGW_UIO_NONE);
+ }
+ }
+ int ret;
+ if (object_open) {
+ ret = rgw_close(fs, object_fh, RGW_CLOSE_FLAG_NONE);
+ ASSERT_EQ(ret, 0);
+ }
+ if (object_fh) {
+ ret = rgw_fh_rele(fs, object_fh, 0 /* flags */);
+ ASSERT_EQ(ret, 0);
+ }
+ ret = rgw_fh_rele(fs, bucket_fh, 0 /* flags */);
+ ASSERT_EQ(ret, 0);
+}
+
+TEST(LibRGW, UMOUNT) {
+ if (! fs)
+ return;
+
+ int ret = rgw_umount(fs, RGW_UMOUNT_FLAG_NONE);
+ ASSERT_EQ(ret, 0);
+}
+
+TEST(LibRGW, SHUTDOWN) {
+ librgw_shutdown(rgw);
+}
+
+int main(int argc, char *argv[])
+{
+ auto args = argv_to_vec(argc, argv);
+ env_to_vec(args);
+
+ char* v = getenv("AWS_ACCESS_KEY_ID");
+ if (v) {
+ access_key = v;
+ }
+
+ v = getenv("AWS_SECRET_ACCESS_KEY");
+ if (v) {
+ secret_key = v;
+ }
+
+ string val;
+ for (auto arg_iter = args.begin(); arg_iter != args.end();) {
+ if (ceph_argparse_witharg(args, arg_iter, &val, "--access",
+ (char*) nullptr)) {
+ access_key = val;
+ } else if (ceph_argparse_witharg(args, arg_iter, &val, "--secret",
+ (char*) nullptr)) {
+ secret_key = val;
+ } else if (ceph_argparse_witharg(args, arg_iter, &val, "--uid",
+ (char*) nullptr)) {
+ uid = val;
+ } else if (ceph_argparse_witharg(args, arg_iter, &val, "--bn",
+ (char*) nullptr)) {
+ bucket_name = val;
+ } else if (ceph_argparse_flag(args, arg_iter, "--get",
+ (char*) nullptr)) {
+ do_get = true;
+ } else if (ceph_argparse_flag(args, arg_iter, "--stat",
+ (char*) nullptr)) {
+ do_stat = true;
+ } else if (ceph_argparse_flag(args, arg_iter, "--put",
+ (char*) nullptr)) {
+ do_put = true;
+ } else if (ceph_argparse_flag(args, arg_iter, "--bulk",
+ (char*) nullptr)) {
+ do_bulk = true;
+ } else if (ceph_argparse_flag(args, arg_iter, "--writev",
+ (char*) nullptr)) {
+ do_writev = true;
+ } else if (ceph_argparse_flag(args, arg_iter, "--readv",
+ (char*) nullptr)) {
+ do_readv = true;
+ } else if (ceph_argparse_flag(args, arg_iter, "--verify",
+ (char*) nullptr)) {
+ do_verify = true;
+ } else if (ceph_argparse_flag(args, arg_iter, "--delete",
+ (char*) nullptr)) {
+ do_delete = true;
+ } else if (ceph_argparse_flag(args, arg_iter, "--prelist",
+ (char*) nullptr)) {
+ do_pre_list = true;
+ } else if (ceph_argparse_flag(args, arg_iter, "--create",
+ (char*) nullptr)) {
+ do_create = true;
+ } else if (ceph_argparse_flag(args, arg_iter, "--hexdump",
+ (char*) nullptr)) {
+ do_hexdump = true;
+ } else {
+ ++arg_iter;
+ }
+ }
+
+ /* don't accidentally run as anonymous */
+ if ((access_key == "") ||
+ (secret_key == "")) {
+ std::cout << argv[0] << " no AWS credentials, exiting" << std::endl;
+ return EPERM;
+ }
+
+ saved_args.argc = argc;
+ saved_args.argv = argv;
+
+ ::testing::InitGoogleTest(&argc, argv);
+ return RUN_ALL_TESTS();
+}