diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-27 18:24:20 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-27 18:24:20 +0000 |
commit | 483eb2f56657e8e7f419ab1a4fab8dce9ade8609 (patch) | |
tree | e5d88d25d870d5dedacb6bbdbe2a966086a0a5cf /src/spdk/test/blobfs | |
parent | Initial commit. (diff) | |
download | ceph-upstream.tar.xz ceph-upstream.zip |
Adding upstream version 14.2.21.upstream/14.2.21upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'src/spdk/test/blobfs')
-rw-r--r-- | src/spdk/test/blobfs/Makefile | 49 | ||||
-rw-r--r-- | src/spdk/test/blobfs/fuse/.gitignore | 1 | ||||
-rw-r--r-- | src/spdk/test/blobfs/fuse/Makefile | 60 | ||||
-rw-r--r-- | src/spdk/test/blobfs/fuse/fuse.c | 348 | ||||
-rw-r--r-- | src/spdk/test/blobfs/mkfs/.gitignore | 1 | ||||
-rw-r--r-- | src/spdk/test/blobfs/mkfs/Makefile | 59 | ||||
-rw-r--r-- | src/spdk/test/blobfs/mkfs/mkfs.c | 150 | ||||
-rw-r--r-- | src/spdk/test/blobfs/rocksdb/.gitignore | 1 | ||||
-rw-r--r-- | src/spdk/test/blobfs/rocksdb/common_flags.txt | 27 | ||||
-rwxr-xr-x | src/spdk/test/blobfs/rocksdb/postprocess.py | 70 | ||||
-rwxr-xr-x | src/spdk/test/blobfs/rocksdb/rocksdb.sh | 134 | ||||
-rwxr-xr-x | src/spdk/test/blobfs/rocksdb/run_tests.sh | 197 | ||||
-rw-r--r-- | src/spdk/test/blobfs/test_plan.md | 67 |
13 files changed, 1164 insertions, 0 deletions
diff --git a/src/spdk/test/blobfs/Makefile b/src/spdk/test/blobfs/Makefile new file mode 100644 index 00000000..1a9dfefa --- /dev/null +++ b/src/spdk/test/blobfs/Makefile @@ -0,0 +1,49 @@ +# +# BSD LICENSE +# +# Copyright (c) Intel Corporation. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..) +include $(SPDK_ROOT_DIR)/mk/spdk.common.mk + +DIRS-y = mkfs + +# TODO: do not check a hardcoded path here +ifneq (,$(wildcard /usr/local/include/fuse3)) +DIRS-y += fuse +endif + +.PHONY: all clean $(DIRS-y) + +all: $(DIRS-y) +clean: $(DIRS-y) + +include $(SPDK_ROOT_DIR)/mk/spdk.subdirs.mk diff --git a/src/spdk/test/blobfs/fuse/.gitignore b/src/spdk/test/blobfs/fuse/.gitignore new file mode 100644 index 00000000..a517c488 --- /dev/null +++ b/src/spdk/test/blobfs/fuse/.gitignore @@ -0,0 +1 @@ +fuse diff --git a/src/spdk/test/blobfs/fuse/Makefile b/src/spdk/test/blobfs/fuse/Makefile new file mode 100644 index 00000000..847da50e --- /dev/null +++ b/src/spdk/test/blobfs/fuse/Makefile @@ -0,0 +1,60 @@ +# +# BSD LICENSE +# +# Copyright (c) Intel Corporation. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..) +include $(SPDK_ROOT_DIR)/mk/spdk.common.mk +include $(SPDK_ROOT_DIR)/mk/spdk.app.mk +include $(SPDK_ROOT_DIR)/mk/spdk.modules.mk + +APP = fuse + +C_SRCS := fuse.c + +SPDK_LIB_LIST = event_bdev event_copy +SPDK_LIB_LIST += blobfs blob bdev blob_bdev copy event thread util conf trace \ + log jsonrpc json rpc + +LIBS += $(COPY_MODULES_LINKER_ARGS) $(BLOCKDEV_MODULES_LINKER_ARGS) +LIBS += $(SPDK_LIB_LINKER_ARGS) $(ENV_LINKER_ARGS) +LIBS+= -L/usr/local/lib -lfuse3 + +all : $(APP) + @: + +$(APP) : $(OBJS) $(SPDK_LIB_FILES) $(COPY_MODULES_FILES) $(BLOCKDEV_MODULES_FILES) $(ENV_LIBS) + $(LINK_C) + +clean : + $(CLEAN_C) $(APP) + +include $(SPDK_ROOT_DIR)/mk/spdk.deps.mk diff --git a/src/spdk/test/blobfs/fuse/fuse.c b/src/spdk/test/blobfs/fuse/fuse.c new file mode 100644 index 00000000..9d11dfa0 --- /dev/null +++ b/src/spdk/test/blobfs/fuse/fuse.c @@ -0,0 +1,348 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "spdk/stdinc.h" + +#define FUSE_USE_VERSION 30 +#include "fuse3/fuse.h" +#include "fuse3/fuse_lowlevel.h" + +#include "spdk/blobfs.h" +#include "spdk/bdev.h" +#include "spdk/event.h" +#include "spdk/thread.h" +#include "spdk/blob_bdev.h" +#include "spdk/log.h" + +struct fuse *g_fuse; +char *g_bdev_name; +char *g_mountpoint; +pthread_t g_fuse_thread; + +struct spdk_bs_dev *g_bs_dev; +struct spdk_filesystem *g_fs; +struct spdk_io_channel *g_channel; +struct spdk_file *g_file; +int g_fserrno; +int g_fuse_argc = 0; +char **g_fuse_argv = NULL; + +static void +__call_fn(void *arg1, void *arg2) +{ + fs_request_fn fn; + + fn = (fs_request_fn)arg1; + fn(arg2); +} + +static void +__send_request(fs_request_fn fn, void *arg) +{ + struct spdk_event *event; + + event = spdk_event_allocate(0, __call_fn, (void *)fn, arg); + spdk_event_call(event); +} + +static int +spdk_fuse_getattr(const char *path, struct stat *stbuf, struct fuse_file_info *fi) +{ + struct spdk_file_stat stat; + int rc; + + if (!strcmp(path, "/")) { + stbuf->st_mode = S_IFDIR | 0755; + stbuf->st_nlink = 2; + return 0; + } + + rc = spdk_fs_file_stat(g_fs, g_channel, path, &stat); + if (rc == 0) { + stbuf->st_mode = S_IFREG | 0644; + stbuf->st_nlink = 1; + stbuf->st_size = stat.size; + } + + return rc; +} + +static int +spdk_fuse_readdir(const char *path, void *buf, fuse_fill_dir_t filler, + off_t offset, struct fuse_file_info *fi, + enum fuse_readdir_flags flags) +{ + struct spdk_file *file; + const char *filename; + spdk_fs_iter iter; + + filler(buf, ".", NULL, 0, 0); + filler(buf, "..", NULL, 0, 0); + + iter = spdk_fs_iter_first(g_fs); + while (iter != NULL) { + file = spdk_fs_iter_get_file(iter); + iter = spdk_fs_iter_next(iter); + filename = spdk_file_get_name(file); + filler(buf, &filename[1], NULL, 0, 0); + } + + return 0; +} + +static int +spdk_fuse_mknod(const char *path, mode_t mode, dev_t rdev) +{ + return spdk_fs_create_file(g_fs, g_channel, path); +} + +static int +spdk_fuse_unlink(const char *path) +{ + return spdk_fs_delete_file(g_fs, g_channel, path); +} + +static int +spdk_fuse_truncate(const char *path, off_t size, struct fuse_file_info *fi) +{ + struct spdk_file *file; + int rc; + + rc = spdk_fs_open_file(g_fs, g_channel, path, 0, &file); + if (rc != 0) { + return -rc; + } + + rc = spdk_file_truncate(file, g_channel, size); + if (rc != 0) { + return -rc; + } + + spdk_file_close(file, g_channel); + + return 0; +} + +static int +spdk_fuse_utimens(const char *path, const struct timespec tv[2], struct fuse_file_info *fi) +{ + return 0; +} + +static int +spdk_fuse_open(const char *path, struct fuse_file_info *info) +{ + struct spdk_file *file; + int rc; + + rc = spdk_fs_open_file(g_fs, g_channel, path, 0, &file); + if (rc != 0) { + return -rc; + } + + info->fh = (uintptr_t)file; + return 0; +} + +static int +spdk_fuse_release(const char *path, struct fuse_file_info *info) +{ + struct spdk_file *file = (struct spdk_file *)info->fh; + + return spdk_file_close(file, g_channel); +} + +static int +spdk_fuse_read(const char *path, char *buf, size_t len, off_t offset, struct fuse_file_info *info) +{ + struct spdk_file *file = (struct spdk_file *)info->fh; + + return spdk_file_read(file, g_channel, buf, offset, len); +} + +static int +spdk_fuse_write(const char *path, const char *buf, size_t len, off_t offset, + struct fuse_file_info *info) +{ + struct spdk_file *file = (struct spdk_file *)info->fh; + int rc; + + rc = spdk_file_write(file, g_channel, (void *)buf, offset, len); + if (rc == 0) { + return len; + } else { + return rc; + } +} + +static int +spdk_fuse_flush(const char *path, struct fuse_file_info *info) +{ + return 0; +} + +static int +spdk_fuse_fsync(const char *path, int datasync, struct fuse_file_info *info) +{ + return 0; +} + +static int +spdk_fuse_rename(const char *old_path, const char *new_path, unsigned int flags) +{ + return spdk_fs_rename_file(g_fs, g_channel, old_path, new_path); +} + +static struct fuse_operations spdk_fuse_oper = { + .getattr = spdk_fuse_getattr, + .readdir = spdk_fuse_readdir, + .mknod = spdk_fuse_mknod, + .unlink = spdk_fuse_unlink, + .truncate = spdk_fuse_truncate, + .utimens = spdk_fuse_utimens, + .open = spdk_fuse_open, + .release = spdk_fuse_release, + .read = spdk_fuse_read, + .write = spdk_fuse_write, + .flush = spdk_fuse_flush, + .fsync = spdk_fuse_fsync, + .rename = spdk_fuse_rename, +}; + +static void +construct_targets(void) +{ + struct spdk_bdev *bdev; + + bdev = spdk_bdev_get_by_name(g_bdev_name); + if (bdev == NULL) { + SPDK_ERRLOG("bdev %s not found\n", g_bdev_name); + exit(1); + } + + g_bs_dev = spdk_bdev_create_bs_dev(bdev, NULL, NULL); + + printf("Mounting BlobFS on bdev %s\n", spdk_bdev_get_name(bdev)); +} + +static void +start_fuse_fn(void *arg1, void *arg2) +{ + struct fuse_args args = FUSE_ARGS_INIT(g_fuse_argc, g_fuse_argv); + int rc; + struct fuse_cmdline_opts opts = {}; + + g_fuse_thread = pthread_self(); + rc = fuse_parse_cmdline(&args, &opts); + if (rc != 0) { + spdk_app_stop(-1); + fuse_opt_free_args(&args); + return; + } + g_fuse = fuse_new(&args, &spdk_fuse_oper, sizeof(spdk_fuse_oper), NULL); + fuse_opt_free_args(&args); + + rc = fuse_mount(g_fuse, g_mountpoint); + if (rc != 0) { + spdk_app_stop(-1); + return; + } + + fuse_daemonize(true /* true = run in foreground */); + + fuse_loop(g_fuse); + + fuse_unmount(g_fuse); + fuse_destroy(g_fuse); +} + +static void +init_cb(void *ctx, struct spdk_filesystem *fs, int fserrno) +{ + struct spdk_event *event; + + g_fs = fs; + g_channel = spdk_fs_alloc_io_channel_sync(g_fs); + event = spdk_event_allocate(1, start_fuse_fn, NULL, NULL); + spdk_event_call(event); +} + +static void +spdk_fuse_run(void *arg1, void *arg2) +{ + construct_targets(); + spdk_fs_load(g_bs_dev, __send_request, init_cb, NULL); +} + +static void +shutdown_cb(void *ctx, int fserrno) +{ + fuse_session_exit(fuse_get_session(g_fuse)); + pthread_kill(g_fuse_thread, SIGINT); + spdk_fs_free_io_channel(g_channel); + spdk_app_stop(0); +} + +static void +spdk_fuse_shutdown(void) +{ + spdk_fs_unload(g_fs, shutdown_cb, NULL); +} + +int main(int argc, char **argv) +{ + struct spdk_app_opts opts = {}; + int rc = 0; + + if (argc < 4) { + fprintf(stderr, "usage: %s <conffile> <bdev name> <mountpoint>\n", argv[0]); + exit(1); + } + + spdk_app_opts_init(&opts); + opts.name = "spdk_fuse"; + opts.config_file = argv[1]; + opts.reactor_mask = "0x3"; + opts.mem_size = 6144; + opts.shutdown_cb = spdk_fuse_shutdown; + + g_bdev_name = argv[2]; + g_mountpoint = argv[3]; + g_fuse_argc = argc - 2; + g_fuse_argv = &argv[2]; + + rc = spdk_app_start(&opts, spdk_fuse_run, NULL, NULL); + spdk_app_fini(); + + return rc; +} diff --git a/src/spdk/test/blobfs/mkfs/.gitignore b/src/spdk/test/blobfs/mkfs/.gitignore new file mode 100644 index 00000000..54e292c6 --- /dev/null +++ b/src/spdk/test/blobfs/mkfs/.gitignore @@ -0,0 +1 @@ +mkfs diff --git a/src/spdk/test/blobfs/mkfs/Makefile b/src/spdk/test/blobfs/mkfs/Makefile new file mode 100644 index 00000000..8367571e --- /dev/null +++ b/src/spdk/test/blobfs/mkfs/Makefile @@ -0,0 +1,59 @@ +# +# BSD LICENSE +# +# Copyright (c) Intel Corporation. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..) +include $(SPDK_ROOT_DIR)/mk/spdk.common.mk +include $(SPDK_ROOT_DIR)/mk/spdk.app.mk +include $(SPDK_ROOT_DIR)/mk/spdk.modules.mk + +APP = mkfs + +C_SRCS := mkfs.c + +SPDK_LIB_LIST = event_bdev event_copy +SPDK_LIB_LIST += blobfs blob bdev blob_bdev copy event thread util conf trace \ + log jsonrpc json rpc + +LIBS += $(COPY_MODULES_LINKER_ARGS) $(BLOCKDEV_MODULES_LINKER_ARGS) $(SOCK_MODULES_LINKER_ARGS) +LIBS += $(SPDK_LIB_LINKER_ARGS) $(ENV_LINKER_ARGS) + +all : $(APP) + @: + +$(APP) : $(OBJS) $(SPDK_LIB_FILES) $(COPY_MODULES_FILES) $(BLOCKDEV_MODULES_FILES) $(SOCK_MODULES_FILES) $(ENV_LIBS) + $(LINK_C) + +clean : + $(CLEAN_C) $(APP) + +include $(SPDK_ROOT_DIR)/mk/spdk.deps.mk diff --git a/src/spdk/test/blobfs/mkfs/mkfs.c b/src/spdk/test/blobfs/mkfs/mkfs.c new file mode 100644 index 00000000..0bedd84a --- /dev/null +++ b/src/spdk/test/blobfs/mkfs/mkfs.c @@ -0,0 +1,150 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "spdk/stdinc.h" + +#include "spdk/blobfs.h" +#include "spdk/bdev.h" +#include "spdk/event.h" +#include "spdk/blob_bdev.h" +#include "spdk/log.h" +#include "spdk/string.h" + +struct spdk_bs_dev *g_bs_dev; +const char *g_bdev_name; +static uint64_t g_cluster_size; + +static void +stop_cb(void *ctx, int fserrno) +{ + spdk_app_stop(0); +} + +static void +shutdown_cb(void *arg1, void *arg2) +{ + struct spdk_filesystem *fs = arg1; + + printf("done.\n"); + spdk_fs_unload(fs, stop_cb, NULL); +} + +static void +init_cb(void *ctx, struct spdk_filesystem *fs, int fserrno) +{ + struct spdk_event *event; + + event = spdk_event_allocate(0, shutdown_cb, fs, NULL); + spdk_event_call(event); +} + +static void +spdk_mkfs_run(void *arg1, void *arg2) +{ + struct spdk_bdev *bdev; + struct spdk_blobfs_opts blobfs_opt; + + bdev = spdk_bdev_get_by_name(g_bdev_name); + + if (bdev == NULL) { + SPDK_ERRLOG("bdev %s not found\n", g_bdev_name); + spdk_app_stop(-1); + return; + } + + printf("Initializing filesystem on bdev %s...", g_bdev_name); + fflush(stdout); + + spdk_fs_opts_init(&blobfs_opt); + if (g_cluster_size) { + blobfs_opt.cluster_sz = g_cluster_size; + } + g_bs_dev = spdk_bdev_create_bs_dev(bdev, NULL, NULL); + if (blobfs_opt.cluster_sz) { + spdk_fs_init(g_bs_dev, &blobfs_opt, NULL, init_cb, NULL); + } else { + spdk_fs_init(g_bs_dev, NULL, NULL, init_cb, NULL); + } +} + +static void +mkfs_usage(void) +{ + printf(" -C <size> cluster size\n"); +} + +static void +mkfs_parse_arg(int ch, char *arg) +{ + bool has_prefix; + + switch (ch) { + case 'C': + spdk_parse_capacity(arg, &g_cluster_size, &has_prefix); + break; + default: + break; + } + +} + +int main(int argc, char **argv) +{ + struct spdk_app_opts opts = {}; + int rc = 0; + + if (argc < 3) { + SPDK_ERRLOG("usage: %s <conffile> <bdevname>\n", argv[0]); + exit(1); + } + + spdk_app_opts_init(&opts); + opts.name = "spdk_mkfs"; + opts.config_file = argv[1]; + opts.reactor_mask = "0x3"; + opts.mem_size = 1024; + opts.shutdown_cb = NULL; + + spdk_fs_set_cache_size(512); + g_bdev_name = argv[2]; + if ((rc = spdk_app_parse_args(argc, argv, &opts, "C:", NULL, + mkfs_parse_arg, mkfs_usage)) != + SPDK_APP_PARSE_ARGS_SUCCESS) { + exit(rc); + } + + rc = spdk_app_start(&opts, spdk_mkfs_run, NULL, NULL); + spdk_app_fini(); + + return rc; +} diff --git a/src/spdk/test/blobfs/rocksdb/.gitignore b/src/spdk/test/blobfs/rocksdb/.gitignore new file mode 100644 index 00000000..1a06816d --- /dev/null +++ b/src/spdk/test/blobfs/rocksdb/.gitignore @@ -0,0 +1 @@ +results diff --git a/src/spdk/test/blobfs/rocksdb/common_flags.txt b/src/spdk/test/blobfs/rocksdb/common_flags.txt new file mode 100644 index 00000000..6390c7a4 --- /dev/null +++ b/src/spdk/test/blobfs/rocksdb/common_flags.txt @@ -0,0 +1,27 @@ +--disable_seek_compaction=1 +--mmap_read=0 +--statistics=1 +--histogram=1 +--key_size=16 +--value_size=1000 +--block_size=4096 +--cache_size=0 +--bloom_bits=10 +--cache_numshardbits=4 +--open_files=500000 +--verify_checksum=1 +--db=/mnt/rocksdb +--sync=0 +--compression_type=none +--stats_interval=1000000 +--compression_ratio=1 +--disable_data_sync=0 +--target_file_size_base=67108864 +--max_write_buffer_number=3 +--max_bytes_for_level_multiplier=10 +--max_background_compactions=10 +--num_levels=10 +--delete_obsolete_files_period_micros=3000000 +--max_grandparent_overlap_factor=10 +--stats_per_interval=1 +--max_bytes_for_level_base=10485760 diff --git a/src/spdk/test/blobfs/rocksdb/postprocess.py b/src/spdk/test/blobfs/rocksdb/postprocess.py new file mode 100755 index 00000000..1ba8a730 --- /dev/null +++ b/src/spdk/test/blobfs/rocksdb/postprocess.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +from collections import namedtuple +from itertools import islice +import operator +import sys + +total_samples = 0 +thread_module_samples = {} +function_module_samples = {} +module_samples = {} +threads = set() + +ThreadModule = namedtuple('ThreadModule', ['thread', 'module']) +FunctionModule = namedtuple('FunctionModule', ['function', 'module']) + +with open(sys.argv[1] + "/" + sys.argv[2] + ".perf.txt") as f: + for line in f: + fields = line.split() + total_samples += int(fields[1]) + key = ThreadModule(fields[2], fields[3]) + thread_module_samples.setdefault(key, 0) + thread_module_samples[key] += int(fields[1]) + key = FunctionModule(fields[5], fields[3]) + function_module_samples.setdefault(key, 0) + function_module_samples[key] += int(fields[1]) + threads.add(fields[2]) + + key = fields[3] + module_samples.setdefault(key, 0) + module_samples[key] += int(fields[1]) + +for thread in sorted(threads): + thread_pct = 0 + print("") + print("Thread: {:s}".format(thread)) + print(" Percent Module") + print("============================") + for key, value in sorted(list(thread_module_samples.items()), key=operator.itemgetter(1), reverse=True): + if key.thread == thread: + print("{:8.4f} {:20s}".format(float(value) * 100 / total_samples, key.module)) + thread_pct += float(value) * 100 / total_samples + print("============================") + print("{:8.4f} Total".format(thread_pct)) + +print("") +print(" Percent Module Function") +print("=================================================================") +for key, value in islice(sorted(list(function_module_samples.items()), key=operator.itemgetter(1), reverse=True), 100): + print(("{:8.4f} {:20s} {:s}".format(float(value) * 100 / total_samples, key.module, key.function))) + +print("") +print("") +print(" Percent Module") +print("=================================") +for key, value in sorted(list(module_samples.items()), key=operator.itemgetter(1), reverse=True): + print("{:8.4f} {:s}".format(float(value) * 100 / total_samples, key)) + +print("") +with open(sys.argv[1] + "/" + sys.argv[2] + "_db_bench.txt") as f: + for line in f: + if "maxresident" in line: + fields = line.split() + print("Wall time elapsed: {:s}".format(fields[2].split("e")[0])) + print("CPU utilization: {:s}".format(fields[3].split('C')[0])) + user = float(fields[0].split('u')[0]) + system = float(fields[1].split('s')[0]) + print("User: {:8.2f} ({:5.2f}%)".format(user, user * 100 / (user + system))) + print("System: {:8.2f} ({:5.2f}%)".format(system, system * 100 / (user + system))) + +print("") diff --git a/src/spdk/test/blobfs/rocksdb/rocksdb.sh b/src/spdk/test/blobfs/rocksdb/rocksdb.sh new file mode 100755 index 00000000..1a73ee55 --- /dev/null +++ b/src/spdk/test/blobfs/rocksdb/rocksdb.sh @@ -0,0 +1,134 @@ +#!/usr/bin/env bash + +set -ex + +run_step() { + if [ -z "$1" ]; then + echo run_step called with no parameter + exit 1 + fi + + echo "--spdk=$ROCKSDB_CONF" >> "$1"_flags.txt + echo "--spdk_bdev=Nvme0n1" >> "$1"_flags.txt + echo "--spdk_cache_size=$CACHE_SIZE" >> "$1"_flags.txt + + echo -n Start $1 test phase... + /usr/bin/time taskset 0xFF $DB_BENCH --flagfile="$1"_flags.txt &> "$1"_db_bench.txt + echo done. +} + +run_bsdump() { + $rootdir/examples/blob/cli/blobcli -c $ROCKSDB_CONF -b Nvme0n1 -D &> bsdump.txt +} + +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../../..) +source $rootdir/test/common/autotest_common.sh + +DB_BENCH_DIR=/usr/src/rocksdb +DB_BENCH=$DB_BENCH_DIR/db_bench +ROCKSDB_CONF=$testdir/rocksdb.conf + +if [ ! -e $DB_BENCH_DIR ]; then + echo $DB_BENCH_DIR does not exist, skipping rocksdb tests + exit 0 +fi + +timing_enter rocksdb + +timing_enter db_bench_build + +pushd $DB_BENCH_DIR +git clean -x -f -d +$MAKE db_bench $MAKEFLAGS $MAKECONFIG DEBUG_LEVEL=0 SPDK_DIR=$rootdir +popd + +timing_exit db_bench_build + +$rootdir/scripts/gen_nvme.sh > $ROCKSDB_CONF + +trap 'run_bsdump; rm -f $ROCKSDB_CONF; exit 1' SIGINT SIGTERM EXIT + +timing_enter mkfs +$rootdir/test/blobfs/mkfs/mkfs $ROCKSDB_CONF Nvme0n1 +timing_exit mkfs + +mkdir $output_dir/rocksdb +RESULTS_DIR=$output_dir/rocksdb +CACHE_SIZE=4096 +if [ $RUN_NIGHTLY_FAILING -eq 1 ]; then + DURATION=60 + NUM_KEYS=100000000 +else + DURATION=20 + NUM_KEYS=20000000 +fi + +cd $RESULTS_DIR +cp $testdir/common_flags.txt insert_flags.txt +echo "--benchmarks=fillseq" >> insert_flags.txt +echo "--threads=1" >> insert_flags.txt +echo "--disable_wal=1" >> insert_flags.txt +echo "--use_existing_db=0" >> insert_flags.txt +echo "--num=$NUM_KEYS" >> insert_flags.txt + +cp $testdir/common_flags.txt randread_flags.txt +echo "--benchmarks=readrandom" >> randread_flags.txt +echo "--threads=16" >> randread_flags.txt +echo "--duration=$DURATION" >> randread_flags.txt +echo "--disable_wal=1" >> randread_flags.txt +echo "--use_existing_db=1" >> randread_flags.txt +echo "--num=$NUM_KEYS" >> randread_flags.txt + +cp $testdir/common_flags.txt overwrite_flags.txt +echo "--benchmarks=overwrite" >> overwrite_flags.txt +echo "--threads=1" >> overwrite_flags.txt +echo "--duration=$DURATION" >> overwrite_flags.txt +echo "--disable_wal=1" >> overwrite_flags.txt +echo "--use_existing_db=1" >> overwrite_flags.txt +echo "--num=$NUM_KEYS" >> overwrite_flags.txt + +cp $testdir/common_flags.txt readwrite_flags.txt +echo "--benchmarks=readwhilewriting" >> readwrite_flags.txt +echo "--threads=4" >> readwrite_flags.txt +echo "--duration=$DURATION" >> readwrite_flags.txt +echo "--disable_wal=1" >> readwrite_flags.txt +echo "--use_existing_db=1" >> readwrite_flags.txt +echo "--num=$NUM_KEYS" >> readwrite_flags.txt + +cp $testdir/common_flags.txt writesync_flags.txt +echo "--benchmarks=overwrite" >> writesync_flags.txt +echo "--threads=1" >> writesync_flags.txt +echo "--duration=$DURATION" >> writesync_flags.txt +echo "--disable_wal=0" >> writesync_flags.txt +echo "--use_existing_db=1" >> writesync_flags.txt +echo "--sync=1" >> writesync_flags.txt +echo "--num=$NUM_KEYS" >> writesync_flags.txt + +timing_enter rocksdb_insert +run_step insert +timing_exit rocksdb_insert + +timing_enter rocksdb_overwrite +run_step overwrite +timing_exit rocksdb_overwrite + +timing_enter rocksdb_readwrite +run_step readwrite +timing_exit rocksdb_readwrite + +timing_enter rocksdb_writesync +run_step writesync +timing_exit rocksdb_writesync + +timing_enter rocksdb_randread +run_step randread +timing_exit rocksdb_randread + +trap - SIGINT SIGTERM EXIT + +run_bsdump +rm -f $ROCKSDB_CONF + +report_test_completion "blobfs" +timing_exit rocksdb diff --git a/src/spdk/test/blobfs/rocksdb/run_tests.sh b/src/spdk/test/blobfs/rocksdb/run_tests.sh new file mode 100755 index 00000000..d65d3b10 --- /dev/null +++ b/src/spdk/test/blobfs/rocksdb/run_tests.sh @@ -0,0 +1,197 @@ +#!/bin/bash +set -e + +if [ $# -eq 0 ] +then + echo "usage: $0 <location of db_bench>" + exit 1 +fi + +DB_BENCH=$(readlink -f $1) +[ -e $DB_BENCH ] || (echo "$DB_BENCH does not exist - needs to be built" && exit 1) + +hash mkfs.xfs +: ${USE_PERF:=1} +if ! hash perf; then + USE_PERF=0 +fi +hash python +[ -e /usr/include/gflags/gflags.h ] || (echo "gflags not installed." && exit 1) + +# Increase max number of file descriptors. This will be inherited +# by processes spawned from this script. +ulimit -n 16384 + +TESTDIR=$(readlink -f $(dirname $0)) + +if ls $TESTDIR/results/testrun_* &> /dev/null; then + mkdir -p $TESTDIR/results/old + mv $TESTDIR/results/testrun_* $TESTDIR/results/old +fi + +if [ -z "$RESULTS_DIR" ]; then + RESULTS_DIR=$TESTDIR/results/testrun_`date +%Y%m%d_%H%M%S` + mkdir -p $RESULTS_DIR + rm -f $TESTDIR/results/last + ln -s $RESULTS_DIR $TESTDIR/results/last +fi + +: ${CACHE_SIZE:=4096} +: ${DURATION:=120} +: ${NUM_KEYS:=500000000} +: ${ROCKSDB_CONF:=/usr/local/etc/spdk/rocksdb.conf} + +if [ "$NO_SPDK" = "1" ] +then + [ -e /dev/nvme0n1 ] || (echo "No /dev/nvme0n1 device node found." && exit 1) +else + [ -e /dev/nvme0n1 ] && (echo "/dev/nvme0n1 device found - need to run SPDK setup.sh script to bind to UIO." && exit 1) +fi + +cd $RESULTS_DIR + +SYSINFO_FILE=sysinfo.txt +COMMAND="hostname" +echo ">> $COMMAND : " >> $SYSINFO_FILE +$COMMAND >> $SYSINFO_FILE +echo >> $SYSINFO_FILE + +COMMAND="cat /proc/cpuinfo" +echo ">> $COMMAND : " >> $SYSINFO_FILE +$COMMAND >> $SYSINFO_FILE +echo >> $SYSINFO_FILE + +COMMAND="cat /proc/meminfo" +echo ">> $COMMAND : " >> $SYSINFO_FILE +$COMMAND >> $SYSINFO_FILE +echo >> $SYSINFO_FILE + +if [ "$NO_SPDK" = "1" ] +then + echo -n Creating and mounting XFS filesystem... + sudo mkdir -p /mnt/rocksdb + sudo umount /mnt/rocksdb || true &> /dev/null + sudo mkfs.xfs -d agcount=32 -l su=4096 -f /dev/nvme0n1 &> mkfs_xfs.txt + sudo mount -o discard /dev/nvme0n1 /mnt/rocksdb + sudo chown $USER /mnt/rocksdb + echo done. +fi + +cp $TESTDIR/common_flags.txt insert_flags.txt +echo "--benchmarks=fillseq" >> insert_flags.txt +echo "--threads=1" >> insert_flags.txt +echo "--disable_wal=1" >> insert_flags.txt +echo "--use_existing_db=0" >> insert_flags.txt +echo "--num=$NUM_KEYS" >> insert_flags.txt + +cp $TESTDIR/common_flags.txt randread_flags.txt +echo "--benchmarks=readrandom" >> randread_flags.txt +echo "--threads=16" >> randread_flags.txt +echo "--duration=$DURATION" >> randread_flags.txt +echo "--disable_wal=1" >> randread_flags.txt +echo "--use_existing_db=1" >> randread_flags.txt +echo "--num=$NUM_KEYS" >> randread_flags.txt + +cp $TESTDIR/common_flags.txt overwrite_flags.txt +echo "--benchmarks=overwrite" >> overwrite_flags.txt +echo "--threads=1" >> overwrite_flags.txt +echo "--duration=$DURATION" >> overwrite_flags.txt +echo "--disable_wal=1" >> overwrite_flags.txt +echo "--use_existing_db=1" >> overwrite_flags.txt +echo "--num=$NUM_KEYS" >> overwrite_flags.txt + +cp $TESTDIR/common_flags.txt readwrite_flags.txt +echo "--benchmarks=readwhilewriting" >> readwrite_flags.txt +echo "--threads=4" >> readwrite_flags.txt +echo "--duration=$DURATION" >> readwrite_flags.txt +echo "--disable_wal=1" >> readwrite_flags.txt +echo "--use_existing_db=1" >> readwrite_flags.txt +echo "--num=$NUM_KEYS" >> readwrite_flags.txt + +cp $TESTDIR/common_flags.txt writesync_flags.txt +echo "--benchmarks=overwrite" >> writesync_flags.txt +echo "--threads=1" >> writesync_flags.txt +echo "--duration=$DURATION" >> writesync_flags.txt +echo "--disable_wal=0" >> writesync_flags.txt +echo "--use_existing_db=1" >> writesync_flags.txt +echo "--sync=1" >> writesync_flags.txt +echo "--num=$NUM_KEYS" >> writesync_flags.txt + +run_step() { + if [ -z "$1" ] + then + echo run_step called with no parameter + exit 1 + fi + + if [ -z "$NO_SPDK" ] + then + echo "--spdk=$ROCKSDB_CONF" >> "$1"_flags.txt + echo "--spdk_bdev=Nvme0n1" >> "$1"_flags.txt + echo "--spdk_cache_size=$CACHE_SIZE" >> "$1"_flags.txt + fi + + if [ "$NO_SPDK" = "1" ] + then + echo "--bytes_per_sync=262144" >> "$1"_flags.txt + cat /sys/block/nvme0n1/stat > "$1"_blockdev_stats.txt + fi + + echo -n Start $1 test phase... + if [ "$USE_PERF" = "1" ] + then + sudo /usr/bin/time taskset 0xFF perf record $DB_BENCH --flagfile="$1"_flags.txt &> "$1"_db_bench.txt + else + sudo /usr/bin/time taskset 0xFF $DB_BENCH --flagfile="$1"_flags.txt &> "$1"_db_bench.txt + fi + echo done. + + if [ "$NO_SPDK" = "1" ] + then + drop_caches + cat /sys/block/nvme0n1/stat >> "$1"_blockdev_stats.txt + fi + + if [ "$USE_PERF" = "1" ] + then + echo -n Generating perf report for $1 test phase... + sudo perf report -f -n | sed '/#/d' | sed '/%/!d' | sort -r > $1.perf.txt + sudo rm perf.data + $TESTDIR/postprocess.py `pwd` $1 > $1_summary.txt + echo done. + fi +} + +drop_caches() { + echo -n Cleaning Page Cache... + echo 3 > /proc/sys/vm/drop_caches + echo done. +} + +if [ -z "$SKIP_INSERT" ] +then + run_step insert +fi +if [ -z "$SKIP_OVERWRITE" ] +then + run_step overwrite +fi +if [ -z "$SKIP_READWRITE" ] +then + run_step readwrite +fi +if [ -z "$SKIP_WRITESYNC" ] +then + run_step writesync +fi +if [ -z "$SKIP_RANDREAD" ] +then + run_step randread +fi + +if [ "$NO_SPDK" = "1" ] +then + echo -n Unmounting XFS filesystem... + sudo umount /mnt/rocksdb || true &> /dev/null + echo done. +fi diff --git a/src/spdk/test/blobfs/test_plan.md b/src/spdk/test/blobfs/test_plan.md new file mode 100644 index 00000000..080427b2 --- /dev/null +++ b/src/spdk/test/blobfs/test_plan.md @@ -0,0 +1,67 @@ +# SPDK BlobFS Test Plan + +## Current Tests + +# Unit tests (asynchronous API) + +- Tests BlobFS w/ Blobstore with no dependencies on SPDK bdev layer or event framework. + Uses simple DRAM buffer to simulate a block device - all block operations are immediately + completed so no special event handling is required. +- Current tests include: + - basic fs initialization and unload + - open non-existent file fails if SPDK_BLOBFS_OPEN_CREATE not specified + - open non-existent file creates the file if SPDK_BLOBFS_OPEN_CREATE is specified + - close a file fails if there are still open references + - closing a file with no open references fails + - files can be truncated up and down in length + - three-way rename + - operations for inserting and traversing buffers in a cache tree + - allocating and freeing I/O channels + +# Unit tests (synchronous API) + +- Tests BlobFS w/ Blobstore with no dependencies on SPDK bdev layer or event framework. + The synchronous API requires a separate thread to handle any asynchronous handoffs such as + I/O to disk. + - basic read/write I/O operations + - appending to a file whose cache has been flushed and evicted + +# RocksDB + +- Tests BlobFS as the backing store for a RocksDB database. BlobFS uses the SPDK NVMe driver + through the SPDK bdev layer as its block device. Uses RocksDB db_bench utility to drive + the workloads. Each workload (after the initial sequential insert) reloads the database + which validates metadata operations completed correctly in the previous run via the + RocksDB MANIFEST file. RocksDB also runs checksums on key/value blocks read from disk, + verifying data integrity. + - initialize BlobFS filesystem on NVMe SSD + - bulk sequential insert of up to 500M keys (16B key, 1000B value) + - overwrite test - randomly overwrite one of the keys in the database (driving both + flush and compaction traffic) + - readwrite test - one thread randomly overwrites a key in the database, up to 16 + threads randomly read a key in the database. + - writesync - same as overwrite, but enables a WAL (write-ahead log) + - randread - up to 16 threads randomly read a key in the database + +## Future tests to add + +# Unit tests + +- Corrupt data in DRAM buffer, and confirm subsequent operations such as BlobFS load or + opening a blob fail as expected (no panics, etc.) +- Test synchronous API with multiple synchronous threads. May be implemented separately + from existing synchronous unit tests to allow for more sophisticated thread + synchronization. +- Add tests for out of capacity (no more space on disk for additional blobs/files) +- Pending addition of BlobFS superblob, verify that BlobFS load fails with missing or + corrupt superblob +- Additional tests to reach 100% unit test coverage + +# System/integration tests + +- Use fio with BlobFS fuse module for more focused data integrity testing on individual + files. +- Pending directory support (via an SPDK btree module), use BlobFS fuse module to do + things like a Linux kernel compilation. Performance may be poor but this will heavily + stress the mechanics of BlobFS. +- Run RocksDB tests with varying amounts of BlobFS cache |