diff options
Diffstat (limited to 'qa/workunits/direct_io')
| -rw-r--r-- | qa/workunits/direct_io/.gitignore | 3 | ||||
| -rw-r--r-- | qa/workunits/direct_io/Makefile | 11 | ||||
| -rwxr-xr-x | qa/workunits/direct_io/big.sh | 6 | ||||
| -rw-r--r-- | qa/workunits/direct_io/direct_io_test.c | 312 | ||||
| -rwxr-xr-x | qa/workunits/direct_io/misc.sh | 16 | ||||
| -rw-r--r-- | qa/workunits/direct_io/test_short_dio_read.c | 57 | ||||
| -rw-r--r-- | qa/workunits/direct_io/test_sync_io.c | 250 |
7 files changed, 655 insertions, 0 deletions
diff --git a/qa/workunits/direct_io/.gitignore b/qa/workunits/direct_io/.gitignore new file mode 100644 index 000000000..80f1fd1aa --- /dev/null +++ b/qa/workunits/direct_io/.gitignore @@ -0,0 +1,3 @@ +/direct_io_test +/test_sync_io +/test_short_dio_read diff --git a/qa/workunits/direct_io/Makefile b/qa/workunits/direct_io/Makefile new file mode 100644 index 000000000..20fec0be5 --- /dev/null +++ b/qa/workunits/direct_io/Makefile @@ -0,0 +1,11 @@ +CFLAGS = -Wall -Wextra -D_GNU_SOURCE + +TARGETS = direct_io_test test_sync_io test_short_dio_read + +.c: + $(CC) $(CFLAGS) $@.c -o $@ + +all: $(TARGETS) + +clean: + rm $(TARGETS) diff --git a/qa/workunits/direct_io/big.sh b/qa/workunits/direct_io/big.sh new file mode 100755 index 000000000..43bd6d72b --- /dev/null +++ b/qa/workunits/direct_io/big.sh @@ -0,0 +1,6 @@ +#!/bin/sh -ex + +echo "test large (16MB) dio write" +dd if=/dev/zero of=foo.big bs=16M count=1 oflag=direct + +echo OK diff --git a/qa/workunits/direct_io/direct_io_test.c b/qa/workunits/direct_io/direct_io_test.c new file mode 100644 index 000000000..ccfbbb860 --- /dev/null +++ b/qa/workunits/direct_io/direct_io_test.c @@ -0,0 +1,312 @@ +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2011 New Dream Network + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include <errno.h> +#include <inttypes.h> +#include <fcntl.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <time.h> +#include <unistd.h> + +/* + * direct_io_test + * + * This test does some I/O using O_DIRECT. + * + * Semantics of O_DIRECT can be found at http://lwn.net/Articles/348739/ + * + */ + +static int g_num_pages = 100; + +static int g_duration = 10; + +struct chunk { + uint64_t offset; + uint64_t pad0; + uint64_t pad1; + uint64_t pad2; + uint64_t pad3; + uint64_t pad4; + uint64_t pad5; + uint64_t not_offset; +} __attribute__((packed)); + +static int page_size; + +static char temp_file[] = "direct_io_temp_file_XXXXXX"; + +static int safe_write(int fd, const void *buf, signed int len) +{ + const char *b = (const char*)buf; + /* Handle EINTR and short writes */ + while (1) { + int res = write(fd, b, len); + if (res < 0) { + int err = errno; + if (err != EINTR) { + return err; + } + } + len -= res; + b += res; + if (len <= 0) + return 0; + } +} + +static int do_read(int fd, char *buf, int buf_sz) +{ + /* We assume no short reads or EINTR. It's not really clear how + * those things interact with O_DIRECT. */ + int ret = read(fd, buf, buf_sz); + if (ret < 0) { + int err = errno; + printf("do_read: error: %d (%s)\n", err, strerror(err)); + return err; + } + if (ret != buf_sz) { + printf("do_read: short read\n"); + return -EIO; + } + return 0; +} + +static int setup_temp_file(void) +{ + int fd; + int64_t num_chunks, i; + + if (page_size % sizeof(struct chunk)) { + printf("setup_big_file: page_size doesn't divide evenly " + "into data blocks.\n"); + return -EINVAL; + } + + fd = mkstemp(temp_file); + if (fd < 0) { + int err = errno; + printf("setup_big_file: mkostemps failed with error %d\n", err); + return err; + } + + num_chunks = g_num_pages * (page_size / sizeof(struct chunk)); + for (i = 0; i < num_chunks; ++i) { + int ret; + struct chunk c; + memset(&c, 0, sizeof(c)); + c.offset = i * sizeof(struct chunk); + c.pad0 = 0; + c.pad1 = 1; + c.pad2 = 2; + c.pad3 = 3; + c.pad4 = 4; + c.pad5 = 5; + c.not_offset = ~c.offset; + ret = safe_write(fd, &c, sizeof(struct chunk)); + if (ret) { + printf("setup_big_file: safe_write failed with " + "error: %d\n", ret); + TEMP_FAILURE_RETRY(close(fd)); + unlink(temp_file); + return ret; + } + } + TEMP_FAILURE_RETRY(close(fd)); + return 0; +} + +static int verify_chunk(const struct chunk *c, uint64_t offset) +{ + if (c->offset != offset) { + printf("verify_chunk(%" PRId64 "): bad offset value (got: %" + PRId64 ", expected: %" PRId64 "\n", offset, c->offset, offset); + return EIO; + } + if (c->pad0 != 0) { + printf("verify_chunk(%" PRId64 "): bad pad0 value\n", offset); + return EIO; + } + if (c->pad1 != 1) { + printf("verify_chunk(%" PRId64 "): bad pad1 value\n", offset); + return EIO; + } + if (c->pad2 != 2) { + printf("verify_chunk(%" PRId64 "): bad pad2 value\n", offset); + return EIO; + } + if (c->pad3 != 3) { + printf("verify_chunk(%" PRId64 "): bad pad3 value\n", offset); + return EIO; + } + if (c->pad4 != 4) { + printf("verify_chunk(%" PRId64 "): bad pad4 value\n", offset); + return EIO; + } + if (c->pad5 != 5) { + printf("verify_chunk(%" PRId64 "): bad pad5 value\n", offset); + return EIO; + } + if (c->not_offset != ~offset) { + printf("verify_chunk(%" PRId64 "): bad not_offset value\n", + offset); + return EIO; + } + return 0; +} + +static int do_o_direct_reads(void) +{ + int fd, ret; + unsigned int i; + void *buf = 0; + time_t cur_time, end_time; + ret = posix_memalign(&buf, page_size, page_size); + if (ret) { + printf("do_o_direct_reads: posix_memalign returned %d\n", ret); + goto done; + } + + fd = open(temp_file, O_RDONLY | O_DIRECT); + if (fd < 0) { + ret = errno; + printf("do_o_direct_reads: error opening fd: %d\n", ret); + goto free_buf; + } + + // read the first chunk and see if it looks OK + ret = do_read(fd, buf, page_size); + if (ret) + goto close_fd; + ret = verify_chunk((struct chunk*)buf, 0); + if (ret) + goto close_fd; + + // read some random chunks and see how they look + cur_time = time(NULL); + end_time = cur_time + g_duration; + i = 0; + do { + time_t next_time; + uint64_t offset; + int page; + unsigned int seed; + + seed = i++; + page = rand_r(&seed) % g_num_pages; + offset = page; + offset *= page_size; + if (lseek64(fd, offset, SEEK_SET) == -1) { + int err = errno; + printf("lseek64(%" PRId64 ") failed: error %d (%s)\n", + offset, err, strerror(err)); + goto close_fd; + } + ret = do_read(fd, buf, page_size); + if (ret) + goto close_fd; + ret = verify_chunk((struct chunk*)buf, offset); + if (ret) + goto close_fd; + next_time = time(NULL); + if (next_time > cur_time) { + printf("."); + } + cur_time = next_time; + } while (time(NULL) < end_time); + + printf("\ndo_o_direct_reads: SUCCESS\n"); +close_fd: + TEMP_FAILURE_RETRY(close(fd)); +free_buf: + free(buf); +done: + return ret; +} + +static void usage(char *argv0) +{ + printf("%s: tests direct I/O\n", argv0); + printf("-d <seconds>: sets duration to <seconds>\n"); + printf("-h: this help\n"); + printf("-p <pages>: sets number of pages to allocate\n"); +} + +static void parse_args(int argc, char *argv[]) +{ + int c; + while ((c = getopt (argc, argv, "d:hp:")) != -1) { + switch (c) { + case 'd': + g_duration = atoi(optarg); + if (g_duration <= 0) { + printf("tried to set invalid value of " + "g_duration: %d\n", g_num_pages); + exit(1); + } + break; + case 'h': + usage(argv[0]); + exit(0); + break; + case 'p': + g_num_pages = atoi(optarg); + if (g_num_pages <= 0) { + printf("tried to set invalid value of " + "g_num_pages: %d\n", g_num_pages); + exit(1); + } + break; + case '?': + usage(argv[0]); + exit(1); + break; + default: + usage(argv[0]); + exit(1); + break; + } + } +} + +int main(int argc, char *argv[]) +{ + int ret; + + parse_args(argc, argv); + + setvbuf(stdout, NULL, _IONBF, 0); + + page_size = getpagesize(); + + ret = setup_temp_file(); + if (ret) { + printf("setup_temp_file failed with error %d\n", ret); + goto done; + } + + ret = do_o_direct_reads(); + if (ret) { + printf("do_o_direct_reads failed with error %d\n", ret); + goto unlink_temp_file; + } + +unlink_temp_file: + unlink(temp_file); +done: + return ret; +} diff --git a/qa/workunits/direct_io/misc.sh b/qa/workunits/direct_io/misc.sh new file mode 100755 index 000000000..6de080d2d --- /dev/null +++ b/qa/workunits/direct_io/misc.sh @@ -0,0 +1,16 @@ +#!/bin/sh -ex + +# a few test cases from henry +echo "test read from hole" +dd if=/dev/zero of=dd3 bs=1 seek=1048576 count=0 +dd if=dd3 of=/tmp/ddout1 skip=8 bs=512 count=2 iflag=direct +dd if=/dev/zero of=/tmp/dd3 bs=512 count=2 +cmp /tmp/dd3 /tmp/ddout1 + +echo "other thing" +dd if=/dev/urandom of=/tmp/dd10 bs=500 count=1 +dd if=/tmp/dd10 of=dd10 bs=512 seek=8388 count=1 +dd if=dd10 of=/tmp/dd10out bs=512 skip=8388 count=1 iflag=direct +cmp /tmp/dd10 /tmp/dd10out + +echo OK diff --git a/qa/workunits/direct_io/test_short_dio_read.c b/qa/workunits/direct_io/test_short_dio_read.c new file mode 100644 index 000000000..502485557 --- /dev/null +++ b/qa/workunits/direct_io/test_short_dio_read.c @@ -0,0 +1,57 @@ +#include <unistd.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <stdio.h> +#include <errno.h> +#include <string.h> +#include <stdlib.h> + +int main() +{ + char buf[409600]; + ssize_t r; + int err; + int fd = open("shortfile", O_WRONLY|O_CREAT, 0644); + + if (fd < 0) { + err = errno; + printf("error: open() failed with: %d (%s)\n", err, strerror(err)); + exit(err); + } + + printf("writing first 3 bytes of 10k file\n"); + r = write(fd, "foo", 3); + if (r == -1) { + err = errno; + printf("error: write() failed with: %d (%s)\n", err, strerror(err)); + close(fd); + exit(err); + } + r = ftruncate(fd, 10000); + if (r == -1) { + err = errno; + printf("error: ftruncate() failed with: %d (%s)\n", err, strerror(err)); + close(fd); + exit(err); + } + + fsync(fd); + close(fd); + + printf("reading O_DIRECT\n"); + fd = open("shortfile", O_RDONLY|O_DIRECT); + if (fd < 0) { + err = errno; + printf("error: open() failed with: %d (%s)\n", err, strerror(err)); + exit(err); + } + + r = read(fd, buf, sizeof(buf)); + close(fd); + + printf("got %d\n", (int)r); + if (r != 10000) + return 1; + return 0; +} diff --git a/qa/workunits/direct_io/test_sync_io.c b/qa/workunits/direct_io/test_sync_io.c new file mode 100644 index 000000000..f393fa6e8 --- /dev/null +++ b/qa/workunits/direct_io/test_sync_io.c @@ -0,0 +1,250 @@ +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <unistd.h> +#include <stdio.h> +#include <inttypes.h> +#include <linux/types.h> +#include <stdlib.h> +#include <string.h> +#include <sys/ioctl.h> +#include <errno.h> + +//#include "../client/ioctl.h" + +#include <linux/ioctl.h> +#define CEPH_IOCTL_MAGIC 0x97 +#define CEPH_IOC_SYNCIO _IO(CEPH_IOCTL_MAGIC, 5) + +void write_pattern() +{ + printf("writing pattern\n"); + + uint64_t i; + int r; + + int fd = open("foo", O_CREAT|O_WRONLY, 0644); + if (fd < 0) { + r = errno; + printf("write_pattern: error: open() failed with: %d (%s)\n", r, strerror(r)); + exit(r); + } + for (i=0; i<1048576 * sizeof(i); i += sizeof(i)) { + r = write(fd, &i, sizeof(i)); + if (r == -1) { + r = errno; + printf("write_pattern: error: write() failed with: %d (%s)\n", r, strerror(r)); + break; + } + } + + close(fd); +} + +int verify_pattern(char *buf, size_t len, uint64_t off) +{ + size_t i; + + for (i = 0; i < len; i += sizeof(uint64_t)) { + uint64_t expected = i + off; + uint64_t actual = *(uint64_t*)(buf + i); + if (expected != actual) { + printf("error: offset %llu had %llu\n", (unsigned long long)expected, + (unsigned long long)actual); + exit(1); + } + } + return 0; +} + +void generate_pattern(void *buf, size_t len, uint64_t offset) +{ + uint64_t *v = buf; + size_t i; + + for (i=0; i<len / sizeof(v); i++) + v[i] = i * sizeof(v) + offset; + verify_pattern(buf, len, offset); +} + +int read_file(int buf_align, uint64_t offset, int len, int direct) { + + printf("read_file buf_align %d offset %llu len %d\n", buf_align, + (unsigned long long)offset, len); + void *rawbuf; + int r; + int flags; + int err = 0; + + if(direct) + flags = O_RDONLY|O_DIRECT; + else + flags = O_RDONLY; + + int fd = open("foo", flags); + if (fd < 0) { + err = errno; + printf("read_file: error: open() failed with: %d (%s)\n", err, strerror(err)); + exit(err); + } + + if (!direct) + ioctl(fd, CEPH_IOC_SYNCIO); + + if ((r = posix_memalign(&rawbuf, 4096, len + buf_align)) != 0) { + printf("read_file: error: posix_memalign failed with %d", r); + close(fd); + exit (r); + } + + void *buf = (char *)rawbuf + buf_align; + memset(buf, 0, len); + r = pread(fd, buf, len, offset); + if (r == -1) { + err = errno; + printf("read_file: error: pread() failed with: %d (%s)\n", err, strerror(err)); + goto out; + } + r = verify_pattern(buf, len, offset); + +out: + close(fd); + free(rawbuf); + return r; +} + +int read_direct(int buf_align, uint64_t offset, int len) +{ + printf("read_direct buf_align %d offset %llu len %d\n", buf_align, + (unsigned long long)offset, len); + return read_file(buf_align, offset, len, 1); +} + +int read_sync(int buf_align, uint64_t offset, int len) +{ + printf("read_sync buf_align %d offset %llu len %d\n", buf_align, + (unsigned long long)offset, len); + return read_file(buf_align, offset, len, 0); +} + +int write_file(int buf_align, uint64_t offset, int len, int direct) +{ + printf("write_file buf_align %d offset %llu len %d\n", buf_align, + (unsigned long long)offset, len); + void *rawbuf; + int r; + int err = 0; + int flags; + if (direct) + flags = O_WRONLY|O_DIRECT|O_CREAT; + else + flags = O_WRONLY|O_CREAT; + + int fd = open("foo", flags, 0644); + if (fd < 0) { + int err = errno; + printf("write_file: error: open() failed with: %d (%s)\n", err, strerror(err)); + exit(err); + } + + if ((r = posix_memalign(&rawbuf, 4096, len + buf_align)) != 0) { + printf("write_file: error: posix_memalign failed with %d", r); + err = r; + goto out_close; + } + + if (!direct) + ioctl(fd, CEPH_IOC_SYNCIO); + + void *buf = (char *)rawbuf + buf_align; + + generate_pattern(buf, len, offset); + + r = pwrite(fd, buf, len, offset); + close(fd); + + fd = open("foo", O_RDONLY); + if (fd < 0) { + err = errno; + printf("write_file: error: open() failed with: %d (%s)\n", err, strerror(err)); + free(rawbuf); + goto out_unlink; + } + void *buf2 = malloc(len); + if (!buf2) { + err = -ENOMEM; + printf("write_file: error: malloc failed\n"); + goto out_free; + } + + memset(buf2, 0, len); + r = pread(fd, buf2, len, offset); + if (r == -1) { + err = errno; + printf("write_file: error: pread() failed with: %d (%s)\n", err, strerror(err)); + goto out_free_buf; + } + r = verify_pattern(buf2, len, offset); + +out_free_buf: + free(buf2); +out_free: + free(rawbuf); +out_close: + close(fd); +out_unlink: + unlink("foo"); + if (err) + exit(err); + return r; +} + +int write_direct(int buf_align, uint64_t offset, int len) +{ + printf("write_direct buf_align %d offset %llu len %d\n", buf_align, + (unsigned long long)offset, len); + return write_file (buf_align, offset, len, 1); +} + +int write_sync(int buf_align, uint64_t offset, int len) +{ + printf("write_sync buf_align %d offset %llu len %d\n", buf_align, + (unsigned long long)offset, len); + return write_file (buf_align, offset, len, 0); +} + +int main(int argc, char **argv) +{ + uint64_t i, j, k; + int read = 1; + int write = 1; + + if (argc >= 2 && strcmp(argv[1], "read") == 0) + write = 0; + if (argc >= 2 && strcmp(argv[1], "write") == 0) + read = 0; + + if (read) { + write_pattern(); + + for (i = 0; i < 4096; i += 512) + for (j = 4*1024*1024 - 4096; j < 4*1024*1024 + 4096; j += 512) + for (k = 1024; k <= 16384; k *= 2) { + read_direct(i, j, k); + read_sync(i, j, k); + } + + } + unlink("foo"); + if (write) { + for (i = 0; i < 4096; i += 512) + for (j = 4*1024*1024 - 4096 + 512; j < 4*1024*1024 + 4096; j += 512) + for (k = 1024; k <= 16384; k *= 2) { + write_direct(i, j, k); + write_sync(i, j, k); + } + } + + + return 0; +} |