1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
|
#!/usr/bin/env bash
set -ex
rbd create --size 256 img
IMAGE_SIZE=$(rbd info --format=json img | python3 -c 'import sys, json; print(json.load(sys.stdin)["size"])')
OBJECT_SIZE=$(rbd info --format=json img | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_size"])')
NUM_OBJECTS=$((IMAGE_SIZE / OBJECT_SIZE))
[[ $((IMAGE_SIZE % OBJECT_SIZE)) -eq 0 ]]
OP_SIZE=16384
DEV=$(sudo rbd map img)
{
for ((i = 0; i < $NUM_OBJECTS; i++)); do
echo pwrite -b $OP_SIZE -S $i $((i * OBJECT_SIZE)) $OP_SIZE
done
echo fsync
echo quit
} | xfs_io $DEV
sudo rbd unmap $DEV
g++ -xc++ -o racereads - -lpthread <<EOF
#include <assert.h>
#include <fcntl.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <thread>
#include <vector>
const int object_size = $OBJECT_SIZE;
const int num_objects = $NUM_OBJECTS;
const int read_len = $OP_SIZE;
const int num_reads = 1024;
int main() {
int fd = open("$DEV", O_DIRECT | O_RDONLY);
assert(fd >= 0);
void *buf;
int r = posix_memalign(&buf, 512, read_len);
assert(r == 0);
std::vector<std::thread> threads;
for (int i = 0; i < num_objects; i++) {
threads.emplace_back(
[fd, buf, read_off = static_cast<off_t>(i) * object_size]() {
for (int i = 0; i < num_reads; i++) {
auto len = pread(fd, buf, read_len, read_off);
assert(len == read_len);
}
});
}
for (auto &t : threads) {
t.join();
}
}
EOF
DEV=$(sudo rbd map -o ms_mode=legacy img)
sudo dmesg -C
./racereads
[[ $(dmesg | grep -c 'libceph: osd.* bad crc/signature') -gt 100 ]]
sudo rbd unmap $DEV
DEV=$(sudo rbd map -o ms_mode=legacy,rxbounce img)
sudo dmesg -C
./racereads
[[ $(dmesg | grep -c 'libceph: osd.* bad crc/signature') -eq 0 ]]
sudo rbd unmap $DEV
DEV=$(sudo rbd map -o ms_mode=crc img)
sudo dmesg -C
./racereads
[[ $(dmesg | grep -c 'libceph: osd.* integrity error') -gt 100 ]]
sudo rbd unmap $DEV
DEV=$(sudo rbd map -o ms_mode=crc,rxbounce img)
sudo dmesg -C
./racereads
[[ $(dmesg | grep -c 'libceph: osd.* integrity error') -eq 0 ]]
sudo rbd unmap $DEV
# rxbounce is a no-op for secure mode
DEV=$(sudo rbd map -o ms_mode=secure img)
sudo dmesg -C
./racereads
[[ $(dmesg | grep -c 'libceph: osd.* integrity error') -eq 0 ]]
sudo rbd unmap $DEV
DEV=$(sudo rbd map -o ms_mode=secure,rxbounce img)
sudo dmesg -C
./racereads
[[ $(dmesg | grep -c 'libceph: osd.* integrity error') -eq 0 ]]
sudo rbd unmap $DEV
rbd rm img
echo OK
|
