blob: e8fc8348c60155ec82846457fb21385a50896157 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
|
#!/usr/bin/env bash
set -ex
export RBD_FORCE_ALLOW_V1=1
function fill_image() {
local spec=$1
local dev
dev=$(sudo rbd map $spec)
xfs_io -c "pwrite -b $OBJECT_SIZE -S 0x78 -W 0 $IMAGE_SIZE" $dev
sudo rbd unmap $dev
}
function create_clones() {
local spec=$1
rbd snap create $spec@snap
rbd snap protect $spec@snap
local pool=${spec%/*} # pool/image is assumed
local image=${spec#*/}
local child_pool
for child_pool in $pool clonesonly; do
rbd clone $spec@snap $child_pool/$pool-$image-clone1
rbd clone $spec@snap --data-pool repdata $child_pool/$pool-$image-clone2
rbd clone $spec@snap --data-pool ecdata $child_pool/$pool-$image-clone3
done
}
function trigger_copyup() {
local spec=$1
local dev
dev=$(sudo rbd map $spec)
local i
{
for ((i = 0; i < $NUM_OBJECTS; i++)); do
echo pwrite -b $OBJECT_SIZE -S 0x59 $((i * OBJECT_SIZE + OBJECT_SIZE / 2)) $((OBJECT_SIZE / 2))
done
echo fsync
echo quit
} | xfs_io $dev
sudo rbd unmap $dev
}
function compare() {
local spec=$1
local object=$2
local dev
dev=$(sudo rbd map $spec)
local i
for ((i = 0; i < $NUM_OBJECTS; i++)); do
dd if=$dev bs=$OBJECT_SIZE count=1 skip=$i | cmp $object -
done
sudo rbd unmap $dev
}
function mkfs_and_mount() {
local spec=$1
local dev
dev=$(sudo rbd map $spec)
blkdiscard $dev
mkfs.ext4 -q -E nodiscard $dev
sudo mount $dev /mnt
sudo umount /mnt
sudo rbd unmap $dev
}
function list_HEADs() {
local pool=$1
rados -p $pool ls | while read obj; do
if rados -p $pool stat $obj >/dev/null 2>&1; then
echo $obj
fi
done
}
function count_data_objects() {
local spec=$1
local pool
pool=$(rbd info $spec | grep 'data_pool: ' | awk '{ print $NF }')
if [[ -z $pool ]]; then
pool=${spec%/*} # pool/image is assumed
fi
local prefix
prefix=$(rbd info $spec | grep 'block_name_prefix: ' | awk '{ print $NF }')
rados -p $pool ls | grep -c $prefix
}
function get_num_clones() {
local pool=$1
rados -p $pool --format=json df |
python -c 'import sys, json; print json.load(sys.stdin)["pools"][0]["num_object_clones"]'
}
ceph osd pool create repdata 24 24
rbd pool init repdata
ceph osd erasure-code-profile set teuthologyprofile crush-failure-domain=osd m=1 k=2
ceph osd pool create ecdata 24 24 erasure teuthologyprofile
rbd pool init ecdata
ceph osd pool set ecdata allow_ec_overwrites true
ceph osd pool create rbdnonzero 24 24
rbd pool init rbdnonzero
ceph osd pool create clonesonly 24 24
rbd pool init clonesonly
for pool in rbd rbdnonzero; do
rbd create --size 200 --image-format 1 $pool/img0
rbd create --size 200 $pool/img1
rbd create --size 200 --data-pool repdata $pool/img2
rbd create --size 200 --data-pool ecdata $pool/img3
done
IMAGE_SIZE=$(rbd info --format=json img1 | python -c 'import sys, json; print json.load(sys.stdin)["size"]')
OBJECT_SIZE=$(rbd info --format=json img1 | python -c 'import sys, json; print json.load(sys.stdin)["object_size"]')
NUM_OBJECTS=$((IMAGE_SIZE / OBJECT_SIZE))
[[ $((IMAGE_SIZE % OBJECT_SIZE)) -eq 0 ]]
OBJECT_X=$(mktemp) # xxxx
xfs_io -c "pwrite -b $OBJECT_SIZE -S 0x78 0 $OBJECT_SIZE" $OBJECT_X
OBJECT_XY=$(mktemp) # xxYY
xfs_io -c "pwrite -b $OBJECT_SIZE -S 0x78 0 $((OBJECT_SIZE / 2))" \
-c "pwrite -b $OBJECT_SIZE -S 0x59 $((OBJECT_SIZE / 2)) $((OBJECT_SIZE / 2))" \
$OBJECT_XY
for pool in rbd rbdnonzero; do
for i in {0..3}; do
fill_image $pool/img$i
if [[ $i -ne 0 ]]; then
create_clones $pool/img$i
for child_pool in $pool clonesonly; do
for j in {1..3}; do
trigger_copyup $child_pool/$pool-img$i-clone$j
done
done
fi
done
done
# rbd_directory, rbd_children, rbd_info + img0 header + ...
NUM_META_RBDS=$((3 + 1 + 3 * (1*2 + 3*2)))
# rbd_directory, rbd_children, rbd_info + ...
NUM_META_CLONESONLY=$((3 + 2 * 3 * (3*2)))
[[ $(rados -p rbd ls | wc -l) -eq $((NUM_META_RBDS + 5 * NUM_OBJECTS)) ]]
[[ $(rados -p repdata ls | wc -l) -eq $((1 + 14 * NUM_OBJECTS)) ]]
[[ $(rados -p ecdata ls | wc -l) -eq $((1 + 14 * NUM_OBJECTS)) ]]
[[ $(rados -p rbdnonzero ls | wc -l) -eq $((NUM_META_RBDS + 5 * NUM_OBJECTS)) ]]
[[ $(rados -p clonesonly ls | wc -l) -eq $((NUM_META_CLONESONLY + 6 * NUM_OBJECTS)) ]]
for pool in rbd rbdnonzero; do
for i in {0..3}; do
[[ $(count_data_objects $pool/img$i) -eq $NUM_OBJECTS ]]
if [[ $i -ne 0 ]]; then
for child_pool in $pool clonesonly; do
for j in {1..3}; do
[[ $(count_data_objects $child_pool/$pool-img$i-clone$j) -eq $NUM_OBJECTS ]]
done
done
fi
done
done
[[ $(get_num_clones rbd) -eq 0 ]]
[[ $(get_num_clones repdata) -eq 0 ]]
[[ $(get_num_clones ecdata) -eq 0 ]]
[[ $(get_num_clones rbdnonzero) -eq 0 ]]
[[ $(get_num_clones clonesonly) -eq 0 ]]
for pool in rbd rbdnonzero; do
for i in {0..3}; do
compare $pool/img$i $OBJECT_X
mkfs_and_mount $pool/img$i
if [[ $i -ne 0 ]]; then
for child_pool in $pool clonesonly; do
for j in {1..3}; do
compare $child_pool/$pool-img$i-clone$j $OBJECT_XY
done
done
fi
done
done
# mkfs_and_mount should discard some objects everywhere but in clonesonly
[[ $(list_HEADs rbd | wc -l) -lt $((NUM_META_RBDS + 5 * NUM_OBJECTS)) ]]
[[ $(list_HEADs repdata | wc -l) -lt $((1 + 14 * NUM_OBJECTS)) ]]
[[ $(list_HEADs ecdata | wc -l) -lt $((1 + 14 * NUM_OBJECTS)) ]]
[[ $(list_HEADs rbdnonzero | wc -l) -lt $((NUM_META_RBDS + 5 * NUM_OBJECTS)) ]]
[[ $(list_HEADs clonesonly | wc -l) -eq $((NUM_META_CLONESONLY + 6 * NUM_OBJECTS)) ]]
[[ $(get_num_clones rbd) -eq $NUM_OBJECTS ]]
[[ $(get_num_clones repdata) -eq $((2 * NUM_OBJECTS)) ]]
[[ $(get_num_clones ecdata) -eq $((2 * NUM_OBJECTS)) ]]
[[ $(get_num_clones rbdnonzero) -eq $NUM_OBJECTS ]]
[[ $(get_num_clones clonesonly) -eq 0 ]]
echo OK
|
