1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
|
// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab
#include "include/compat.h"
#include "include/types.h"
#include "include/rados/librados.h"
#include "include/rados/librados.hpp"
#include "include/radosstriper/libradosstriper.h"
#include "include/radosstriper/libradosstriper.hpp"
#include "include/ceph_fs.h"
#include "test/librados/test.h"
#include "test/libradosstriper/TestCase.h"
#include <string>
#include <errno.h>
using namespace librados;
using namespace libradosstriper;
class StriperTestRT : public StriperTestParam {
public:
StriperTestRT() : StriperTestParam() {}
protected:
char* getObjName(const std::string& soid, uint64_t nb)
{
char name[soid.size()+18];
sprintf(name, "%s.%016llx", soid.c_str(), (long long unsigned int)nb);
return strdup(name);
}
void checkObjectFromRados(const std::string& soid, bufferlist &bl,
uint64_t exp_stripe_unit, uint64_t exp_stripe_count,
uint64_t exp_object_size, size_t size)
{
checkObjectFromRados(soid, bl, exp_stripe_unit, exp_stripe_count, exp_object_size, size, size);
}
void checkObjectFromRados(const std::string& soid, bufferlist &bl,
uint64_t exp_stripe_unit, uint64_t exp_stripe_count,
uint64_t exp_object_size, size_t size,
size_t actual_size_if_sparse)
{
// checking first object's rados xattrs
bufferlist xattrbl;
char* firstOid = getObjName(soid, 0);
ASSERT_LT(0, ioctx.getxattr(firstOid, "striper.layout.stripe_unit", xattrbl));
std::string s_xattr(xattrbl.c_str(), xattrbl.length()); // adds 0 byte at the end
uint64_t stripe_unit = strtoll(s_xattr.c_str(), NULL, 10);
ASSERT_LT((unsigned)0, stripe_unit);
ASSERT_EQ(stripe_unit, exp_stripe_unit);
xattrbl.clear();
ASSERT_LT(0, ioctx.getxattr(firstOid, "striper.layout.stripe_count", xattrbl));
s_xattr = std::string(xattrbl.c_str(), xattrbl.length()); // adds 0 byte at the end
uint64_t stripe_count = strtoll(s_xattr.c_str(), NULL, 10);
ASSERT_LT(0U, stripe_count);
ASSERT_EQ(stripe_count, exp_stripe_count);
xattrbl.clear();
ASSERT_LT(0, ioctx.getxattr(firstOid, "striper.layout.object_size", xattrbl));
s_xattr = std::string(xattrbl.c_str(), xattrbl.length()); // adds 0 byte at the end
uint64_t object_size = strtoll(s_xattr.c_str(), NULL, 10);
ASSERT_EQ(object_size, exp_object_size);
xattrbl.clear();
ASSERT_LT(0, ioctx.getxattr(firstOid, "striper.size", xattrbl));
s_xattr = std::string(xattrbl.c_str(), xattrbl.length()); // adds 0 byte at the end
uint64_t xa_size = strtoll(s_xattr.c_str(), NULL, 10);
ASSERT_EQ(xa_size, size);
// checking object content from rados point of view
// we will go stripe by stripe, read the content of each of them and
// check with expectations
uint64_t stripe_per_object = object_size / stripe_unit;
uint64_t stripe_per_objectset = stripe_per_object * stripe_count;
uint64_t nb_stripes_in_object = (size+stripe_unit-1)/stripe_unit;
for (uint64_t stripe_nb = 0;
stripe_nb < nb_stripes_in_object;
stripe_nb++) {
// find out where this stripe is stored
uint64_t objectset = stripe_nb / stripe_per_objectset;
uint64_t stripe_in_object_set = stripe_nb % stripe_per_objectset;
uint64_t object_in_set = stripe_in_object_set % stripe_count;
uint64_t stripe_in_object = stripe_in_object_set / stripe_count;
uint64_t object_nb = objectset * stripe_count + object_in_set;
uint64_t start = stripe_in_object * stripe_unit;
uint64_t len = stripe_unit;
if (stripe_nb == nb_stripes_in_object-1 and size % stripe_unit != 0) {
len = size % stripe_unit;
}
// handle case of sparse object (can only be sparse at the end in our tests)
if (actual_size_if_sparse < size and
((actual_size_if_sparse+stripe_unit-1)/stripe_unit)-1 == stripe_nb) {
len = actual_size_if_sparse % stripe_unit;
if (0 == len) len = stripe_unit;
}
bufferlist stripe_data;
// check object content
char* oid = getObjName(soid, object_nb);
int rc = ioctx.read(oid, stripe_data, len, start);
if (actual_size_if_sparse < size and
(actual_size_if_sparse+stripe_unit-1)/stripe_unit <= stripe_nb) {
// sparse object case : the stripe does not exist, but the rados object may
uint64_t object_start = (object_in_set + objectset*stripe_per_objectset) * stripe_unit;
if (actual_size_if_sparse <= object_start) {
ASSERT_EQ(rc, -ENOENT);
} else {
ASSERT_EQ(rc, 0);
}
} else {
ASSERT_EQ((uint64_t)rc, len);
bufferlist original_data;
original_data.substr_of(bl, stripe_nb*stripe_unit, len);
ASSERT_EQ(0, memcmp(original_data.c_str(), stripe_data.c_str(), len));
}
free(oid);
}
// checking rados object sizes; we go object by object
uint64_t nb_full_object_sets = nb_stripes_in_object / stripe_per_objectset;
uint64_t nb_extra_objects = nb_stripes_in_object % stripe_per_objectset;
if (nb_extra_objects > stripe_count) nb_extra_objects = stripe_count;
uint64_t nb_objects = nb_full_object_sets * stripe_count + nb_extra_objects;
for (uint64_t object_nb = 0; object_nb < nb_objects; object_nb++) {
uint64_t rados_size;
time_t mtime;
char* oid = getObjName(soid, object_nb);
uint64_t nb_full_object_set = object_nb / stripe_count;
uint64_t object_index_in_set = object_nb % stripe_count;
uint64_t object_start_stripe = nb_full_object_set * stripe_per_objectset + object_index_in_set;
uint64_t object_start_off = object_start_stripe * stripe_unit;
if (actual_size_if_sparse < size and actual_size_if_sparse <= object_start_off) {
ASSERT_EQ(-ENOENT, ioctx.stat(oid, &rados_size, &mtime));
} else {
ASSERT_EQ(0, ioctx.stat(oid, &rados_size, &mtime));
uint64_t offset;
uint64_t stripe_size = stripe_count * stripe_unit;
uint64_t set_size = stripe_count * object_size;
uint64_t len = 0;
for (offset = object_start_off;
(offset < (object_start_off) + set_size) && (offset < actual_size_if_sparse);
offset += stripe_size) {
if (offset + stripe_unit > actual_size_if_sparse) {
len += actual_size_if_sparse-offset;
} else {
len += stripe_unit;
}
}
ASSERT_EQ(len, rados_size);
}
free(oid);
}
// check we do not have an extra object behind
uint64_t rados_size;
time_t mtime;
char* oid = getObjName(soid, nb_objects);
ASSERT_EQ(-ENOENT, ioctx.stat(oid, &rados_size, &mtime));
free(oid);
free(firstOid);
}
};
TEST_P(StriperTestRT, StripedRoundtrip) {
// get striping parameters and apply them
TestData testData = GetParam();
ASSERT_EQ(0, striper.set_object_layout_stripe_unit(testData.stripe_unit));
ASSERT_EQ(0, striper.set_object_layout_stripe_count(testData.stripe_count));
ASSERT_EQ(0, striper.set_object_layout_object_size(testData.object_size));
std::ostringstream oss;
oss << "StripedRoundtrip_" << testData.stripe_unit << "_"
<< testData.stripe_count << "_" << testData.object_size
<< "_" << testData.size;
std::string soid = oss.str();
// writing striped data
std::unique_ptr<char[]> buf1;
bufferlist bl1;
{
SCOPED_TRACE("Writing initial object");
buf1 = std::make_unique<char[]>(testData.size);
for (unsigned int i = 0; i < testData.size; i++) buf1[i] = 13*((unsigned char)i);
bl1.append(buf1.get(), testData.size);
ASSERT_EQ(0, striper.write(soid, bl1, testData.size, 0));
// checking object state from Rados point of view
ASSERT_NO_FATAL_FAILURE(checkObjectFromRados(soid, bl1, testData.stripe_unit,
testData.stripe_count, testData.object_size,
testData.size));
}
// adding more data to object and checking again
std::unique_ptr<char[]> buf2;
bufferlist bl2;
{
SCOPED_TRACE("Testing append");
buf2 = std::make_unique<char[]>(testData.size);
for (unsigned int i = 0; i < testData.size; i++) buf2[i] = 17*((unsigned char)i);
bl2.append(buf2.get(), testData.size);
ASSERT_EQ(0, striper.append(soid, bl2, testData.size));
bl1.append(buf2.get(), testData.size);
ASSERT_NO_FATAL_FAILURE(checkObjectFromRados(soid, bl1, testData.stripe_unit,
testData.stripe_count, testData.object_size,
testData.size*2));
}
// truncating to half original size and checking again
{
SCOPED_TRACE("Testing trunc to truncate object");
ASSERT_EQ(0, striper.trunc(soid, testData.size/2));
ASSERT_NO_FATAL_FAILURE(checkObjectFromRados(soid, bl1, testData.stripe_unit,
testData.stripe_count, testData.object_size,
testData.size/2));
}
// truncating back to original size and checking again (especially for 0s)
{
SCOPED_TRACE("Testing trunc to extend object with 0s");
ASSERT_EQ(0, striper.trunc(soid, testData.size));
bufferlist bl3;
bl3.substr_of(bl1, 0, testData.size/2);
bl3.append_zero(testData.size - testData.size/2);
ASSERT_NO_FATAL_FAILURE(checkObjectFromRados(soid, bl3, testData.stripe_unit,
testData.stripe_count, testData.object_size,
testData.size, testData.size/2));
}
{
SCOPED_TRACE("Testing write_full");
// using write_full and checking again
ASSERT_EQ(0, striper.write_full(soid, bl2));
checkObjectFromRados(soid, bl2, testData.stripe_unit,
testData.stripe_count, testData.object_size,
testData.size);
}
{
SCOPED_TRACE("Testing standard remove");
// call remove
ASSERT_EQ(0, striper.remove(soid));
// check that the removal was successful
uint64_t size;
time_t mtime;
for (uint64_t object_nb = 0;
object_nb < testData.size*2/testData.object_size + testData.stripe_count;
object_nb++) {
char* oid = getObjName(soid, object_nb);
ASSERT_EQ(-ENOENT, ioctx.stat(oid, &size, &mtime));
free(oid);
}
}
{
SCOPED_TRACE("Testing remove when no object size");
// recreate object
ASSERT_EQ(0, striper.write(soid, bl1, testData.size*2, 0));
// remove the object size attribute from the striped object
char* firstOid = getObjName(soid, 0);
ASSERT_EQ(0, ioctx.rmxattr(firstOid, "striper.size"));
free(firstOid);
// check that stat fails
uint64_t size;
time_t mtime;
ASSERT_EQ(-ENODATA, striper.stat(soid, &size, &mtime));
// call remove
ASSERT_EQ(0, striper.remove(soid));
// check that the removal was successful
for (uint64_t object_nb = 0;
object_nb < testData.size*2/testData.object_size + testData.stripe_count;
object_nb++) {
char* oid = getObjName(soid, object_nb);
ASSERT_EQ(-ENOENT, ioctx.stat(oid, &size, &mtime));
free(oid);
}
}
}
const TestData simple_stripe_schemes[] = {
// stripe_unit, stripe_count, object_size, size
{CEPH_MIN_STRIPE_UNIT, 5, CEPH_MIN_STRIPE_UNIT, 2},
{CEPH_MIN_STRIPE_UNIT, 5, CEPH_MIN_STRIPE_UNIT, CEPH_MIN_STRIPE_UNIT},
{CEPH_MIN_STRIPE_UNIT, 5, CEPH_MIN_STRIPE_UNIT, CEPH_MIN_STRIPE_UNIT-1},
{CEPH_MIN_STRIPE_UNIT, 5, CEPH_MIN_STRIPE_UNIT, 2*CEPH_MIN_STRIPE_UNIT},
{CEPH_MIN_STRIPE_UNIT, 5, CEPH_MIN_STRIPE_UNIT, 12*CEPH_MIN_STRIPE_UNIT},
{CEPH_MIN_STRIPE_UNIT, 5, CEPH_MIN_STRIPE_UNIT, 2*CEPH_MIN_STRIPE_UNIT-1},
{CEPH_MIN_STRIPE_UNIT, 5, CEPH_MIN_STRIPE_UNIT, 12*CEPH_MIN_STRIPE_UNIT-1},
{CEPH_MIN_STRIPE_UNIT, 5, CEPH_MIN_STRIPE_UNIT, 2*CEPH_MIN_STRIPE_UNIT+100},
{CEPH_MIN_STRIPE_UNIT, 5, CEPH_MIN_STRIPE_UNIT, 12*CEPH_MIN_STRIPE_UNIT+100},
{CEPH_MIN_STRIPE_UNIT, 5, 3*CEPH_MIN_STRIPE_UNIT, 2*CEPH_MIN_STRIPE_UNIT+100},
{CEPH_MIN_STRIPE_UNIT, 5, 3*CEPH_MIN_STRIPE_UNIT, 8*CEPH_MIN_STRIPE_UNIT+100},
{CEPH_MIN_STRIPE_UNIT, 5, 3*CEPH_MIN_STRIPE_UNIT, 12*CEPH_MIN_STRIPE_UNIT+100},
{CEPH_MIN_STRIPE_UNIT, 5, 3*CEPH_MIN_STRIPE_UNIT, 15*CEPH_MIN_STRIPE_UNIT+100},
{CEPH_MIN_STRIPE_UNIT, 5, 3*CEPH_MIN_STRIPE_UNIT, 25*CEPH_MIN_STRIPE_UNIT+100},
{CEPH_MIN_STRIPE_UNIT, 5, 3*CEPH_MIN_STRIPE_UNIT, 45*CEPH_MIN_STRIPE_UNIT+100},
{262144, 5, 262144, 2},
{262144, 5, 262144, 262144},
{262144, 5, 262144, 262144-1},
{262144, 5, 262144, 2*262144},
{262144, 5, 262144, 12*262144},
{262144, 5, 262144, 2*262144-1},
{262144, 5, 262144, 12*262144-1},
{262144, 5, 262144, 2*262144+100},
{262144, 5, 262144, 12*262144+100},
{262144, 5, 3*262144, 2*262144+100},
{262144, 5, 3*262144, 8*262144+100},
{262144, 5, 3*262144, 12*262144+100},
{262144, 5, 3*262144, 15*262144+100},
{262144, 5, 3*262144, 25*262144+100},
{262144, 5, 3*262144, 45*262144+100},
{CEPH_MIN_STRIPE_UNIT, 1, CEPH_MIN_STRIPE_UNIT, 2},
{CEPH_MIN_STRIPE_UNIT, 1, CEPH_MIN_STRIPE_UNIT, CEPH_MIN_STRIPE_UNIT},
{CEPH_MIN_STRIPE_UNIT, 1, CEPH_MIN_STRIPE_UNIT, CEPH_MIN_STRIPE_UNIT-1},
{CEPH_MIN_STRIPE_UNIT, 1, CEPH_MIN_STRIPE_UNIT, 2*CEPH_MIN_STRIPE_UNIT},
{CEPH_MIN_STRIPE_UNIT, 1, CEPH_MIN_STRIPE_UNIT, 12*CEPH_MIN_STRIPE_UNIT},
{CEPH_MIN_STRIPE_UNIT, 1, CEPH_MIN_STRIPE_UNIT, 2*CEPH_MIN_STRIPE_UNIT-1},
{CEPH_MIN_STRIPE_UNIT, 1, CEPH_MIN_STRIPE_UNIT, 12*CEPH_MIN_STRIPE_UNIT-1},
{CEPH_MIN_STRIPE_UNIT, 1, CEPH_MIN_STRIPE_UNIT, 2*CEPH_MIN_STRIPE_UNIT+100},
{CEPH_MIN_STRIPE_UNIT, 1, CEPH_MIN_STRIPE_UNIT, 12*CEPH_MIN_STRIPE_UNIT+100},
{CEPH_MIN_STRIPE_UNIT, 1, 3*CEPH_MIN_STRIPE_UNIT, 2*CEPH_MIN_STRIPE_UNIT+100},
{CEPH_MIN_STRIPE_UNIT, 1, 3*CEPH_MIN_STRIPE_UNIT, 8*CEPH_MIN_STRIPE_UNIT+100},
{CEPH_MIN_STRIPE_UNIT, 1, 3*CEPH_MIN_STRIPE_UNIT, 12*CEPH_MIN_STRIPE_UNIT+100},
{CEPH_MIN_STRIPE_UNIT, 1, 3*CEPH_MIN_STRIPE_UNIT, 15*CEPH_MIN_STRIPE_UNIT+100},
{CEPH_MIN_STRIPE_UNIT, 1, 3*CEPH_MIN_STRIPE_UNIT, 25*CEPH_MIN_STRIPE_UNIT+100},
{CEPH_MIN_STRIPE_UNIT, 1, 3*CEPH_MIN_STRIPE_UNIT, 45*CEPH_MIN_STRIPE_UNIT+100},
{CEPH_MIN_STRIPE_UNIT, 50, CEPH_MIN_STRIPE_UNIT, 2},
{CEPH_MIN_STRIPE_UNIT, 50, CEPH_MIN_STRIPE_UNIT, CEPH_MIN_STRIPE_UNIT},
{CEPH_MIN_STRIPE_UNIT, 50, CEPH_MIN_STRIPE_UNIT, CEPH_MIN_STRIPE_UNIT-1},
{CEPH_MIN_STRIPE_UNIT, 50, CEPH_MIN_STRIPE_UNIT, 2*CEPH_MIN_STRIPE_UNIT},
{CEPH_MIN_STRIPE_UNIT, 50, CEPH_MIN_STRIPE_UNIT, 12*CEPH_MIN_STRIPE_UNIT},
{CEPH_MIN_STRIPE_UNIT, 50, CEPH_MIN_STRIPE_UNIT, 2*CEPH_MIN_STRIPE_UNIT-1},
{CEPH_MIN_STRIPE_UNIT, 50, CEPH_MIN_STRIPE_UNIT, 12*CEPH_MIN_STRIPE_UNIT-1},
{CEPH_MIN_STRIPE_UNIT, 50, CEPH_MIN_STRIPE_UNIT, 2*CEPH_MIN_STRIPE_UNIT+100},
{CEPH_MIN_STRIPE_UNIT, 50, CEPH_MIN_STRIPE_UNIT, 12*CEPH_MIN_STRIPE_UNIT+100},
{CEPH_MIN_STRIPE_UNIT, 50, 3*CEPH_MIN_STRIPE_UNIT, 2*CEPH_MIN_STRIPE_UNIT+100},
{CEPH_MIN_STRIPE_UNIT, 50, 3*CEPH_MIN_STRIPE_UNIT, 8*CEPH_MIN_STRIPE_UNIT+100},
{CEPH_MIN_STRIPE_UNIT, 50, 3*CEPH_MIN_STRIPE_UNIT, 12*CEPH_MIN_STRIPE_UNIT+100},
{CEPH_MIN_STRIPE_UNIT, 50, 3*CEPH_MIN_STRIPE_UNIT, 15*CEPH_MIN_STRIPE_UNIT+100},
{CEPH_MIN_STRIPE_UNIT, 50, 3*CEPH_MIN_STRIPE_UNIT, 25*CEPH_MIN_STRIPE_UNIT+100},
{CEPH_MIN_STRIPE_UNIT, 50, 3*CEPH_MIN_STRIPE_UNIT, 45*CEPH_MIN_STRIPE_UNIT+100}
};
INSTANTIATE_TEST_SUITE_P(SimpleStriping,
StriperTestRT,
::testing::ValuesIn(simple_stripe_schemes));
|
