diff options
Diffstat (limited to 'src/test/test_mempool.cc')
-rw-r--r-- | src/test/test_mempool.cc | 460 |
1 files changed, 460 insertions, 0 deletions
diff --git a/src/test/test_mempool.cc b/src/test/test_mempool.cc new file mode 100644 index 000000000..9dd96682f --- /dev/null +++ b/src/test/test_mempool.cc @@ -0,0 +1,460 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph distributed storage system + * + * Copyright (C) 2016 Western Digital Corporation + * + * Author: Allen Samuels <allen.samuels@sandisk.com> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +#include <stdio.h> + +#include "global/global_init.h" +#include "common/ceph_argparse.h" +#include "global/global_context.h" +#include "gtest/gtest.h" +#include "include/btree_map.h" +#include "include/mempool.h" + +using namespace std; + +void check_usage(mempool::pool_index_t ix) +{ + mempool::pool_t *pool = &mempool::get_pool(ix); + mempool::stats_t total; + map<std::string,mempool::stats_t> m; + pool->get_stats(&total, &m); + size_t usage = pool->allocated_bytes(); + size_t sum = 0; + for (auto& p : m) { + sum += p.second.bytes; + } + if (sum != usage) { + ceph::TableFormatter jf; + pool->dump(&jf); + jf.flush(std::cout); + } + EXPECT_EQ(sum, usage); +} + +template<typename A, typename B> +void eq_elements(const A& a, const B& b) +{ + auto lhs = a.begin(); + auto rhs = b.begin(); + while (lhs != a.end()) { + EXPECT_EQ(*lhs,*rhs); + lhs++; + rhs++; + } + EXPECT_EQ(rhs,b.end()); +} + +template<typename A, typename B> +void eq_pairs(const A& a, const B& b) +{ + auto lhs = a.begin(); + auto rhs = b.begin(); + while (lhs != a.end()) { + EXPECT_EQ(lhs->first,rhs->first); + EXPECT_EQ(lhs->second,rhs->second); + lhs++; + rhs++; + } + EXPECT_EQ(rhs,b.end()); +} + +#define MAKE_INSERTER(inserter) \ + template<typename A,typename B> \ +void do_##inserter(A& a, B& b, int count, int base) { \ + for (int i = 0; i < count; ++i) { \ + a.inserter(base + i); \ + b.inserter(base + i); \ + } \ +} + +MAKE_INSERTER(push_back); +MAKE_INSERTER(insert); + +template<typename A,typename B> +void do_insert_key(A& a, B& b, int count, int base) +{ + for (int i = 0; i < count; ++i) { + a.insert(make_pair(base+i,base+i)); + b.insert(make_pair(base+i,base+i)); + check_usage(mempool::osd::id); + } +} + +TEST(mempool, vector_context) +{ + check_usage(mempool::osd::id); + EXPECT_EQ(mempool::osd::allocated_bytes(), 0u); + EXPECT_EQ(mempool::osd::allocated_items(), 0u); + for (unsigned i = 0; i < 10; ++i) { + vector<int> a; + mempool::osd::vector<int> b,c; + eq_elements(a,b); + do_push_back(a,b,i,i); + eq_elements(a,b); + check_usage(mempool::osd::id); + + mempool::stats_t total; + map<std::string,mempool::stats_t> by_type; + mempool::get_pool(mempool::osd::id).get_stats(&total, &by_type); + EXPECT_GE(mempool::osd::allocated_bytes(), i * 4u); + EXPECT_GE(mempool::osd::allocated_items(), i); + + c.swap(b); + eq_elements(a,c); + check_usage(mempool::osd::id); + a.clear(); + b.clear(); + c.clear(); + } +} + +TEST(mempool, list_context) +{ + for (unsigned i = 1; i < 10; ++i) { + list<int> a; + mempool::osd::list<int> b,c; + eq_elements(a,b); + do_push_back(a,b,i,i); + eq_elements(a,b); + c.swap(b); + eq_elements(a,c); + a.erase(a.begin()); + c.erase(c.begin()); + eq_elements(a,c); + a.clear(); + b.clear(); + c.clear(); + do_push_back(a,b,i,i); + c.splice(c.begin(),b,b.begin(),b.end()); + + mempool::stats_t total; + map<std::string,mempool::stats_t> by_type; + mempool::get_pool(mempool::osd::id).get_stats(&total, &by_type); + EXPECT_GE(mempool::osd::allocated_bytes(), i * 4u); + EXPECT_EQ(mempool::osd::allocated_items(), i); + + eq_elements(a,c); + check_usage(mempool::osd::id); + } +} + +TEST(mempool, set_context) +{ + for (int i = 0; i < 10; ++i) { + set<int> a; + mempool::osd::set<int> b; + do_insert(a,b,i,i); + eq_elements(a,b); + check_usage(mempool::osd::id); + } + + for (int i = 1; i < 10; ++i) { + set<int> a; + mempool::osd::set<int> b; + do_insert(a,b,i,0); + EXPECT_NE(a.find(i/2),a.end()); + EXPECT_NE(b.find(i/2),b.end()); + a.erase(a.find(i/2)); + b.erase(b.find(i/2)); + eq_elements(a,b); + check_usage(mempool::osd::id); + } +} + +struct obj { + MEMPOOL_CLASS_HELPERS(); + int a; + int b; + obj() : a(1), b(1) {} + explicit obj(int _a) : a(_a), b(2) {} + obj(int _a,int _b) : a(_a), b(_b) {} + friend inline bool operator<(const obj& l, const obj& r) { + return l.a < r.a; + } +}; +MEMPOOL_DEFINE_OBJECT_FACTORY(obj, obj, osdmap); + +TEST(mempool, test_factory) +{ + obj *o1 = new obj(); + obj *o2 = new obj(10); + obj *o3 = new obj(20,30); + check_usage(mempool::osdmap::id); + EXPECT_NE(o1,nullptr); + EXPECT_EQ(o1->a,1); + EXPECT_EQ(o1->b,1); + EXPECT_EQ(o2->a,10); + EXPECT_EQ(o2->b,2); + EXPECT_EQ(o3->a,20); + EXPECT_EQ(o3->b,30); + + delete o1; + delete o2; + delete o3; + check_usage(mempool::osdmap::id); +} + +TEST(mempool, vector) +{ + { + mempool::osd::vector<int> v; + v.push_back(1); + v.push_back(2); + } + { + mempool::osdmap::vector<obj> v; + v.push_back(obj()); + v.push_back(obj(1)); + } +} + +TEST(mempool, set) +{ + mempool::osd::set<int> set_int; + set_int.insert(1); + set_int.insert(2); + mempool::osdmap::set<obj> set_obj; + set_obj.insert(obj()); + set_obj.insert(obj(1)); + set_obj.insert(obj(1, 2)); +} + +TEST(mempool, map) +{ + { + mempool::osd::map<int,int> v; + v[1] = 2; + v[3] = 4; + } + { + mempool::osdmap::map<int,obj> v; + v[1] = obj(); + v[2] = obj(2); + v[3] = obj(2, 3); + } +} + +TEST(mempool, list) +{ + { + mempool::osd::list<int> v; + v.push_back(1); + v.push_back(2); + } + { + mempool::osdmap::list<obj> v; + v.push_back(obj()); + v.push_back(obj(1)); + } + +} + +TEST(mempool, dump) +{ + ostringstream ostr; + + Formatter* f = Formatter::create("xml-pretty", "xml-pretty", "xml-pretty"); + mempool::dump(f); + f->flush(ostr); + + delete f; + ASSERT_NE(ostr.str().find(mempool::get_pool_name((mempool::pool_index_t)0)), + std::string::npos); + + ostr.str(""); + + f = Formatter::create("html-pretty", "html-pretty", "html-pretty"); + mempool::dump(f); + f->flush(ostr); + + delete f; + ASSERT_NE(ostr.str().find(mempool::get_pool_name((mempool::pool_index_t)0)), + std::string::npos); + + ostr.str(""); + f = Formatter::create("table", "table", "table"); + mempool::dump(f); + f->flush(ostr); + + delete f; + ASSERT_NE(ostr.str().find(mempool::get_pool_name((mempool::pool_index_t)0)), + std::string::npos); + + ostr.str(""); + + f = Formatter::create("json-pretty", "json-pretty", "json-pretty"); + mempool::dump(f); + f->flush(ostr); + delete f; + + ASSERT_NE(ostr.str().find(mempool::get_pool_name((mempool::pool_index_t)0)), + std::string::npos); +} + +TEST(mempool, unordered_map) +{ + mempool::osdmap::unordered_map<int,obj> h; + h[1] = obj(); + h[2] = obj(1); +} + +TEST(mempool, string_test) +{ + mempool::osdmap::string s; + s.reserve(100); + EXPECT_GE(mempool::osdmap::allocated_items(), s.capacity() + 1u); // +1 for zero-byte termination : + for (size_t i = 0; i < 10; ++i) { + s += '1'; + s.append(s); + EXPECT_GE(mempool::osdmap::allocated_items(), s.capacity() + 1u); + } +} + +TEST(mempool, bufferlist) +{ + bufferlist bl; + int len = 1048576; + size_t before = mempool::buffer_anon::allocated_bytes(); + cout << "before " << before << std::endl; + bl.append(buffer::create_aligned(len, 4096)); + size_t after = mempool::buffer_anon::allocated_bytes(); + cout << "after " << after << std::endl; + ASSERT_GE(after, before + len); +} + +TEST(mempool, bufferlist_reassign) +{ + bufferlist bl; + size_t items_before = mempool::buffer_anon::allocated_items(); + size_t bytes_before = mempool::buffer_anon::allocated_bytes(); + bl.append("fooo"); + ASSERT_EQ(items_before + 1, mempool::buffer_anon::allocated_items()); + ASSERT_LT(bytes_before, mempool::buffer_anon::allocated_bytes()); + + // move existing bl + bl.reassign_to_mempool(mempool::mempool_osd); + ASSERT_EQ(items_before, mempool::buffer_anon::allocated_items()); + ASSERT_EQ(bytes_before, mempool::buffer_anon::allocated_bytes()); + + // additional appends should go to the same pool + items_before = mempool::osd::allocated_items(); + bytes_before = mempool::osd::allocated_bytes(); + cout << "anon b " << mempool::buffer_anon::allocated_bytes() << std::endl; + for (unsigned i = 0; i < 1000; ++i) { + bl.append("asdfddddddddddddddddddddddasfdasdfasdfasdfasdfasdf"); + } + cout << "anon a " << mempool::buffer_anon::allocated_bytes() << std::endl; + ASSERT_LT(items_before, mempool::osd::allocated_items()); + ASSERT_LT(bytes_before, mempool::osd::allocated_bytes()); + + // try_.. won't + items_before = mempool::osd::allocated_items(); + bytes_before = mempool::osd::allocated_bytes(); + bl.try_assign_to_mempool(mempool::mempool_bloom_filter); + ASSERT_EQ(items_before, mempool::osd::allocated_items()); + ASSERT_EQ(bytes_before, mempool::osd::allocated_bytes()); +} + +TEST(mempool, bufferlist_c_str) +{ + bufferlist bl; + int len = 1048576; + size_t before = mempool::osd::allocated_bytes(); + bl.append(buffer::create_aligned(len, 4096)); + bl.append(buffer::create_aligned(len, 4096)); + bl.reassign_to_mempool(mempool::mempool_osd); + size_t after = mempool::osd::allocated_bytes(); + ASSERT_GE(after, before + len * 2); + bl.c_str(); + size_t after_c_str = mempool::osd::allocated_bytes(); + ASSERT_EQ(after, after_c_str); +} + +TEST(mempool, btree_map_test) +{ + typedef mempool::pool_allocator<mempool::mempool_osd, + pair<const uint64_t,uint64_t>> allocator_t; + typedef btree::btree_map<uint64_t,uint64_t,std::less<uint64_t>,allocator_t> btree_t; + + { + btree_t btree; + ASSERT_EQ(0, mempool::osd::allocated_items()); + ASSERT_EQ(0, mempool::osd::allocated_bytes()); + for (size_t i = 0; i < 1000; ++i) { + btree[rand()] = rand(); + } + ASSERT_LT(0, mempool::osd::allocated_items()); + ASSERT_LT(0, mempool::osd::allocated_bytes()); + } + + ASSERT_EQ(0, mempool::osd::allocated_items()); + ASSERT_EQ(0, mempool::osd::allocated_bytes()); +} + +TEST(mempool, check_shard_select) +{ + const size_t samples = mempool::num_shards * 100; + std::atomic_int shards[mempool::num_shards] = {0}; + std::vector<std::thread> workers; + for (size_t i = 0; i < samples; i++) { + workers.push_back( + std::thread([&](){ + size_t i = mempool::pool_t::pick_a_shard_int(); + shards[i]++; + })); + } + for (auto& t:workers) { + t.join(); + } + workers.clear(); + + size_t missed = 0; + for (size_t i = 0; i < mempool::num_shards; i++) { + if (shards[i] == 0) { + missed++; + } + } + + // If more than half of the shards did not get anything, + // the distribution is bad enough to deserve a failure. + EXPECT_LT(missed, mempool::num_shards / 2); +} + + +int main(int argc, char **argv) +{ + auto args = argv_to_vec(argc, argv); + + auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, + CODE_ENVIRONMENT_UTILITY, + CINIT_FLAG_NO_DEFAULT_CONFIG_FILE); + common_init_finish(g_ceph_context); + + // enable debug mode for the tests + mempool::set_debug_mode(true); + + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + + +/* + * Local Variables: + * compile-command: "cd ../../build ; make -j4 && + * make unittest_mempool && + * valgrind --tool=memcheck ./unittest_mempool --gtest_filter=*.*" + * End: + */ |