From 19fcec84d8d7d21e796c7624e521b60d28ee21ed Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 7 Apr 2024 20:45:59 +0200 Subject: Adding upstream version 16.2.11+ds. Signed-off-by: Daniel Baumann --- src/test/rgw/test_rgw_compression.cc | 186 +++++++++++++++++++++++++++++++++++ 1 file changed, 186 insertions(+) create mode 100644 src/test/rgw/test_rgw_compression.cc (limited to 'src/test/rgw/test_rgw_compression.cc') diff --git a/src/test/rgw/test_rgw_compression.cc b/src/test/rgw/test_rgw_compression.cc new file mode 100644 index 000000000..b1e340328 --- /dev/null +++ b/src/test/rgw/test_rgw_compression.cc @@ -0,0 +1,186 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +#include "gtest/gtest.h" + +#include "rgw/rgw_compression.h" + +class ut_get_sink : public RGWGetObj_Filter { + bufferlist sink; +public: + ut_get_sink() {} + virtual ~ut_get_sink() {} + + int handle_data(bufferlist& bl, off_t bl_ofs, off_t bl_len) override + { + auto& bl_buffers = bl.buffers(); + auto i = bl_buffers.begin(); + while (bl_len > 0) + { + ceph_assert(i != bl_buffers.end()); + off_t len = std::min(bl_len, i->length()); + sink.append(*i, 0, len); + bl_len -= len; + i++; + } + return 0; + } + bufferlist& get_sink() + { + return sink; + } +}; + +class ut_get_sink_size : public RGWGetObj_Filter { + size_t max_size = 0; +public: + ut_get_sink_size() {} + virtual ~ut_get_sink_size() {} + + int handle_data(bufferlist& bl, off_t bl_ofs, off_t bl_len) override + { + if (bl_len > (off_t)max_size) + max_size = bl_len; + return 0; + } + size_t get_size() + { + return max_size; + } +}; + +class ut_put_sink: public rgw::putobj::DataProcessor +{ + bufferlist sink; +public: + int process(bufferlist&& bl, uint64_t ofs) override + { + sink.claim_append(bl); + return 0; + } + bufferlist& get_sink() + { + return sink; + } +}; + + +struct MockGetDataCB : public RGWGetObj_Filter { + int handle_data(bufferlist& bl, off_t bl_ofs, off_t bl_len) override { + return 0; + } +} cb; + +using range_t = std::pair; + +// call filter->fixup_range() and return the range as a pair. this makes it easy +// to fit on a single line for ASSERT_EQ() +range_t fixup_range(RGWGetObj_Decompress *filter, off_t ofs, off_t end) +{ + filter->fixup_range(ofs, end); + return {ofs, end}; +} + + +TEST(Decompress, FixupRangePartial) +{ + RGWCompressionInfo cs_info; + + // array of blocks with original len=8, compressed to len=6 + auto& blocks = cs_info.blocks; + blocks.emplace_back(compression_block{0, 0, 6}); + blocks.emplace_back(compression_block{8, 6, 6}); + blocks.emplace_back(compression_block{16, 12, 6}); + blocks.emplace_back(compression_block{24, 18, 6}); + + const bool partial = true; + RGWGetObj_Decompress decompress(g_ceph_context, &cs_info, partial, &cb); + + // test translation from logical ranges to compressed ranges + ASSERT_EQ(range_t(0, 5), fixup_range(&decompress, 0, 1)); + ASSERT_EQ(range_t(0, 5), fixup_range(&decompress, 1, 7)); + ASSERT_EQ(range_t(0, 11), fixup_range(&decompress, 7, 8)); + ASSERT_EQ(range_t(0, 11), fixup_range(&decompress, 0, 9)); + ASSERT_EQ(range_t(0, 11), fixup_range(&decompress, 7, 9)); + ASSERT_EQ(range_t(6, 11), fixup_range(&decompress, 8, 9)); + ASSERT_EQ(range_t(6, 17), fixup_range(&decompress, 8, 16)); + ASSERT_EQ(range_t(6, 17), fixup_range(&decompress, 8, 17)); + ASSERT_EQ(range_t(12, 23), fixup_range(&decompress, 16, 24)); + ASSERT_EQ(range_t(12, 23), fixup_range(&decompress, 16, 999)); + ASSERT_EQ(range_t(18, 23), fixup_range(&decompress, 998, 999)); +} + +TEST(Compress, LimitedChunkSize) +{ + CompressorRef plugin; + plugin = Compressor::create(g_ceph_context, Compressor::COMP_ALG_ZLIB); + ASSERT_NE(plugin.get(), nullptr); + + for (size_t s = 100 ; s < 10000000 ; s = s*5/4) + { + bufferptr bp(s); + bufferlist bl; + bl.append(bp); + + ut_put_sink c_sink; + RGWPutObj_Compress compressor(g_ceph_context, plugin, &c_sink); + compressor.process(std::move(bl), 0); + compressor.process({}, s); // flush + + RGWCompressionInfo cs_info; + cs_info.compression_type = plugin->get_type_name(); + cs_info.orig_size = s; + cs_info.compressor_message = compressor.get_compressor_message(); + cs_info.blocks = move(compressor.get_compression_blocks()); + + ut_get_sink_size d_sink; + RGWGetObj_Decompress decompress(g_ceph_context, &cs_info, false, &d_sink); + + off_t f_begin = 0; + off_t f_end = s - 1; + decompress.fixup_range(f_begin, f_end); + + decompress.handle_data(c_sink.get_sink(), 0, c_sink.get_sink().length()); + bufferlist empty; + decompress.handle_data(empty, 0, 0); + + ASSERT_LE(d_sink.get_size(), (size_t)g_ceph_context->_conf->rgw_max_chunk_size); + } +} + + +TEST(Compress, BillionZeros) +{ + CompressorRef plugin; + ut_put_sink c_sink; + plugin = Compressor::create(g_ceph_context, Compressor::COMP_ALG_ZLIB); + ASSERT_NE(plugin.get(), nullptr); + RGWPutObj_Compress compressor(g_ceph_context, plugin, &c_sink); + + constexpr size_t size = 1000000; + bufferptr bp(size); + bufferlist bl; + bl.append(bp); + + for (int i=0; i<1000;i++) + compressor.process(bufferlist{bl}, size*i); + compressor.process({}, size*1000); // flush + + RGWCompressionInfo cs_info; + cs_info.compression_type = plugin->get_type_name(); + cs_info.orig_size = size*1000; + cs_info.compressor_message = compressor.get_compressor_message(); + cs_info.blocks = move(compressor.get_compression_blocks()); + + ut_get_sink d_sink; + RGWGetObj_Decompress decompress(g_ceph_context, &cs_info, false, &d_sink); + + off_t f_begin = 0; + off_t f_end = size*1000 - 1; + decompress.fixup_range(f_begin, f_end); + + decompress.handle_data(c_sink.get_sink(), 0, c_sink.get_sink().length()); + bufferlist empty; + decompress.handle_data(empty, 0, 0); + + ASSERT_EQ(d_sink.get_sink().length() , size*1000); +} -- cgit v1.2.3