diff options
Diffstat (limited to '')
-rw-r--r-- | src/compressor/brotli/BrotliCompressor.cc | 95 | ||||
-rw-r--r-- | src/compressor/brotli/BrotliCompressor.h | 31 | ||||
-rw-r--r-- | src/compressor/brotli/CMakeLists.txt | 39 | ||||
-rw-r--r-- | src/compressor/brotli/CompressionPluginBrotli.cc | 19 | ||||
-rw-r--r-- | src/compressor/brotli/CompressionPluginBrotli.h | 36 |
5 files changed, 220 insertions, 0 deletions
diff --git a/src/compressor/brotli/BrotliCompressor.cc b/src/compressor/brotli/BrotliCompressor.cc new file mode 100644 index 00000000..27685da3 --- /dev/null +++ b/src/compressor/brotli/BrotliCompressor.cc @@ -0,0 +1,95 @@ +#include "brotli/encode.h" +#include "brotli/decode.h" +#include "BrotliCompressor.h" +#include "include/scope_guard.h" + +#define MAX_LEN (CEPH_PAGE_SIZE) + +int BrotliCompressor::compress(const bufferlist &in, bufferlist &out) +{ + BrotliEncoderState* s = BrotliEncoderCreateInstance(nullptr, + nullptr, + nullptr); + if (!s) { + return -1; + } + auto sg = make_scope_guard([&s] { BrotliEncoderDestroyInstance(s); }); + BrotliEncoderSetParameter(s, BROTLI_PARAM_QUALITY, (uint32_t)9); + BrotliEncoderSetParameter(s, BROTLI_PARAM_LGWIN, 22); + for (auto i = in.buffers().begin(); i != in.buffers().end();) { + size_t available_in = i->length(); + size_t max_comp_size = BrotliEncoderMaxCompressedSize(available_in); + size_t available_out = max_comp_size; + bufferptr ptr = buffer::create_small_page_aligned(max_comp_size); + uint8_t* next_out = (uint8_t*)ptr.c_str(); + const uint8_t* next_in = (uint8_t*)i->c_str(); + ++i; + BrotliEncoderOperation finish = i != in.buffers().end() ? + BROTLI_OPERATION_PROCESS : + BROTLI_OPERATION_FINISH; + do { + if (!BrotliEncoderCompressStream(s, + finish, + &available_in, + &next_in, + &available_out, + &next_out, + nullptr)) { + return -1; + } + unsigned have = max_comp_size - available_out; + out.append(ptr, 0, have); + } while (available_out == 0); + if (BrotliEncoderIsFinished(s)) { + break; + } + } + return 0; +} + +int BrotliCompressor::decompress(bufferlist::const_iterator &p, + size_t compressed_size, + bufferlist &out) +{ + BrotliDecoderState* s = BrotliDecoderCreateInstance(nullptr, + nullptr, + nullptr); + if (!s) { + return -1; + } + auto sg = make_scope_guard([&s] { BrotliDecoderDestroyInstance(s); }); + size_t remaining = std::min<size_t>(p.get_remaining(), compressed_size); + while (remaining) { + const uint8_t* next_in; + size_t len = p.get_ptr_and_advance(remaining, (const char**)&next_in); + remaining -= len; + size_t available_in = len; + BrotliDecoderResult result = BROTLI_DECODER_RESULT_ERROR; + do { + size_t available_out = MAX_LEN; + bufferptr ptr = buffer::create_page_aligned(MAX_LEN); + uint8_t* next_out = (uint8_t*)ptr.c_str(); + result = BrotliDecoderDecompressStream(s, + &available_in, + &next_in, + &available_out, + &next_out, + 0); + if (!result) { + return -1; + } + unsigned have = MAX_LEN - available_out; + out.append(ptr, 0, have); + } while (result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT); + if (BrotliDecoderIsFinished(s)) { + break; + } + } + return 0; +} + +int BrotliCompressor::decompress(const bufferlist &in, bufferlist &out) +{ + auto i = std::cbegin(in); + return decompress(i, in.length(), out); +} diff --git a/src/compressor/brotli/BrotliCompressor.h b/src/compressor/brotli/BrotliCompressor.h new file mode 100644 index 00000000..482fe5e2 --- /dev/null +++ b/src/compressor/brotli/BrotliCompressor.h @@ -0,0 +1,31 @@ +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2017 BI SHUN KE <aionshun@livemail.tw> + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#ifndef CEPH_BROTLICOMPRESSOR_H +#define CEPH_BROTLICOMPRESSOR_H + + +#include "include/buffer.h" +#include "compressor/Compressor.h" + +class BrotliCompressor : public Compressor +{ + public: + BrotliCompressor() : Compressor(COMP_ALG_BROTLI, "brotli") {} + + int compress(const bufferlist &in, bufferlist &out) override; + int decompress(const bufferlist &in, bufferlist &out) override; + int decompress(bufferlist::const_iterator &p, size_t compressed_len, bufferlist &out) override; +}; + +#endif //CEPH_BROTLICOMPRESSOR_H + diff --git a/src/compressor/brotli/CMakeLists.txt b/src/compressor/brotli/CMakeLists.txt new file mode 100644 index 00000000..f1992428 --- /dev/null +++ b/src/compressor/brotli/CMakeLists.txt @@ -0,0 +1,39 @@ +# brotli + +set(brotli_sources + CompressionPluginBrotli.cc + BrotliCompressor.cc +) +include(ExternalProject) +ExternalProject_Add(brotli_ext + DOWNLOAD_DIR ${CMAKE_BINARY_DIR}/src/ + GIT_REPOSITORY "https://github.com/google/brotli.git" + GIT_TAG "v1.0.7" + SOURCE_DIR ${CMAKE_BINARY_DIR}/src/brotli + CONFIGURE_COMMAND ./configure-cmake --disable-debug + INSTALL_COMMAND "" + BUILD_COMMAND $(MAKE) + BUILD_IN_SOURCE 1 + INSTALL_COMMAND "") + +ExternalProject_Add_Step(brotli_ext forcebuild + DEPENDEES configure + DEPENDERS build + COMMAND "true" + ALWAYS 1) + +set(bortli_libs enc dec common) +file(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/src/brotli/c/include") +foreach(lib ${bortli_libs}) + add_library(brotli::${lib} STATIC IMPORTED) + add_dependencies(brotli::${lib} brotli_ext) + set_target_properties(brotli::${lib} PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_BINARY_DIR}/src/brotli/c/include" + IMPORTED_LOCATION "${CMAKE_BINARY_DIR}/src/brotli/libbrotli${lib}-static.a") + list(APPEND BROTLI_LIBRARIES brotli::${lib}) +endforeach() + +add_library(ceph_brotli SHARED ${brotli_sources}) +list(REVERSE bortli_libs) +target_link_libraries(ceph_brotli PRIVATE ${BROTLI_LIBRARIES}) +install(TARGETS ceph_brotli DESTINATION ${compressor_plugin_dir}) diff --git a/src/compressor/brotli/CompressionPluginBrotli.cc b/src/compressor/brotli/CompressionPluginBrotli.cc new file mode 100644 index 00000000..245f49db --- /dev/null +++ b/src/compressor/brotli/CompressionPluginBrotli.cc @@ -0,0 +1,19 @@ +#include "acconfig.h" +#include "ceph_ver.h" +#include "CompressionPluginBrotli.h" +#include "common/ceph_context.h" + + +const char *__ceph_plugin_version() +{ + return CEPH_GIT_NICE_VER; +} + +int __ceph_plugin_init(CephContext *cct, + const std::string& type, + const std::string& name) +{ + PluginRegistry *instance = cct->get_plugin_registry(); + return instance->add(type, name, new CompressionPluginBrotli(cct)); +} + diff --git a/src/compressor/brotli/CompressionPluginBrotli.h b/src/compressor/brotli/CompressionPluginBrotli.h new file mode 100644 index 00000000..641a6e1c --- /dev/null +++ b/src/compressor/brotli/CompressionPluginBrotli.h @@ -0,0 +1,36 @@ +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2017 BI SHUN KE <aionshun@livemail.tw> + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#ifndef CEPH_COMPRESSION_PLUGIN_BROTLI_H +#define CEPH_COMPRESSION_PLUGIN_BROTLI_H + +#include "ceph_ver.h" +#include "compressor/CompressionPlugin.h" +#include "BrotliCompressor.h" + +class CompressionPluginBrotli : public CompressionPlugin { +public: + explicit CompressionPluginBrotli(CephContext *cct) : CompressionPlugin(cct) + {} + + virtual int factory(CompressorRef *cs, std::ostream *ss) + { + if (compressor == nullptr) { + BrotliCompressor *interface = new BrotliCompressor(); + compressor = CompressorRef(interface); + } + *cs = compressor; + return 0; + } +}; + +#endif |