From 19fcec84d8d7d21e796c7624e521b60d28ee21ed Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 7 Apr 2024 20:45:59 +0200 Subject: Adding upstream version 16.2.11+ds. Signed-off-by: Daniel Baumann --- src/tools/erasure-code/ceph-erasure-code-tool.cc | 322 +++++++++++++++++++++++ 1 file changed, 322 insertions(+) create mode 100644 src/tools/erasure-code/ceph-erasure-code-tool.cc (limited to 'src/tools/erasure-code/ceph-erasure-code-tool.cc') diff --git a/src/tools/erasure-code/ceph-erasure-code-tool.cc b/src/tools/erasure-code/ceph-erasure-code-tool.cc new file mode 100644 index 000000000..6c99abf46 --- /dev/null +++ b/src/tools/erasure-code/ceph-erasure-code-tool.cc @@ -0,0 +1,322 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "include/buffer.h" +#include "include/stringify.h" +#include "common/ceph_argparse.h" +#include "common/config_proxy.h" +#include "common/errno.h" +#include "erasure-code/ErasureCode.h" +#include "erasure-code/ErasureCodePlugin.h" +#include "global/global_context.h" +#include "global/global_init.h" +#include "osd/ECUtil.h" + +#include +#include +#include +#include + +#include + +std::vector display_params = { + "chunk_count", "data_chunk_count", "coding_chunk_count" +}; + +void usage(const std::string message, ostream &out) { + if (!message.empty()) { + out << message << std::endl; + out << "" << std::endl; + } + out << "usage: ceph-erasure-code-tool test-plugin-exists " << std::endl; + out << " ceph-erasure-code-tool validate-profile [ ...]" << std::endl; + out << " ceph-erasure-code-tool calc-chunk-size " << std::endl; + out << " ceph-erasure-code-tool encode " << std::endl; + out << " ceph-erasure-code-tool decode " << std::endl; + out << "" << std::endl; + out << " plugin - plugin name" << std::endl; + out << " profile - comma separated list of erasure-code profile settings" << std::endl; + out << " example: plugin=jerasure,technique=reed_sol_van,k=3,m=2" << std::endl; + out << " display-param - parameter to display (display all if empty)" << std::endl; + out << " may be: " << display_params << std::endl; + out << " object_size - object size" << std::endl; + out << " stripe_unit - stripe unit" << std::endl; + out << " want_to_encode - comma separated list of shards to encode" << std::endl; + out << " want_to_decode - comma separated list of shards to decode" << std::endl; + out << " fname - name for input/output files" << std::endl; + out << " when encoding input is read form {fname} file," << std::endl; + out << " result is stored in {fname}.{shard} files" << std::endl; + out << " when decoding input is read form {fname}.{shard} files," << std::endl; + out << " result is stored in {fname} file" << std::endl; +} + +int ec_init(const std::string &profile_str, + const std::string &stripe_unit_str, + ceph::ErasureCodeInterfaceRef *ec_impl, + std::unique_ptr *sinfo) { + ceph::ErasureCodeProfile profile; + std::vector opts; + boost::split(opts, profile_str, boost::is_any_of(", ")); + for (auto &opt_str : opts) { + std::vector opt; + boost::split(opt, opt_str, boost::is_any_of("=")); + if (opt.size() <= 1) { + usage("invalid profile", std::cerr); + return 1; + } + profile[opt[0]] = opt[1]; + } + auto plugin = profile.find("plugin"); + if (plugin == profile.end()) { + usage("invalid profile: plugin not specified", std::cerr); + return 1; + } + + stringstream ss; + ceph::ErasureCodePluginRegistry::instance().factory( + plugin->second, g_conf().get_val("erasure_code_dir"), + profile, ec_impl, &ss); + if (!*ec_impl) { + usage("invalid profile: " + ss.str(), std::cerr); + return 1; + } + + if (sinfo == nullptr) { + return 0; + } + + uint64_t stripe_unit = atoi(stripe_unit_str.c_str()); + if (stripe_unit <= 0) { + usage("invalid stripe unit", std::cerr); + return 1; + } + + uint64_t stripe_size = atoi(profile["k"].c_str()); + ceph_assert(stripe_size > 0); + uint64_t stripe_width = stripe_size * stripe_unit; + sinfo->reset(new ECUtil::stripe_info_t(stripe_size, stripe_width)); + + return 0; +} + +int do_test_plugin_exists(const std::vector &args) { + if (args.size() < 1) { + usage("not enought arguments", std::cerr); + return 1; + } + + ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance(); + ErasureCodePlugin *plugin; + stringstream ss; + + std::lock_guard l{instance.lock}; + int r = instance.load( + args[0], g_conf().get_val("erasure_code_dir"), &plugin, &ss); + std::cerr << ss.str() << endl; + return r; +} + +int do_validate_profile(const std::vector &args) { + if (args.size() < 1) { + usage("not enought arguments", std::cerr); + return 1; + } + + ceph::ErasureCodeInterfaceRef ec_impl; + int r = ec_init(args[0], {}, &ec_impl, nullptr); + if (r < 0) { + return r; + } + + if (args.size() > 1) { + std::set valid_params(display_params.begin(), + display_params.end()); + display_params.clear(); + for (size_t i = 1; i < args.size(); i++) { + if (!valid_params.count(args[i])) { + usage("invalid display param: " + std::string(args[i]), std::cerr); + return 1; + } + display_params.push_back(args[i]); + } + } + + for (auto ¶m : display_params) { + if (display_params.size() > 1) { + std::cout << param << ": "; + } + if (param == "chunk_count") { + std::cout << ec_impl->get_chunk_count() << std::endl; + } else if (param == "data_chunk_count") { + std::cout << ec_impl->get_data_chunk_count() << std::endl; + } else if (param == "coding_chunk_count") { + std::cout << ec_impl->get_coding_chunk_count() << std::endl; + } else { + ceph_abort_msgf("unknown display_param: %s", param.c_str()); + } + } + + return 0; +} + +int do_calc_chunk_size(const std::vector &args) { + if (args.size() < 2) { + usage("not enought arguments", std::cerr); + return 1; + } + + ceph::ErasureCodeInterfaceRef ec_impl; + int r = ec_init(args[0], {}, &ec_impl, nullptr); + if (r < 0) { + return r; + } + + uint64_t object_size = atoi(args[1]); + if (object_size <= 0) { + usage("invalid object size", std::cerr); + return 1; + } + + std::cout << ec_impl->get_chunk_size(object_size) << std::endl; + return 0; +} + +int do_encode(const std::vector &args) { + if (args.size() < 4) { + usage("not enought arguments", std::cerr); + return 1; + } + + ceph::ErasureCodeInterfaceRef ec_impl; + std::unique_ptr sinfo; + int r = ec_init(args[0], args[1], &ec_impl, &sinfo); + if (r < 0) { + return r; + } + + std::set want; + std::vector shards; + boost::split(shards, args[2], boost::is_any_of(",")); + for (auto &shard : shards) { + want.insert(atoi(shard.c_str())); + } + ceph::bufferlist decoded_data; + std::string fname = args[3]; + + std::string error; + r = decoded_data.read_file(fname.c_str(), &error); + if (r < 0) { + std::cerr << "failed to read " << fname << ": " << error << std::endl; + return 1; + } + + uint64_t stripe_width = sinfo->get_stripe_width(); + if (decoded_data.length() % stripe_width != 0) { + uint64_t pad = stripe_width - decoded_data.length() % stripe_width; + decoded_data.append_zero(pad); + } + + std::map encoded_data; + r = ECUtil::encode(*sinfo, ec_impl, decoded_data, want, &encoded_data); + if (r < 0) { + std::cerr << "failed to encode: " << cpp_strerror(r) << std::endl; + return 1; + } + + for (auto &[shard, bl] : encoded_data) { + std::string name = fname + "." + stringify(shard); + r = bl.write_file(name.c_str()); + if (r < 0) { + std::cerr << "failed to write " << name << ": " << cpp_strerror(r) + << std::endl; + return 1; + } + } + + return 0; +} + +int do_decode(const std::vector &args) { + if (args.size() < 4) { + usage("not enought arguments", std::cerr); + return 1; + } + + ceph::ErasureCodeInterfaceRef ec_impl; + std::unique_ptr sinfo; + int r = ec_init(args[0], args[1], &ec_impl, &sinfo); + if (r < 0) { + return r; + } + + std::map encoded_data; + std::vector shards; + boost::split(shards, args[2], boost::is_any_of(",")); + for (auto &shard : shards) { + encoded_data[atoi(shard.c_str())] = {}; + } + ceph::bufferlist decoded_data; + std::string fname = args[3]; + + for (auto &[shard, bl] : encoded_data) { + std::string name = fname + "." + stringify(shard); + std::string error; + r = bl.read_file(name.c_str(), &error); + if (r < 0) { + std::cerr << "failed to read " << name << ": " << error << std::endl; + return 1; + } + } + + r = ECUtil::decode(*sinfo, ec_impl, encoded_data, &decoded_data); + if (r < 0) { + std::cerr << "failed to decode: " << cpp_strerror(r) << std::endl; + return 1; + } + + r = decoded_data.write_file(fname.c_str()); + if (r < 0) { + std::cerr << "failed to write " << fname << ": " << cpp_strerror(r) + << std::endl; + return 1; + } + + return 0; +} + +int main(int argc, const char **argv) { + std::vector args; + argv_to_vec(argc, argv, args); + auto cct = global_init(nullptr, args, CEPH_ENTITY_TYPE_CLIENT, + CODE_ENVIRONMENT_UTILITY, + CINIT_FLAG_NO_MON_CONFIG); + + if (args.empty() || args[0] == std::string("-h") || + args[0] == std::string("--help")) { + usage("", std::cout); + return 0; + } + + if (args.size() < 1) { + usage("not enought arguments", std::cerr); + return 1; + } + + std::string cmd = args[0]; + std::vector cmd_args(args.begin() + 1, args.end()); + + if (cmd == "test-plugin-exists") { + return do_test_plugin_exists(cmd_args); + } else if (cmd == "validate-profile") { + return do_validate_profile(cmd_args); + } else if (cmd == "calc-chunk-size") { + return do_calc_chunk_size(cmd_args); + } else if (cmd == "encode") { + return do_encode(cmd_args); + } else if (cmd == "decode") { + return do_decode(cmd_args); + } + + usage("invalid command: " + cmd, std::cerr); + return 1; +} -- cgit v1.2.3