From 19fcec84d8d7d21e796c7624e521b60d28ee21ed Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 7 Apr 2024 20:45:59 +0200 Subject: Adding upstream version 16.2.11+ds. Signed-off-by: Daniel Baumann --- src/tools/ceph-dencoder/CMakeLists.txt | 109 ++++++++ src/tools/ceph-dencoder/ceph_dencoder.cc | 272 ++++++++++++++++++ src/tools/ceph-dencoder/ceph_time.h | 68 +++++ src/tools/ceph-dencoder/common_types.cc | 36 +++ src/tools/ceph-dencoder/common_types.h | 457 +++++++++++++++++++++++++++++++ src/tools/ceph-dencoder/denc_plugin.h | 82 ++++++ src/tools/ceph-dencoder/denc_registry.h | 241 ++++++++++++++++ src/tools/ceph-dencoder/mds_types.cc | 36 +++ src/tools/ceph-dencoder/mds_types.h | 111 ++++++++ src/tools/ceph-dencoder/osd_types.cc | 39 +++ src/tools/ceph-dencoder/osd_types.h | 153 +++++++++++ src/tools/ceph-dencoder/rbd_types.cc | 36 +++ src/tools/ceph-dencoder/rbd_types.h | 52 ++++ src/tools/ceph-dencoder/rgw_types.cc | 36 +++ src/tools/ceph-dencoder/rgw_types.h | 131 +++++++++ src/tools/ceph-dencoder/sstring.h | 40 +++ 16 files changed, 1899 insertions(+) create mode 100644 src/tools/ceph-dencoder/CMakeLists.txt create mode 100644 src/tools/ceph-dencoder/ceph_dencoder.cc create mode 100644 src/tools/ceph-dencoder/ceph_time.h create mode 100644 src/tools/ceph-dencoder/common_types.cc create mode 100644 src/tools/ceph-dencoder/common_types.h create mode 100644 src/tools/ceph-dencoder/denc_plugin.h create mode 100644 src/tools/ceph-dencoder/denc_registry.h create mode 100644 src/tools/ceph-dencoder/mds_types.cc create mode 100644 src/tools/ceph-dencoder/mds_types.h create mode 100644 src/tools/ceph-dencoder/osd_types.cc create mode 100644 src/tools/ceph-dencoder/osd_types.h create mode 100644 src/tools/ceph-dencoder/rbd_types.cc create mode 100644 src/tools/ceph-dencoder/rbd_types.h create mode 100644 src/tools/ceph-dencoder/rgw_types.cc create mode 100644 src/tools/ceph-dencoder/rgw_types.h create mode 100644 src/tools/ceph-dencoder/sstring.h (limited to 'src/tools/ceph-dencoder') diff --git a/src/tools/ceph-dencoder/CMakeLists.txt b/src/tools/ceph-dencoder/CMakeLists.txt new file mode 100644 index 000000000..a92ac5e69 --- /dev/null +++ b/src/tools/ceph-dencoder/CMakeLists.txt @@ -0,0 +1,109 @@ +## dencoder +set_source_files_properties( + ceph_dencoder.cc + APPEND PROPERTY OBJECT_DEPENDS ${CMAKE_BINARY_DIR}/src/include/ceph_ver.h) + +if(HAS_VTA) + set_source_files_properties(ceph_dencoder.cc + PROPERTIES COMPILE_FLAGS -fno-var-tracking-assignments) +endif() + +set(dencoder_srcs + ceph_dencoder.cc + ../../include/uuid.cc + ../../include/utime.cc + $) +add_executable(ceph-dencoder ${dencoder_srcs}) + +set(denc_plugin_dir ${CEPH_INSTALL_PKGLIBDIR}/denc) +add_custom_target(ceph-dencoder-modules) + +function(add_denc_mod name) + add_library(${name} SHARED + ${ARGN}) + set_target_properties(${name} PROPERTIES + PREFIX "" + OUTPUT_NAME ${name} + CXX_VISIBILITY_PRESET hidden + VISIBILITY_INLINES_HIDDEN ON) + install( + TARGETS ${name} + DESTINATION ${denc_plugin_dir}) + add_dependencies(ceph-dencoder-modules + ${name}) +endfunction() + +add_denc_mod(denc-mod-common + common_types.cc) +target_link_libraries(denc-mod-common + journal + cls_cas_internal + cls_lock_client + cls_refcount_client + cls_timeindex_client) +add_denc_mod(denc-mod-osd + osd_types.cc) +target_link_libraries(denc-mod-osd + os + osd + mon + erasure_code) + +if(WITH_RADOSGW) + add_denc_mod(denc-mod-rgw + rgw_types.cc + ${CMAKE_SOURCE_DIR}/src/rgw/rgw_dencoder.cc) + target_link_libraries(denc-mod-rgw + rgw_a + cls_rgw_client + cls_journal_client) + if(WITH_RADOSGW_AMQP_ENDPOINT) + target_link_libraries(denc-mod-rgw + rabbitmq ssl) + endif() + if(WITH_RADOSGW_KAFKA_ENDPOINT) + target_link_libraries(denc-mod-rgw + rdkafka) + endif() + if(WITH_RADOSGW_LUA_PACKAGES) + target_link_libraries(denc-mod-rgw + Boost::filesystem) + endif() +endif() + +if(WITH_RBD) + add_denc_mod(denc-mod-rbd + rbd_types.cc) + target_link_libraries(denc-mod-rbd + cls_rbd_client + rbd_mirror_types + rbd_types + rbd_replay_types) + if(WITH_KRBD) + target_link_libraries(denc-mod-rbd + krbd) + endif() +endif() + +if(WITH_CEPHFS) + add_denc_mod(denc-mod-cephfs + mds_types.cc) + target_link_libraries(denc-mod-cephfs + mds) +endif() + +target_compile_definitions(ceph-dencoder PRIVATE + "CEPH_DENC_MOD_DIR=\"${denc_plugin_dir}\"") + +target_link_libraries(ceph-dencoder + StdFilesystem::filesystem + global + ${DENCODER_EXTRALIBS} + cls_log_client + cls_version_client + cls_user_client + cls_cas_client + ${ALLOC_LIBS} + ${EXTRALIBS} + ${CMAKE_DL_LIBS}) +install(TARGETS ceph-dencoder DESTINATION bin) diff --git a/src/tools/ceph-dencoder/ceph_dencoder.cc b/src/tools/ceph-dencoder/ceph_dencoder.cc new file mode 100644 index 000000000..42060f860 --- /dev/null +++ b/src/tools/ceph-dencoder/ceph_dencoder.cc @@ -0,0 +1,272 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2015 Red Hat + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + + +#include + +#if __has_include() +#include +namespace fs = std::filesystem; +#else +#include +namespace fs = std::experimental::filesystem; +#endif + +#include "ceph_ver.h" +#include "include/types.h" +#include "common/Formatter.h" +#include "common/ceph_argparse.h" +#include "common/errno.h" +#include "denc_plugin.h" +#include "denc_registry.h" + +#define MB(m) ((m) * 1024 * 1024) + +void usage(ostream &out) +{ + out << "usage: ceph-dencoder [commands ...]" << std::endl; + out << "\n"; + out << " version print version string (to stdout)\n"; + out << "\n"; + out << " import read encoded data from encfile\n"; + out << " export write encoded data to outfile\n"; + out << "\n"; + out << " set_features set feature bits used for encoding\n"; + out << " get_features print feature bits (int) to stdout\n"; + out << "\n"; + out << " list_types list supported types\n"; + out << " type select in-memory type\n"; + out << " skip skip leading bytes before decoding\n"; + out << " decode decode into in-memory object\n"; + out << " encode encode in-memory object\n"; + out << " dump_json dump in-memory object as json (to stdout)\n"; + out << " hexdump print encoded data in hex\n"; + out << " get_struct_v print version of the encoded object\n"; + out << " get_struct_compat print the oldest version of decoder that can decode the encoded object\n"; + out << "\n"; + out << " copy copy object (via operator=)\n"; + out << " copy_ctor copy object (via copy ctor)\n"; + out << "\n"; + out << " count_tests print number of generated test objects (to stdout)\n"; + out << " select_test select generated test object as in-memory object\n"; + out << " is_deterministic exit w/ success if type encodes deterministically\n"; +} + +vector load_plugins() +{ + fs::path mod_dir{CEPH_DENC_MOD_DIR}; + if (auto ceph_lib = getenv("CEPH_LIB"); ceph_lib) { + mod_dir = ceph_lib; + } else if (fs::is_regular_file("CMakeCache.txt")) { + mod_dir = fs::canonical("lib"); + } + vector dencoder_plugins; + for (auto& entry : fs::directory_iterator(mod_dir)) { + static const string_view DENC_MOD_PREFIX = "denc-mod-"; + if (entry.path().stem().string().compare(0, DENC_MOD_PREFIX.size(), + DENC_MOD_PREFIX) != 0) { + continue; + } + DencoderPlugin plugin(entry); + if (!plugin.good()) { + continue; + } + dencoder_plugins.push_back(std::move(plugin)); + } + return dencoder_plugins; +} + +int main(int argc, const char **argv) +{ + vector plugins = load_plugins(); + DencoderRegistry registry; + for (auto& plugin : plugins) { + for (auto& [name, denc] : plugin.register_dencoders()) { + registry.register_dencoder(name, denc); + } + } + + vector args; + argv_to_vec(argc, argv, args); + env_to_vec(args); + + Dencoder *den = NULL; + uint64_t features = CEPH_FEATURES_SUPPORTED_DEFAULT; + bufferlist encbl; + uint64_t skip = 0; + + if (args.empty()) { + cerr << "-h for help" << std::endl; + return 1; + } + for (std::vector::iterator i = args.begin(); i != args.end(); ++i) { + string err; + + auto& dencoders = registry.get(); + if (*i == string("help") || *i == string("-h") || *i == string("--help")) { + usage(cout); + return 0; + } else if (*i == string("version")) { + cout << CEPH_GIT_NICE_VER << std::endl; + } else if (*i == string("list_types")) { + for (auto& dencoder : dencoders) + cout << dencoder.first << std::endl; + return 0; + } else if (*i == string("type")) { + ++i; + if (i == args.end()) { + cerr << "expecting type" << std::endl; + return 1; + } + string cname = *i; + if (!dencoders.count(cname)) { + cerr << "class '" << cname << "' unknown" << std::endl; + return 1; + } + den = dencoders[cname]; + den->generate(); + } else if (*i == string("skip")) { + ++i; + if (i == args.end()) { + cerr << "expecting byte count" << std::endl; + return 1; + } + skip = atoi(*i); + } else if (*i == string("get_features")) { + cout << CEPH_FEATURES_SUPPORTED_DEFAULT << std::endl; + return 0; + } else if (*i == string("set_features")) { + ++i; + if (i == args.end()) { + cerr << "expecting features" << std::endl; + return 1; + } + features = atoll(*i); + } else if (*i == string("encode")) { + if (!den) { + cerr << "must first select type with 'type '" << std::endl; + return 1; + } + den->encode(encbl, features | CEPH_FEATURE_RESERVED); // hack for OSDMap + } else if (*i == string("decode")) { + if (!den) { + cerr << "must first select type with 'type '" << std::endl; + return 1; + } + err = den->decode(encbl, skip); + } else if (*i == string("copy_ctor")) { + if (!den) { + cerr << "must first select type with 'type '" << std::endl; + return 1; + } + den->copy_ctor(); + } else if (*i == string("copy")) { + if (!den) { + cerr << "must first select type with 'type '" << std::endl; + return 1; + } + den->copy(); + } else if (*i == string("dump_json")) { + if (!den) { + cerr << "must first select type with 'type '" << std::endl; + return 1; + } + JSONFormatter jf(true); + jf.open_object_section("object"); + den->dump(&jf); + jf.close_section(); + jf.flush(cout); + cout << std::endl; + + } else if (*i == string("hexdump")) { + encbl.hexdump(cout); + } else if (*i == string("get_struct_v")) { + std::cout << den->get_struct_v(encbl, 0) << std::endl; + } else if (*i == string("get_struct_compat")) { + std::cout << den->get_struct_v(encbl, sizeof(uint8_t)) << std::endl; + } else if (*i == string("import")) { + ++i; + if (i == args.end()) { + cerr << "expecting filename" << std::endl; + return 1; + } + int r; + if (*i == string("-")) { + *i = "stdin"; + // Read up to 1mb if stdin specified + r = encbl.read_fd(STDIN_FILENO, MB(1)); + } else { + r = encbl.read_file(*i, &err); + } + if (r < 0) { + cerr << "error reading " << *i << ": " << err << std::endl; + return 1; + } + + } else if (*i == string("export")) { + ++i; + if (i == args.end()) { + cerr << "expecting filename" << std::endl; + return 1; + } + int fd = ::open(*i, O_WRONLY|O_CREAT|O_TRUNC|O_BINARY, 0644); + if (fd < 0) { + cerr << "error opening " << *i << " for write: " << cpp_strerror(errno) << std::endl; + return 1; + } + int r = encbl.write_fd(fd); + if (r < 0) { + cerr << "error writing " << *i << ": " << cpp_strerror(errno) << std::endl; + return 1; + } + ::close(fd); + + } else if (*i == string("count_tests")) { + if (!den) { + cerr << "must first select type with 'type '" << std::endl; + return 1; + } + cout << den->num_generated() << std::endl; + } else if (*i == string("select_test")) { + if (!den) { + cerr << "must first select type with 'type '" << std::endl; + return 1; + } + ++i; + if (i == args.end()) { + cerr << "expecting instance number" << std::endl; + return 1; + } + int n = atoi(*i); + err = den->select_generated(n); + } else if (*i == string("is_deterministic")) { + if (!den) { + cerr << "must first select type with 'type '" << std::endl; + return 1; + } + if (den->is_deterministic()) + return 0; + else + return 1; + } else { + cerr << "unknown option '" << *i << "'" << std::endl; + return 1; + } + if (err.length()) { + cerr << "error: " << err << std::endl; + return 1; + } + } + return 0; +} diff --git a/src/tools/ceph-dencoder/ceph_time.h b/src/tools/ceph-dencoder/ceph_time.h new file mode 100644 index 000000000..c27cb5746 --- /dev/null +++ b/src/tools/ceph-dencoder/ceph_time.h @@ -0,0 +1,68 @@ +#ifndef TEST_CEPH_TIME_H +#define TEST_CEPH_TIME_H + +#include + +#include "include/encoding.h" +#include "common/ceph_time.h" +#include "common/Formatter.h" + +// wrapper for ceph::real_time that implements the dencoder interface +template +class time_point_wrapper { + using time_point = typename Clock::time_point; + time_point t; + public: + time_point_wrapper() = default; + explicit time_point_wrapper(const time_point& t) : t(t) {} + + void encode(bufferlist& bl) const { + using ceph::encode; + encode(t, bl); + } + void decode(bufferlist::const_iterator &p) { + using ceph::decode; + decode(t, p); + } + void dump(Formatter* f) { + auto epoch_time = Clock::to_time_t(t); + f->dump_string("time", std::ctime(&epoch_time)); + } + static void generate_test_instances(std::list& ls) { + constexpr time_t t{455500800}; // Ghostbusters release date + ls.push_back(new time_point_wrapper(Clock::from_time_t(t))); + } +}; + +using real_time_wrapper = time_point_wrapper; +WRITE_CLASS_ENCODER(real_time_wrapper) + +using coarse_real_time_wrapper = time_point_wrapper; +WRITE_CLASS_ENCODER(coarse_real_time_wrapper) + +// wrapper for ceph::timespan that implements the dencoder interface +class timespan_wrapper { + ceph::timespan d; + public: + timespan_wrapper() = default; + explicit timespan_wrapper(const ceph::timespan& d) : d(d) {} + + void encode(bufferlist& bl) const { + using ceph::encode; + encode(d, bl); + } + void decode(bufferlist::const_iterator &p) { + using ceph::decode; + decode(d, p); + } + void dump(Formatter* f) { + f->dump_int("timespan", d.count()); + } + static void generate_test_instances(std::list& ls) { + constexpr std::chrono::seconds d{7377}; // marathon world record (2:02:57) + ls.push_back(new timespan_wrapper(d)); + } +}; +WRITE_CLASS_ENCODER(timespan_wrapper) + +#endif diff --git a/src/tools/ceph-dencoder/common_types.cc b/src/tools/ceph-dencoder/common_types.cc new file mode 100644 index 000000000..fa763c3bb --- /dev/null +++ b/src/tools/ceph-dencoder/common_types.cc @@ -0,0 +1,36 @@ +#include "acconfig.h" +#include +using namespace std; +#include "include/ceph_features.h" + +#define TYPE(t) +#define TYPE_STRAYDATA(t) +#define TYPE_NONDETERMINISTIC(t) +#define TYPE_FEATUREFUL(t) +#define TYPE_FEATUREFUL_STRAYDATA(t) +#define TYPE_FEATUREFUL_NONDETERMINISTIC(t) +#define TYPE_FEATUREFUL_NOCOPY(t) +#define TYPE_NOCOPY(t) +#define MESSAGE(t) +#include "common_types.h" +#undef TYPE +#undef TYPE_STRAYDATA +#undef TYPE_NONDETERMINISTIC +#undef TYPE_NOCOPY +#undef TYPE_FEATUREFUL +#undef TYPE_FEATUREFUL_STRAYDATA +#undef TYPE_FEATUREFUL_NONDETERMINISTIC +#undef TYPE_FEATUREFUL_NOCOPY +#undef MESSAGE + +#include "denc_plugin.h" + +DENC_API void register_dencoders(DencoderPlugin* plugin) +{ +#include "common_types.h" +} + +DENC_API void unregister_dencoders(DencoderPlugin* plugin) +{ + plugin->unregister_dencoders(); +} diff --git a/src/tools/ceph-dencoder/common_types.h b/src/tools/ceph-dencoder/common_types.h new file mode 100644 index 000000000..1eba16bb8 --- /dev/null +++ b/src/tools/ceph-dencoder/common_types.h @@ -0,0 +1,457 @@ +#include "ceph_time.h" +TYPE(real_time_wrapper) +TYPE(coarse_real_time_wrapper) +TYPE(timespan_wrapper) + +#include "include/utime.h" +TYPE(utime_t) + +#include "include/uuid.h" +TYPE(uuid_d) + +#include "sstring.h" +TYPE(sstring_wrapper) + +#include "include/CompatSet.h" +TYPE(CompatSet) + +#include "include/filepath.h" +TYPE(filepath) + +#include "include/fs_types.h" +TYPE_FEATUREFUL(file_layout_t) + +#include "include/util.h" +TYPE(ceph_data_stats) + +#include "common/bit_vector.hpp" +TYPE(BitVector<2>) + +#include "common/bloom_filter.hpp" +TYPE(bloom_filter) +TYPE(compressible_bloom_filter) + +#include "common/DecayCounter.h" +TYPE(DecayCounter) + +#include "common/histogram.h" +TYPE(pow2_hist_t) + +#include "common/hobject.h" +TYPE(hobject_t) +TYPE(ghobject_t) + +#include "common/LogEntry.h" +TYPE_FEATUREFUL(LogEntry) +TYPE_FEATUREFUL(LogSummary) + +#include "common/SloppyCRCMap.h" +TYPE(SloppyCRCMap) + +#include "common/snap_types.h" +TYPE(SnapContext) +TYPE(SnapRealmInfo) + +#include "msg/msg_types.h" +TYPE(entity_name_t) +TYPE_FEATUREFUL(entity_addr_t) +TYPE_FEATUREFUL(entity_addrvec_t) +TYPE_FEATUREFUL(entity_inst_t) + +#include "crush/CrushWrapper.h" +TYPE_FEATUREFUL_NOCOPY(CrushWrapper) + +#include "cls/cas/cls_cas_ops.h" +TYPE(cls_cas_chunk_create_or_get_ref_op) +TYPE(cls_cas_chunk_get_ref_op) +TYPE(cls_cas_chunk_put_ref_op) + +#include "cls/cas/cls_cas_internal.h" +TYPE(chunk_refs_t) + +#include "cls/lock/cls_lock_types.h" +TYPE(rados::cls::lock::locker_id_t) +TYPE_FEATUREFUL(rados::cls::lock::locker_info_t) +TYPE_FEATUREFUL(rados::cls::lock::lock_info_t) + +#include "cls/lock/cls_lock_ops.h" +TYPE(cls_lock_lock_op) +TYPE(cls_lock_unlock_op) +TYPE(cls_lock_break_op) +TYPE(cls_lock_get_info_op) +TYPE_FEATUREFUL(cls_lock_get_info_reply) +TYPE(cls_lock_list_locks_reply) +TYPE(cls_lock_assert_op) +TYPE(cls_lock_set_cookie_op) + +#include "cls/refcount/cls_refcount_ops.h" +TYPE(cls_refcount_get_op) +TYPE(cls_refcount_put_op) +TYPE(cls_refcount_set_op) +TYPE(cls_refcount_read_op) +TYPE(cls_refcount_read_ret) +TYPE(obj_refcount) + +#include "cls/timeindex/cls_timeindex_types.h" +TYPE(cls_timeindex_entry) + +#include "journal/Entry.h" +TYPE(journal::Entry) + +// --- messages --- +#include "messages/MAuth.h" +MESSAGE(MAuth) + +#include "messages/MAuthReply.h" +MESSAGE(MAuthReply) + +#include "messages/MCacheExpire.h" +MESSAGE(MCacheExpire) + +#include "messages/MClientCapRelease.h" +MESSAGE(MClientCapRelease) + +#include "messages/MClientCaps.h" +MESSAGE(MClientCaps) + +#include "messages/MClientLease.h" +MESSAGE(MClientLease) + +#include "messages/MClientReconnect.h" +MESSAGE(MClientReconnect) + +#include "messages/MClientReply.h" +MESSAGE(MClientReply) + +#include "messages/MClientRequest.h" +MESSAGE(MClientRequest) + +#include "messages/MClientRequestForward.h" +MESSAGE(MClientRequestForward) + +#include "messages/MClientQuota.h" +MESSAGE(MClientQuota) + +#include "messages/MClientSession.h" +MESSAGE(MClientSession) + +#include "messages/MClientSnap.h" +MESSAGE(MClientSnap) + +#include "messages/MCommand.h" +MESSAGE(MCommand) + +#include "messages/MCommandReply.h" +MESSAGE(MCommandReply) + +#include "messages/MConfig.h" +MESSAGE(MConfig) + +#include "messages/MDentryLink.h" +MESSAGE(MDentryLink) + +#include "messages/MDentryUnlink.h" +MESSAGE(MDentryUnlink) + +#include "messages/MDirUpdate.h" +MESSAGE(MDirUpdate) + +#include "messages/MDiscover.h" +MESSAGE(MDiscover) + +#include "messages/MDiscoverReply.h" +MESSAGE(MDiscoverReply) + +#include "messages/MExportCaps.h" +MESSAGE(MExportCaps) + +#include "messages/MExportCapsAck.h" +MESSAGE(MExportCapsAck) + +#include "messages/MExportDir.h" +MESSAGE(MExportDir) + +#include "messages/MExportDirAck.h" +MESSAGE(MExportDirAck) + +#include "messages/MExportDirCancel.h" +MESSAGE(MExportDirCancel) + +#include "messages/MExportDirDiscover.h" +MESSAGE(MExportDirDiscover) + +#include "messages/MExportDirDiscoverAck.h" +MESSAGE(MExportDirDiscoverAck) + +#include "messages/MExportDirFinish.h" +MESSAGE(MExportDirFinish) + +#include "messages/MExportDirNotify.h" +MESSAGE(MExportDirNotify) + +#include "messages/MExportDirNotifyAck.h" +MESSAGE(MExportDirNotifyAck) + +#include "messages/MExportDirPrep.h" +MESSAGE(MExportDirPrep) + +#include "messages/MExportDirPrepAck.h" +MESSAGE(MExportDirPrepAck) + +#include "messages/MForward.h" +MESSAGE(MForward) + +#include "messages/MFSMap.h" +MESSAGE(MFSMap) + +#include "messages/MFSMapUser.h" +MESSAGE(MFSMapUser) + +#include "messages/MGatherCaps.h" +MESSAGE(MGatherCaps) + +#include "messages/MGenericMessage.h" +MESSAGE(MGenericMessage) + +#include "messages/MGetConfig.h" +MESSAGE(MGetConfig) + +#include "messages/MGetPoolStats.h" +MESSAGE(MGetPoolStats) + +#include "messages/MGetPoolStatsReply.h" +MESSAGE(MGetPoolStatsReply) + +#include "messages/MHeartbeat.h" +MESSAGE(MHeartbeat) + +#include "messages/MInodeFileCaps.h" +MESSAGE(MInodeFileCaps) + +#include "messages/MLock.h" +MESSAGE(MLock) + +#include "messages/MLog.h" +MESSAGE(MLog) + +#include "messages/MLogAck.h" +MESSAGE(MLogAck) + +#include "messages/MMDSOpenIno.h" +MESSAGE(MMDSOpenIno) + +#include "messages/MMDSOpenInoReply.h" +MESSAGE(MMDSOpenInoReply) + +#include "messages/MMDSBeacon.h" +MESSAGE(MMDSBeacon) + +#include "messages/MMDSCacheRejoin.h" +MESSAGE(MMDSCacheRejoin) + +#include "messages/MMDSFindIno.h" +MESSAGE(MMDSFindIno) + +#include "messages/MMDSFindInoReply.h" +MESSAGE(MMDSFindInoReply) + +#include "messages/MMDSFragmentNotify.h" +MESSAGE(MMDSFragmentNotify) + +#include "messages/MMDSLoadTargets.h" +MESSAGE(MMDSLoadTargets) + +#include "messages/MMDSMap.h" +MESSAGE(MMDSMap) + +#include "messages/MMgrReport.h" +MESSAGE(MMgrReport) + +#include "messages/MMDSResolve.h" +MESSAGE(MMDSResolve) + +#include "messages/MMDSResolveAck.h" +MESSAGE(MMDSResolveAck) + +#include "messages/MMDSPeerRequest.h" +MESSAGE(MMDSPeerRequest) + +#include "messages/MMDSSnapUpdate.h" +MESSAGE(MMDSSnapUpdate) + +#include "messages/MMDSTableRequest.h" +MESSAGE(MMDSTableRequest) + +#include "messages/MMgrClose.h" +MESSAGE(MMgrClose) + +#include "messages/MMgrConfigure.h" +MESSAGE(MMgrConfigure) + +#include "messages/MMgrDigest.h" +MESSAGE(MMgrDigest) + +#include "messages/MMgrMap.h" +MESSAGE(MMgrMap) + +#include "messages/MMgrOpen.h" +MESSAGE(MMgrOpen) + +#include "messages/MMonCommand.h" +MESSAGE(MMonCommand) + +#include "messages/MMonCommandAck.h" +MESSAGE(MMonCommandAck) + +#include "messages/MMonElection.h" +MESSAGE(MMonElection) + +#include "messages/MMonGetMap.h" +MESSAGE(MMonGetMap) + +#include "messages/MMonGetVersion.h" +MESSAGE(MMonGetVersion) + +#include "messages/MMonGetVersionReply.h" +MESSAGE(MMonGetVersionReply) + +#include "messages/MMonGlobalID.h" +MESSAGE(MMonGlobalID) + +#include "messages/MMonJoin.h" +MESSAGE(MMonJoin) + +#include "messages/MMonMap.h" +MESSAGE(MMonMap) + +#include "messages/MMonPaxos.h" +MESSAGE(MMonPaxos) + +#include "messages/MMonProbe.h" +MESSAGE(MMonProbe) + +#include "messages/MMonScrub.h" +MESSAGE(MMonScrub) + +#include "messages/MMonSync.h" +MESSAGE(MMonSync) + +#include "messages/MMonSubscribe.h" +MESSAGE(MMonSubscribe) + +#include "messages/MMonSubscribeAck.h" +MESSAGE(MMonSubscribeAck) + +#include "messages/MOSDAlive.h" +MESSAGE(MOSDAlive) + +#include "messages/MOSDBoot.h" +MESSAGE(MOSDBoot) + +#include "messages/MOSDFailure.h" +MESSAGE(MOSDFailure) + +#include "messages/MOSDMap.h" +MESSAGE(MOSDMap) + +#include "messages/MOSDOp.h" +MESSAGE(MOSDOp) + +#include "messages/MOSDOpReply.h" +MESSAGE(MOSDOpReply) + +#include "messages/MOSDPGBackfill.h" +MESSAGE(MOSDPGBackfill) + +#include "messages/MOSDPGCreate.h" +MESSAGE(MOSDPGCreate) + +#include "messages/MOSDPGCreate2.h" +MESSAGE(MOSDPGCreate2) + +#include "messages/MOSDPGInfo.h" +MESSAGE(MOSDPGInfo) + +#include "messages/MOSDPGLog.h" +MESSAGE(MOSDPGLog) + +#include "messages/MOSDPGNotify.h" +MESSAGE(MOSDPGNotify) + +#include "messages/MOSDPGQuery.h" +MESSAGE(MOSDPGQuery) + +#include "messages/MOSDPGRemove.h" +MESSAGE(MOSDPGRemove) + +#include "messages/MOSDPGRecoveryDelete.h" +MESSAGE(MOSDPGRecoveryDelete) + +#include "messages/MOSDPGRecoveryDeleteReply.h" +MESSAGE(MOSDPGRecoveryDeleteReply) + +#include "messages/MOSDPGScan.h" +MESSAGE(MOSDPGScan) + +#include "messages/MOSDPGTemp.h" +MESSAGE(MOSDPGTemp) + +#include "messages/MOSDPGTrim.h" +MESSAGE(MOSDPGTrim) + +#include "messages/MOSDPing.h" +MESSAGE(MOSDPing) + +#include "messages/MOSDRepScrub.h" +MESSAGE(MOSDRepScrub) + +#include "messages/MOSDScrub.h" +MESSAGE(MOSDScrub) + +#include "messages/MOSDScrub2.h" +MESSAGE(MOSDScrub2) + +#include "messages/MOSDForceRecovery.h" +MESSAGE(MOSDForceRecovery) + +#include "messages/MPGStats.h" +MESSAGE(MPGStats) + +#include "messages/MPGStatsAck.h" +MESSAGE(MPGStatsAck) + +#include "messages/MPing.h" +MESSAGE(MPing) + +#include "messages/MPoolOp.h" +MESSAGE(MPoolOp) + +#include "messages/MPoolOpReply.h" +MESSAGE(MPoolOpReply) + +#include "messages/MRemoveSnaps.h" +MESSAGE(MRemoveSnaps) + +#include "messages/MRoute.h" +MESSAGE(MRoute) + +#include "messages/MServiceMap.h" +MESSAGE(MServiceMap) + +#include "messages/MStatfs.h" +MESSAGE(MStatfs) + +#include "messages/MStatfsReply.h" +MESSAGE(MStatfsReply) + +#include "messages/MTimeCheck.h" +MESSAGE(MTimeCheck) + +#include "messages/MTimeCheck2.h" +MESSAGE(MTimeCheck2) + +#include "messages/MWatchNotify.h" +MESSAGE(MWatchNotify) + +#include "messages/MMgrUpdate.h" +MESSAGE(MMgrUpdate) diff --git a/src/tools/ceph-dencoder/denc_plugin.h b/src/tools/ceph-dencoder/denc_plugin.h new file mode 100644 index 000000000..58690a4b9 --- /dev/null +++ b/src/tools/ceph-dencoder/denc_plugin.h @@ -0,0 +1,82 @@ +#include +#if __has_include() +#include +namespace fs = std::filesystem; +#else +#include +namespace fs = std::experimental::filesystem; +#endif +#include + +#include "denc_registry.h" + +class DencoderPlugin { + using dencoders_t = std::vector>; +public: + DencoderPlugin(const fs::path& path) { + mod = dlopen(path.c_str(), RTLD_NOW); + if (mod == nullptr) { + std::cerr << "failed to dlopen(" << path << "): " << dlerror() << std::endl; + } + } + DencoderPlugin(DencoderPlugin&& other) + : mod{other.mod}, + dencoders{std::move(other.dencoders)} + { + other.mod = nullptr; + other.dencoders.clear(); + } + ~DencoderPlugin() { +#if !defined(__FreeBSD__) + if (mod) { + dlclose(mod); + } +#endif + } + const dencoders_t& register_dencoders() { + static constexpr string_view REGISTER_DENCODERS_FUNCTION = "register_dencoders\0"; + + assert(mod); + using register_dencoders_t = void (*)(DencoderPlugin*); + const auto do_register = + reinterpret_cast(dlsym(mod, REGISTER_DENCODERS_FUNCTION.data())); + if (do_register == nullptr) { + std::cerr << "failed to dlsym(" << REGISTER_DENCODERS_FUNCTION << "): " + << dlerror() << std::endl; + return dencoders; + } + do_register(this); + return dencoders; + } + + bool good() const { + return mod != nullptr; + } + + void unregister_dencoders() { + while (!dencoders.empty()) { + delete dencoders.back().second; + dencoders.pop_back(); + } + } + template + void emplace(const char* name, Args&&...args) { + dencoders.emplace_back(name, new DencoderT(std::forward(args)...)); + } + +private: + void *mod = nullptr; + dencoders_t dencoders; +}; + +#define TYPE(t) plugin->emplace>(#t, false, false); +#define TYPE_STRAYDATA(t) plugin->emplace>(#t, true, false); +#define TYPE_NONDETERMINISTIC(t) plugin->emplace>(#t, false, true); +#define TYPE_FEATUREFUL(t) plugin->emplace>(#t, false, false); +#define TYPE_FEATUREFUL_STRAYDATA(t) plugin->emplace>(#t, true, false); +#define TYPE_FEATUREFUL_NONDETERMINISTIC(t) plugin->emplace>(#t, false, true); +#define TYPE_FEATUREFUL_NOCOPY(t) plugin->emplace>(#t, false, false); +#define TYPE_NOCOPY(t) plugin->emplace>(#t, false, false); +#define MESSAGE(t) plugin->emplace>(#t); + +#define DENC_API extern "C" [[gnu::visibility("default")]] diff --git a/src/tools/ceph-dencoder/denc_registry.h b/src/tools/ceph-dencoder/denc_registry.h new file mode 100644 index 000000000..dc1db36d3 --- /dev/null +++ b/src/tools/ceph-dencoder/denc_registry.h @@ -0,0 +1,241 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#pragma once + +#include +#include +#include + +#include "include/buffer_fwd.h" +#include "msg/Message.h" + +namespace ceph { + class Formatter; +} + +struct Dencoder { + virtual ~Dencoder() {} + virtual std::string decode(bufferlist bl, uint64_t seek) = 0; + virtual void encode(bufferlist& out, uint64_t features) = 0; + virtual void dump(ceph::Formatter *f) = 0; + virtual void copy() { + std::cerr << "copy operator= not supported" << std::endl; + } + virtual void copy_ctor() { + std::cerr << "copy ctor not supported" << std::endl; + } + virtual void generate() = 0; + virtual int num_generated() = 0; + virtual std::string select_generated(unsigned n) = 0; + virtual bool is_deterministic() = 0; + unsigned get_struct_v(bufferlist bl, uint64_t seek) const { + auto p = bl.cbegin(seek); + uint8_t struct_v = 0; + ceph::decode(struct_v, p); + return struct_v; + } + //virtual void print(ostream& out) = 0; +}; + +template +class DencoderBase : public Dencoder { +protected: + T* m_object; + list m_list; + bool stray_okay; + bool nondeterministic; + +public: + DencoderBase(bool stray_okay, bool nondeterministic) + : m_object(new T), + stray_okay(stray_okay), + nondeterministic(nondeterministic) {} + ~DencoderBase() override { + delete m_object; + } + + std::string decode(bufferlist bl, uint64_t seek) override { + auto p = bl.cbegin(); + p.seek(seek); + try { + using ceph::decode; + decode(*m_object, p); + } + catch (buffer::error& e) { + return e.what(); + } + if (!stray_okay && !p.end()) { + ostringstream ss; + ss << "stray data at end of buffer, offset " << p.get_off(); + return ss.str(); + } + return {}; + } + + void encode(bufferlist& out, uint64_t features) override = 0; + + void dump(ceph::Formatter *f) override { + m_object->dump(f); + } + void generate() override { + T::generate_test_instances(m_list); + } + int num_generated() override { + return m_list.size(); + } + string select_generated(unsigned i) override { + // allow 0- or 1-based (by wrapping) + if (i == 0) + i = m_list.size(); + if ((i == 0) || (i > m_list.size())) + return "invalid id for generated object"; + m_object = *(std::next(m_list.begin(), i-1)); + return string(); + } + + bool is_deterministic() override { + return !nondeterministic; + } +}; + +template +class DencoderImplNoFeatureNoCopy : public DencoderBase { +public: + DencoderImplNoFeatureNoCopy(bool stray_ok, bool nondeterministic) + : DencoderBase(stray_ok, nondeterministic) {} + void encode(bufferlist& out, uint64_t features) override { + out.clear(); + using ceph::encode; + encode(*this->m_object, out); + } +}; + +template +class DencoderImplNoFeature : public DencoderImplNoFeatureNoCopy { +public: + DencoderImplNoFeature(bool stray_ok, bool nondeterministic) + : DencoderImplNoFeatureNoCopy(stray_ok, nondeterministic) {} + void copy() override { + T *n = new T; + *n = *this->m_object; + delete this->m_object; + this->m_object = n; + } + void copy_ctor() override { + T *n = new T(*this->m_object); + delete this->m_object; + this->m_object = n; + } +}; + +template +class DencoderImplFeaturefulNoCopy : public DencoderBase { +public: + DencoderImplFeaturefulNoCopy(bool stray_ok, bool nondeterministic) + : DencoderBase(stray_ok, nondeterministic) {} + void encode(bufferlist& out, uint64_t features) override { + out.clear(); + using ceph::encode; + encode(*(this->m_object), out, features); + } +}; + +template +class DencoderImplFeatureful : public DencoderImplFeaturefulNoCopy { +public: + DencoderImplFeatureful(bool stray_ok, bool nondeterministic) + : DencoderImplFeaturefulNoCopy(stray_ok, nondeterministic) {} + void copy() override { + T *n = new T; + *n = *this->m_object; + delete this->m_object; + this->m_object = n; + } + void copy_ctor() override { + T *n = new T(*this->m_object); + delete this->m_object; + this->m_object = n; + } +}; + +template +class MessageDencoderImpl : public Dencoder { + ref_t m_object; + list> m_list; + +public: + MessageDencoderImpl() : m_object{make_message()} {} + ~MessageDencoderImpl() override {} + + string decode(bufferlist bl, uint64_t seek) override { + auto p = bl.cbegin(); + p.seek(seek); + try { + ref_t n(decode_message(g_ceph_context, 0, p), false); + if (!n) + throw std::runtime_error("failed to decode"); + if (n->get_type() != m_object->get_type()) { + stringstream ss; + ss << "decoded type " << n->get_type() << " instead of expected " << m_object->get_type(); + throw std::runtime_error(ss.str()); + } + m_object = ref_cast(n); + } + catch (buffer::error& e) { + return e.what(); + } + if (!p.end()) { + ostringstream ss; + ss << "stray data at end of buffer, offset " << p.get_off(); + return ss.str(); + } + return string(); + } + + void encode(bufferlist& out, uint64_t features) override { + out.clear(); + encode_message(m_object.get(), features, out); + } + + void dump(ceph::Formatter *f) override { + m_object->dump(f); + } + void generate() override { + //T::generate_test_instances(m_list); + } + int num_generated() override { + return m_list.size(); + } + string select_generated(unsigned i) override { + // allow 0- or 1-based (by wrapping) + if (i == 0) + i = m_list.size(); + if ((i == 0) || (i > m_list.size())) + return "invalid id for generated object"; + m_object = *(std::next(m_list.begin(), i-1)); + return string(); + } + bool is_deterministic() override { + return true; + } + + //void print(ostream& out) { + //out << m_object << std::endl; + //} +}; + +class DencoderRegistry +{ + using dencoders_t = std::map; + +public: + dencoders_t& get() { + return dencoders; + } + void register_dencoder(std::string_view name, Dencoder* denc) { + dencoders.emplace(name, denc); + } +private: + dencoders_t dencoders; +}; diff --git a/src/tools/ceph-dencoder/mds_types.cc b/src/tools/ceph-dencoder/mds_types.cc new file mode 100644 index 000000000..94280477a --- /dev/null +++ b/src/tools/ceph-dencoder/mds_types.cc @@ -0,0 +1,36 @@ +#include "acconfig.h" +#include +using namespace std; +#include "include/ceph_features.h" + +#define TYPE(t) +#define TYPE_STRAYDATA(t) +#define TYPE_NONDETERMINISTIC(t) +#define TYPE_FEATUREFUL(t) +#define TYPE_FEATUREFUL_STRAYDATA(t) +#define TYPE_FEATUREFUL_NONDETERMINISTIC(t) +#define TYPE_FEATUREFUL_NOCOPY(t) +#define TYPE_NOCOPY(t) +#define MESSAGE(t) +#include "mds_types.h" +#undef TYPE +#undef TYPE_STRAYDATA +#undef TYPE_NONDETERMINISTIC +#undef TYPE_NOCOPY +#undef TYPE_FEATUREFUL +#undef TYPE_FEATUREFUL_STRAYDATA +#undef TYPE_FEATUREFUL_NONDETERMINISTIC +#undef TYPE_FEATUREFUL_NOCOPY +#undef MESSAGE + +#include "denc_plugin.h" + +DENC_API void register_dencoders(DencoderPlugin* plugin) +{ +#include "mds_types.h" +} + +DENC_API void unregister_dencoders(DencoderPlugin* plugin) +{ + plugin->unregister_dencoders(); +} diff --git a/src/tools/ceph-dencoder/mds_types.h b/src/tools/ceph-dencoder/mds_types.h new file mode 100644 index 000000000..9406bf88b --- /dev/null +++ b/src/tools/ceph-dencoder/mds_types.h @@ -0,0 +1,111 @@ +#ifdef WITH_CEPHFS +#include "mds/JournalPointer.h" +TYPE(JournalPointer) + +#include "osdc/Journaler.h" +TYPE(Journaler::Header) + +#include "mds/snap.h" +TYPE(SnapInfo) +TYPE(snaplink_t) +TYPE(sr_t) + +#include "mds/mdstypes.h" +TYPE(frag_info_t) +TYPE(nest_info_t) +TYPE(quota_info_t) +TYPE(client_writeable_range_t) +TYPE_FEATUREFUL(inode_t) +TYPE_FEATUREFUL(old_inode_t) +TYPE(fnode_t) +TYPE(old_rstat_t) +TYPE_FEATUREFUL(session_info_t) +TYPE(string_snap_t) +TYPE(MDSCacheObjectInfo) +TYPE(mds_table_pending_t) +TYPE(cap_reconnect_t) +TYPE(inode_load_vec_t) +TYPE(dirfrag_load_vec_t) +TYPE(mds_load_t) +TYPE(MDSCacheObjectInfo) +TYPE(inode_backtrace_t) +TYPE(inode_backpointer_t) + +#include "mds/CInode.h" +TYPE_FEATUREFUL(InodeStore) +TYPE_FEATUREFUL(InodeStoreBare) + +#include "mds/MDSMap.h" +TYPE_FEATUREFUL(MDSMap) +TYPE_FEATUREFUL(MDSMap::mds_info_t) + +#include "mds/FSMap.h" +//TYPE_FEATUREFUL(Filesystem) +TYPE_FEATUREFUL(FSMap) + +#include "mds/Capability.h" +TYPE_NOCOPY(Capability) + +#include "mds/inode_backtrace.h" +TYPE(inode_backpointer_t) +TYPE(inode_backtrace_t) + +#include "mds/InoTable.h" +TYPE(InoTable) + +#include "mds/SnapServer.h" +TYPE_STRAYDATA(SnapServer) + +#include "mds/events/ECommitted.h" +TYPE_FEATUREFUL_NOCOPY(ECommitted) + +#include "mds/events/EExport.h" +TYPE_FEATUREFUL_NOCOPY(EExport) + +#include "mds/events/EFragment.h" +TYPE_FEATUREFUL_NOCOPY(EFragment) + +#include "mds/events/EImportFinish.h" +TYPE_FEATUREFUL_NOCOPY(EImportFinish) + +#include "mds/events/EImportStart.h" +TYPE_FEATUREFUL_NOCOPY(EImportStart) + +#include "mds/events/EMetaBlob.h" +TYPE_FEATUREFUL_NOCOPY(EMetaBlob::fullbit) +TYPE(EMetaBlob::remotebit) +TYPE(EMetaBlob::nullbit) +TYPE_FEATUREFUL_NOCOPY(EMetaBlob::dirlump) +TYPE_FEATUREFUL_NOCOPY(EMetaBlob) + +#include "mds/events/EOpen.h" +TYPE_FEATUREFUL_NOCOPY(EOpen) + +#include "mds/events/EResetJournal.h" +TYPE_FEATUREFUL_NOCOPY(EResetJournal) + +#include "mds/events/ESession.h" +TYPE_FEATUREFUL_NOCOPY(ESession) + +#include "mds/events/ESessions.h" +TYPE_FEATUREFUL_NOCOPY(ESessions) + +#include "mds/events/EPeerUpdate.h" +TYPE(link_rollback) +TYPE(rmdir_rollback) +TYPE(rename_rollback::drec) +TYPE(rename_rollback) +TYPE_FEATUREFUL_NOCOPY(EPeerUpdate) + +#include "mds/events/ESubtreeMap.h" +TYPE_FEATUREFUL_NOCOPY(ESubtreeMap) + +#include "mds/events/ETableClient.h" +TYPE_FEATUREFUL_NOCOPY(ETableClient) + +#include "mds/events/ETableServer.h" +TYPE_FEATUREFUL_NOCOPY(ETableServer) + +#include "mds/events/EUpdate.h" +TYPE_FEATUREFUL_NOCOPY(EUpdate) +#endif // WITH_CEPHFS diff --git a/src/tools/ceph-dencoder/osd_types.cc b/src/tools/ceph-dencoder/osd_types.cc new file mode 100644 index 000000000..13a90685b --- /dev/null +++ b/src/tools/ceph-dencoder/osd_types.cc @@ -0,0 +1,39 @@ +#include "acconfig.h" +#include +using namespace std; +#include "include/ceph_features.h" + +#define TYPE(t) +#define TYPE_STRAYDATA(t) +#define TYPE_NONDETERMINISTIC(t) +#define TYPE_FEATUREFUL(t) +#define TYPE_FEATUREFUL_STRAYDATA(t) +#define TYPE_FEATUREFUL_NONDETERMINISTIC(t) +#define TYPE_FEATUREFUL_NOCOPY(t) +#define TYPE_NOCOPY(t) +#define MESSAGE(t) +#include "osd_types.h" +#undef TYPE +#undef TYPE_STRAYDATA +#undef TYPE_NONDETERMINISTIC +#undef TYPE_NOCOPY +#undef TYPE_FEATUREFUL +#undef TYPE_FEATUREFUL_STRAYDATA +#undef TYPE_FEATUREFUL_NONDETERMINISTIC +#undef TYPE_FEATUREFUL_NOCOPY +#undef MESSAGE + +#include "denc_plugin.h" + +// cannot initialize dencoders when initializing static variables, as some of +// the types are allocated using mempool, and the mempools are initialized as +// static variables. +DENC_API void register_dencoders(DencoderPlugin* plugin) +{ +#include "osd_types.h" +} + +DENC_API void unregister_dencoders(DencoderPlugin* plugin) +{ + plugin->unregister_dencoders(); +} diff --git a/src/tools/ceph-dencoder/osd_types.h b/src/tools/ceph-dencoder/osd_types.h new file mode 100644 index 000000000..d60a7b5a8 --- /dev/null +++ b/src/tools/ceph-dencoder/osd_types.h @@ -0,0 +1,153 @@ +#include "osd/OSDMap.h" +TYPE(osd_info_t) +TYPE_FEATUREFUL(osd_xinfo_t) +TYPE_FEATUREFUL_NOCOPY(OSDMap) +TYPE_FEATUREFUL_STRAYDATA(OSDMap::Incremental) + +#include "osd/osd_types.h" +TYPE(osd_reqid_t) +TYPE(object_locator_t) +TYPE(request_redirect_t) +TYPE(pg_t) +TYPE(coll_t) +TYPE_FEATUREFUL(objectstore_perf_stat_t) +TYPE_FEATUREFUL(osd_stat_t) +TYPE(OSDSuperblock) +TYPE_FEATUREFUL(pool_snap_info_t) +TYPE_FEATUREFUL(pg_pool_t) +TYPE(object_stat_sum_t) +TYPE(object_stat_collection_t) +TYPE(pg_stat_t) +TYPE_FEATUREFUL(pool_stat_t) +TYPE(pg_hit_set_info_t) +TYPE(pg_hit_set_history_t) +TYPE(pg_history_t) +TYPE(pg_info_t) +TYPE(PastIntervals) +TYPE_FEATUREFUL(pg_query_t) +TYPE(ObjectModDesc) +TYPE(pg_log_entry_t) +TYPE(pg_log_dup_t) +TYPE(pg_log_t) +TYPE_FEATUREFUL(pg_missing_item) +TYPE_FEATUREFUL(pg_missing_t) +TYPE(pg_nls_response_t) +TYPE(pg_ls_response_t) +TYPE(object_copy_cursor_t) +TYPE_FEATUREFUL(object_copy_data_t) +TYPE(pg_create_t) +TYPE(OSDSuperblock) +TYPE(SnapSet) +TYPE_FEATUREFUL(watch_info_t) +TYPE_FEATUREFUL(watch_item_t) +TYPE(object_manifest_t) +TYPE_FEATUREFUL(object_info_t) +TYPE(SnapSet) +TYPE_FEATUREFUL(ObjectRecoveryInfo) +TYPE(ObjectRecoveryProgress) +TYPE(PushReplyOp) +TYPE_FEATUREFUL(PullOp) +TYPE_FEATUREFUL(PushOp) +TYPE(ScrubMap::object) +TYPE(ScrubMap) +TYPE_FEATUREFUL(obj_list_watch_response_t) +TYPE(clone_info) +TYPE(obj_list_snap_response_t) +TYPE(pool_pg_num_history_t) + +#include "osd/ECUtil.h" +// TYPE(stripe_info_t) non-standard encoding/decoding functions +TYPE(ECUtil::HashInfo) + +#include "osd/ECMsgTypes.h" +TYPE_NOCOPY(ECSubWrite) +TYPE(ECSubWriteReply) +TYPE_FEATUREFUL(ECSubRead) +TYPE(ECSubReadReply) + +#include "osd/HitSet.h" +TYPE_NONDETERMINISTIC(ExplicitHashHitSet) +TYPE_NONDETERMINISTIC(ExplicitObjectHitSet) +TYPE(BloomHitSet) +TYPE_NONDETERMINISTIC(HitSet) // because some subclasses are +TYPE(HitSet::Params) + +#include "os/ObjectStore.h" +TYPE(ObjectStore::Transaction) + +#include "os/filestore/SequencerPosition.h" +TYPE(SequencerPosition) + +#ifdef WITH_BLUESTORE +#include "os/bluestore/bluestore_types.h" +TYPE(bluestore_bdev_label_t) +TYPE(bluestore_cnode_t) +TYPE(bluestore_compression_header_t) +TYPE(bluestore_extent_ref_map_t) +TYPE(bluestore_pextent_t) +TYPE(bluestore_blob_use_tracker_t) +// TODO: bluestore_blob_t repurposes the "feature" param of encode() for its +// struct_v. at a higher level, BlueStore::ExtentMap encodes the extends using +// a different interface than the normal ones. see +// BlueStore::ExtentMap::encode_some(). maybe we can test it using another +// approach. +// TYPE_FEATUREFUL(bluestore_blob_t) +// TYPE(bluestore_shared_blob_t) there is no encode here +TYPE(bluestore_onode_t) +TYPE(bluestore_deferred_op_t) +TYPE(bluestore_deferred_transaction_t) +// TYPE(bluestore_compression_header_t) there is no encode here + +#include "os/bluestore/bluefs_types.h" +TYPE(bluefs_extent_t) +TYPE(bluefs_fnode_t) +TYPE(bluefs_super_t) +TYPE(bluefs_transaction_t) +#endif + +#include "mon/AuthMonitor.h" +TYPE_FEATUREFUL(AuthMonitor::Incremental) + +#include "mon/PGMap.h" +TYPE_FEATUREFUL_NONDETERMINISTIC(PGMapDigest) +TYPE_FEATUREFUL_NONDETERMINISTIC(PGMap) + +#include "mon/MonitorDBStore.h" +TYPE(MonitorDBStore::Transaction) +TYPE(MonitorDBStore::Op) + +#include "mon/MonMap.h" +TYPE_FEATUREFUL(MonMap) + +#include "mon/MonCap.h" +TYPE(MonCap) + +#include "mon/MgrMap.h" +TYPE_FEATUREFUL(MgrMap) + +#include "mon/mon_types.h" +TYPE(LevelDBStoreStats) +TYPE(ScrubResult) + +#include "mon/CreatingPGs.h" +TYPE_FEATUREFUL(creating_pgs_t) + +#include "mgr/ServiceMap.h" +TYPE_FEATUREFUL(ServiceMap) +TYPE_FEATUREFUL(ServiceMap::Service) +TYPE_FEATUREFUL(ServiceMap::Daemon) + +#include "mon/ConnectionTracker.h" +TYPE(ConnectionReport); +TYPE(ConnectionTracker); + +#include "os/filestore/DBObjectMap.h" +TYPE(DBObjectMap::_Header) +TYPE(DBObjectMap::State) + +#include "os/filestore/FileStore.h" +TYPE(FSSuperblock) + +#include "os/kstore/kstore_types.h" +TYPE(kstore_cnode_t) +TYPE(kstore_onode_t) diff --git a/src/tools/ceph-dencoder/rbd_types.cc b/src/tools/ceph-dencoder/rbd_types.cc new file mode 100644 index 000000000..e04efc30d --- /dev/null +++ b/src/tools/ceph-dencoder/rbd_types.cc @@ -0,0 +1,36 @@ +#include "acconfig.h" +#include +using namespace std; +#include "include/ceph_features.h" + +#define TYPE(t) +#define TYPE_STRAYDATA(t) +#define TYPE_NONDETERMINISTIC(t) +#define TYPE_FEATUREFUL(t) +#define TYPE_FEATUREFUL_STRAYDATA(t) +#define TYPE_FEATUREFUL_NONDETERMINISTIC(t) +#define TYPE_FEATUREFUL_NOCOPY(t) +#define TYPE_NOCOPY(t) +#define MESSAGE(t) +#include "rbd_types.h" +#undef TYPE +#undef TYPE_STRAYDATA +#undef TYPE_NONDETERMINISTIC +#undef TYPE_NOCOPY +#undef TYPE_FEATUREFUL +#undef TYPE_FEATUREFUL_STRAYDATA +#undef TYPE_FEATUREFUL_NONDETERMINISTIC +#undef TYPE_FEATUREFUL_NOCOPY +#undef MESSAGE + +#include "denc_plugin.h" + +DENC_API void register_dencoders(DencoderPlugin* plugin) +{ +#include "rbd_types.h" +} + +DENC_API void unregister_dencoders(DencoderPlugin* plugin) +{ + plugin->unregister_dencoders(); +} diff --git a/src/tools/ceph-dencoder/rbd_types.h b/src/tools/ceph-dencoder/rbd_types.h new file mode 100644 index 000000000..6fb84dea6 --- /dev/null +++ b/src/tools/ceph-dencoder/rbd_types.h @@ -0,0 +1,52 @@ +#ifdef WITH_RBD +#include "librbd/journal/Types.h" +TYPE(librbd::journal::EventEntry) +TYPE(librbd::journal::ClientData) +TYPE(librbd::journal::TagData) +#include "librbd/mirroring_watcher/Types.h" +TYPE(librbd::mirroring_watcher::NotifyMessage) +#include "librbd/trash_watcher/Types.h" +TYPE(librbd::mirroring_watcher::NotifyMessage) +#include "librbd/WatchNotifyTypes.h" +TYPE_NOCOPY(librbd::watch_notify::NotifyMessage) +TYPE(librbd::watch_notify::ResponseMessage) + +#include "rbd_replay/ActionTypes.h" +TYPE(rbd_replay::action::Dependency) +TYPE(rbd_replay::action::ActionEntry) + +#include "tools/rbd_mirror/image_map/Types.h" +TYPE(rbd::mirror::image_map::PolicyData) +#endif + +#if defined(WITH_RBD) && defined(WITH_RBD_SSD_CACHE) +#include "librbd/cache/pwl/Types.h" +#include "librbd/cache/pwl/ssd/Types.h" +TYPE(librbd::cache::pwl::WriteLogCacheEntry) +TYPE(librbd::cache::pwl::WriteLogPoolRoot) +TYPE(librbd::cache::pwl::ssd::SuperBlock) +#endif + +#ifdef WITH_RBD +#include "cls/rbd/cls_rbd.h" +TYPE_FEATUREFUL(cls_rbd_parent) +TYPE_FEATUREFUL(cls_rbd_snap) + +#include "cls/rbd/cls_rbd_types.h" +TYPE(cls::rbd::ParentImageSpec) +TYPE(cls::rbd::ChildImageSpec) +TYPE(cls::rbd::MigrationSpec) +TYPE(cls::rbd::MirrorPeer) +TYPE(cls::rbd::MirrorImage) +TYPE(cls::rbd::MirrorImageMap) +TYPE(cls::rbd::MirrorImageStatus) +TYPE(cls::rbd::MirrorImageSiteStatus) +TYPE_FEATUREFUL(cls::rbd::MirrorImageSiteStatusOnDisk) +TYPE(cls::rbd::GroupImageSpec) +TYPE(cls::rbd::GroupImageStatus) +TYPE(cls::rbd::GroupSnapshot) +TYPE(cls::rbd::GroupSpec) +TYPE(cls::rbd::ImageSnapshotSpec) +TYPE(cls::rbd::SnapshotInfo) +TYPE(cls::rbd::SnapshotNamespace) +#endif diff --git a/src/tools/ceph-dencoder/rgw_types.cc b/src/tools/ceph-dencoder/rgw_types.cc new file mode 100644 index 000000000..79688b534 --- /dev/null +++ b/src/tools/ceph-dencoder/rgw_types.cc @@ -0,0 +1,36 @@ +#include "acconfig.h" +#include +using namespace std; +#include "include/ceph_features.h" + +#define TYPE(t) +#define TYPE_STRAYDATA(t) +#define TYPE_NONDETERMINISTIC(t) +#define TYPE_FEATUREFUL(t) +#define TYPE_FEATUREFUL_STRAYDATA(t) +#define TYPE_FEATUREFUL_NONDETERMINISTIC(t) +#define TYPE_FEATUREFUL_NOCOPY(t) +#define TYPE_NOCOPY(t) +#define MESSAGE(t) +#include "rgw_types.h" +#undef TYPE +#undef TYPE_STRAYDATA +#undef TYPE_NONDETERMINISTIC +#undef TYPE_NOCOPY +#undef TYPE_FEATUREFUL +#undef TYPE_FEATUREFUL_STRAYDATA +#undef TYPE_FEATUREFUL_NONDETERMINISTIC +#undef TYPE_FEATUREFUL_NOCOPY +#undef MESSAGE + +#include "denc_plugin.h" + +DENC_API void register_dencoders(DencoderPlugin* plugin) +{ +#include "rgw_types.h" +} + +DENC_API void unregister_dencoders(DencoderPlugin* plugin) +{ + plugin->unregister_dencoders(); +} diff --git a/src/tools/ceph-dencoder/rgw_types.h b/src/tools/ceph-dencoder/rgw_types.h new file mode 100644 index 000000000..bd1443ddf --- /dev/null +++ b/src/tools/ceph-dencoder/rgw_types.h @@ -0,0 +1,131 @@ +#ifdef WITH_RADOSGW + +#include "rgw/rgw_rados.h" +TYPE(RGWOLHInfo) +TYPE(RGWObjManifestPart) +TYPE(RGWObjManifest) +TYPE(objexp_hint_entry) + +#include "rgw/rgw_zone.h" +TYPE(RGWZoneParams) +TYPE(RGWZone) +TYPE(RGWZoneGroup) +TYPE(RGWRealm) +TYPE(RGWPeriod) +TYPE(RGWPeriodLatestEpochInfo) + +#include "rgw/rgw_acl.h" +TYPE(ACLPermission) +TYPE(ACLGranteeType) +TYPE(ACLGrant) +TYPE(RGWAccessControlList) +TYPE(ACLOwner) +TYPE(RGWAccessControlPolicy) + +#include "rgw/rgw_cache.h" +TYPE(ObjectMetaInfo) +TYPE(ObjectCacheInfo) +TYPE(RGWCacheNotifyInfo) + +#include "rgw/rgw_lc.h" +TYPE(RGWLifecycleConfiguration) + +#include "cls/rgw/cls_rgw_types.h" +TYPE(rgw_bucket_pending_info) +TYPE(rgw_bucket_dir_entry_meta) +TYPE(rgw_bucket_entry_ver) +TYPE(rgw_bucket_dir_entry) +TYPE(rgw_bucket_category_stats) +TYPE(rgw_bucket_dir_header) +TYPE(rgw_bucket_dir) +TYPE(rgw_bucket_entry_ver) +TYPE(cls_rgw_obj_key) +TYPE(rgw_bucket_olh_log_entry) +TYPE(rgw_usage_log_entry) + +#include "cls/rgw/cls_rgw_ops.h" +TYPE(rgw_cls_obj_prepare_op) +TYPE(rgw_cls_obj_complete_op) +TYPE(rgw_cls_list_op) +TYPE(rgw_cls_list_ret) +TYPE(cls_rgw_gc_defer_entry_op) +TYPE(cls_rgw_gc_list_op) +TYPE(cls_rgw_gc_list_ret) +TYPE(cls_rgw_gc_obj_info) +TYPE(cls_rgw_gc_remove_op) +TYPE(cls_rgw_gc_set_entry_op) +TYPE(cls_rgw_obj) +TYPE(cls_rgw_obj_chain) +TYPE(rgw_cls_tag_timeout_op) +TYPE(cls_rgw_bi_log_list_op) +TYPE(cls_rgw_bi_log_trim_op) +TYPE(cls_rgw_bi_log_list_ret) +TYPE(rgw_cls_link_olh_op) +TYPE(rgw_cls_unlink_instance_op) +TYPE(rgw_cls_read_olh_log_op) +TYPE(rgw_cls_read_olh_log_ret) +TYPE(rgw_cls_trim_olh_log_op) +TYPE(rgw_cls_bucket_clear_olh_op) +TYPE(rgw_cls_check_index_ret) +TYPE(cls_rgw_reshard_add_op) +TYPE(cls_rgw_reshard_list_op) +TYPE(cls_rgw_reshard_list_ret) +TYPE(cls_rgw_reshard_get_op) +TYPE(cls_rgw_reshard_get_ret) +TYPE(cls_rgw_reshard_remove_op) +TYPE(cls_rgw_set_bucket_resharding_op) +TYPE(cls_rgw_clear_bucket_resharding_op) +TYPE(cls_rgw_lc_obj_head) + +#include "cls/rgw/cls_rgw_client.h" +TYPE(rgw_bi_log_entry) +TYPE(cls_rgw_reshard_entry) +TYPE(cls_rgw_bucket_instance_entry) + +#include "cls/user/cls_user_types.h" +TYPE(cls_user_bucket) +TYPE(cls_user_bucket_entry) +TYPE(cls_user_stats) +TYPE(cls_user_header) + +#include "cls/user/cls_user_ops.h" +TYPE(cls_user_set_buckets_op) +TYPE(cls_user_remove_bucket_op) +TYPE(cls_user_list_buckets_op) +TYPE(cls_user_list_buckets_ret) +TYPE(cls_user_get_header_op) +TYPE(cls_user_get_header_ret) +TYPE(cls_user_complete_stats_sync_op) + +#include "cls/journal/cls_journal_types.h" +TYPE(cls::journal::ObjectPosition) +TYPE(cls::journal::ObjectSetPosition) +TYPE(cls::journal::Client) +TYPE(cls::journal::Tag) + +#include "rgw/rgw_common.h" +TYPE(RGWAccessKey) +TYPE(RGWSubUser) +TYPE(RGWUserInfo) +TYPE(rgw_bucket) +TYPE(RGWBucketInfo) +TYPE(RGWBucketEnt) +TYPE(rgw_obj) + +#include "rgw/rgw_log.h" +TYPE(rgw_log_entry) + +#include "rgw/rgw_meta_sync_status.h" +TYPE(rgw_meta_sync_info) +TYPE(rgw_meta_sync_marker) +TYPE(rgw_meta_sync_status) + +#include "rgw/rgw_multi.h" +TYPE(RGWUploadPartInfo) + +#include "rgw/rgw_data_sync.h" +TYPE(rgw_data_sync_info) +TYPE(rgw_data_sync_marker) +TYPE(rgw_data_sync_status) + +#endif diff --git a/src/tools/ceph-dencoder/sstring.h b/src/tools/ceph-dencoder/sstring.h new file mode 100644 index 000000000..c2493c10e --- /dev/null +++ b/src/tools/ceph-dencoder/sstring.h @@ -0,0 +1,40 @@ +#ifndef TEST_SSTRING_H +#define TEST_SSTRING_H + +#include "common/sstring.hh" + +// wrapper for sstring that implements the dencoder interface +class sstring_wrapper { + using sstring16 = basic_sstring; + sstring16 s1; + using sstring24 = basic_sstring; + sstring24 s2; + public: + sstring_wrapper() = default; + sstring_wrapper(sstring16&& s1, sstring24&& s2) + : s1(std::move(s1)), s2(std::move(s2)) + {} + + DENC(sstring_wrapper, w, p) { + DENC_START(1, 1, p); + denc(w.s1, p); + denc(w.s2, p); + DENC_FINISH(p); + } + void dump(Formatter* f) { + f->dump_string("s1", s1.c_str()); + f->dump_string("s2", reinterpret_cast(s2.c_str())); + } + static void generate_test_instances(std::list& ls) { + ls.push_back(new sstring_wrapper()); + // initialize sstrings that fit in internal storage + constexpr auto cstr6 = "abcdef"; + ls.push_back(new sstring_wrapper(sstring16{cstr6}, sstring24{cstr6})); + // initialize sstrings that overflow into external storage + constexpr auto cstr26 = "abcdefghijklmnopqrstuvwxyz"; + ls.push_back(new sstring_wrapper(sstring16{cstr26}, sstring24{cstr26})); + } +}; +WRITE_CLASS_DENC(sstring_wrapper) + +#endif -- cgit v1.2.3