summaryrefslogtreecommitdiffstats
path: root/src/test/erasure-code
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-21 11:54:28 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-21 11:54:28 +0000
commite6918187568dbd01842d8d1d2c808ce16a894239 (patch)
tree64f88b554b444a49f656b6c656111a145cbbaa28 /src/test/erasure-code
parentInitial commit. (diff)
downloadceph-upstream/18.2.2.tar.xz
ceph-upstream/18.2.2.zip
Adding upstream version 18.2.2.upstream/18.2.2
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'src/test/erasure-code')
-rw-r--r--src/test/erasure-code/CMakeLists.txt259
-rw-r--r--src/test/erasure-code/ErasureCodeExample.h195
-rw-r--r--src/test/erasure-code/ErasureCodePluginExample.cc45
-rw-r--r--src/test/erasure-code/ErasureCodePluginFailToInitialize.cc26
-rw-r--r--src/test/erasure-code/ErasureCodePluginFailToRegister.cc25
-rw-r--r--src/test/erasure-code/ErasureCodePluginHangs.cc27
-rw-r--r--src/test/erasure-code/ErasureCodePluginMissingEntryPoint.cc6
-rw-r--r--src/test/erasure-code/ErasureCodePluginMissingVersion.cc3
-rw-r--r--src/test/erasure-code/TestErasureCode.cc169
-rw-r--r--src/test/erasure-code/TestErasureCodeClay.cc596
-rw-r--r--src/test/erasure-code/TestErasureCodeExample.cc248
-rw-r--r--src/test/erasure-code/TestErasureCodeIsa.cc967
-rw-r--r--src/test/erasure-code/TestErasureCodeJerasure.cc370
-rw-r--r--src/test/erasure-code/TestErasureCodeLrc.cc925
-rw-r--r--src/test/erasure-code/TestErasureCodePlugin.cc131
-rw-r--r--src/test/erasure-code/TestErasureCodePluginClay.cc114
-rw-r--r--src/test/erasure-code/TestErasureCodePluginIsa.cc62
-rw-r--r--src/test/erasure-code/TestErasureCodePluginJerasure.cc71
-rw-r--r--src/test/erasure-code/TestErasureCodePluginLrc.cc50
-rw-r--r--src/test/erasure-code/TestErasureCodePluginShec.cc65
-rw-r--r--src/test/erasure-code/TestErasureCodeShec.cc2823
-rw-r--r--src/test/erasure-code/TestErasureCodeShec_all.cc332
-rw-r--r--src/test/erasure-code/TestErasureCodeShec_arguments.cc370
-rw-r--r--src/test/erasure-code/TestErasureCodeShec_thread.cc219
-rw-r--r--src/test/erasure-code/ceph_erasure_code_benchmark.cc354
-rw-r--r--src/test/erasure-code/ceph_erasure_code_benchmark.h62
-rw-r--r--src/test/erasure-code/ceph_erasure_code_non_regression.cc327
27 files changed, 8841 insertions, 0 deletions
diff --git a/src/test/erasure-code/CMakeLists.txt b/src/test/erasure-code/CMakeLists.txt
new file mode 100644
index 000000000..ab7328eae
--- /dev/null
+++ b/src/test/erasure-code/CMakeLists.txt
@@ -0,0 +1,259 @@
+
+add_executable(ceph_erasure_code_benchmark
+ ${CMAKE_SOURCE_DIR}/src/erasure-code/ErasureCode.cc
+ ceph_erasure_code_benchmark.cc)
+target_link_libraries(ceph_erasure_code_benchmark ceph-common Boost::program_options global ${CMAKE_DL_LIBS})
+install(TARGETS ceph_erasure_code_benchmark
+ DESTINATION bin)
+
+add_executable(ceph_erasure_code_non_regression ceph_erasure_code_non_regression.cc)
+target_link_libraries(ceph_erasure_code_non_regression ceph-common Boost::program_options global ${CMAKE_DL_LIBS})
+
+add_library(ec_example SHARED
+ ErasureCodePluginExample.cc
+ $<TARGET_OBJECTS:erasure_code_objs>)
+target_link_libraries(ec_example pthread ${EXTRALIBS})
+
+add_library(ec_missing_entry_point SHARED ErasureCodePluginMissingEntryPoint.cc)
+target_link_libraries(ec_missing_entry_point pthread ${EXTRALIBS})
+
+add_library(ec_missing_version SHARED ErasureCodePluginMissingVersion.cc)
+target_link_libraries(ec_missing_version pthread ${EXTRALIBS})
+
+add_library(ec_hangs SHARED ErasureCodePluginHangs.cc)
+target_link_libraries(ec_hangs pthread ${EXTRALIBS})
+
+add_library(ec_fail_to_initialize SHARED ErasureCodePluginFailToInitialize.cc)
+target_link_libraries(ec_fail_to_initialize pthread ${EXTRALIBS})
+
+add_library(ec_fail_to_register SHARED ErasureCodePluginFailToRegister.cc)
+target_link_libraries(ec_fail_to_register pthread ${EXTRALIBS})
+
+# unittest_erasure_code_plugin
+add_executable(unittest_erasure_code_plugin
+ ${CMAKE_SOURCE_DIR}/src/erasure-code/ErasureCode.cc
+ TestErasureCodePlugin.cc
+ $<TARGET_OBJECTS:unit-main>
+ )
+add_ceph_unittest(unittest_erasure_code_plugin)
+target_link_libraries(unittest_erasure_code_plugin
+ global
+ ${CMAKE_DL_LIBS}
+ ec_example
+ ceph-common
+ )
+add_dependencies(unittest_erasure_code_plugin
+ ec_example
+ ec_missing_entry_point
+ ec_missing_version
+ ec_hangs
+ ec_fail_to_initialize
+ ec_fail_to_register)
+
+# unittest_erasure_code
+add_executable(unittest_erasure_code
+ ${CMAKE_SOURCE_DIR}/src/erasure-code/ErasureCode.cc
+ TestErasureCode.cc
+ $<TARGET_OBJECTS:unit-main>
+ )
+add_ceph_unittest(unittest_erasure_code)
+target_link_libraries(unittest_erasure_code
+ global
+ ceph-common
+ )
+
+# unittest_erasure_code_plugin_jerasure
+add_executable(unittest_erasure_code_plugin_jerasure
+ TestErasureCodePluginJerasure.cc
+ $<TARGET_OBJECTS:unit-main>
+ )
+add_ceph_unittest(unittest_erasure_code_plugin_jerasure)
+target_link_libraries(unittest_erasure_code_plugin_jerasure
+ global
+ ceph-common)
+add_dependencies(unittest_erasure_code_plugin_jerasure
+ ec_jerasure)
+
+if(WITH_EC_ISA_PLUGIN)
+
+#unittest_erasure_code_isa
+add_executable(unittest_erasure_code_isa
+ ${CMAKE_SOURCE_DIR}/src/erasure-code/ErasureCode.cc
+ TestErasureCodeIsa.cc
+ $<TARGET_OBJECTS:unit-main>
+ )
+add_ceph_unittest(unittest_erasure_code_isa)
+target_link_libraries(unittest_erasure_code_isa
+ global
+ ceph-common
+ ec_isa
+ erasure_code
+ )
+
+#unittest_erasure_code_plugin_isa
+add_executable(unittest_erasure_code_plugin_isa
+ ${CMAKE_SOURCE_DIR}/src/erasure-code/ErasureCode.cc
+ TestErasureCodePluginIsa.cc
+ $<TARGET_OBJECTS:unit-main>
+ )
+add_ceph_unittest(unittest_erasure_code_plugin_isa)
+target_link_libraries(unittest_erasure_code_plugin_isa
+ global
+ ceph-common
+ ${CMAKE_DL_LIBS}
+ erasure_code
+ )
+add_dependencies(unittest_erasure_code_plugin_isa
+ ec_isa)
+endif(WITH_EC_ISA_PLUGIN)
+
+# unittest_erasure_code_lrc
+add_executable(unittest_erasure_code_lrc
+ TestErasureCodeLrc.cc
+ $<TARGET_OBJECTS:unit-main>)
+add_ceph_unittest(unittest_erasure_code_lrc)
+target_link_libraries(unittest_erasure_code_lrc
+ global
+ ${CMAKE_DL_LIBS}
+ ec_lrc
+ ceph-common
+ )
+
+# unittest_erasure_code_plugin_lrc
+add_executable(unittest_erasure_code_plugin_lrc
+ TestErasureCodePluginLrc.cc
+ $<TARGET_OBJECTS:unit-main>
+ )
+add_ceph_unittest(unittest_erasure_code_plugin_lrc)
+add_dependencies(unittest_erasure_code_plugin_lrc
+ ec_lrc
+ ec_jerasure)
+target_link_libraries(unittest_erasure_code_plugin_lrc
+ global
+ ${CMAKE_DL_LIBS}
+ ceph-common)
+
+# unittest_erasure_code_plugin_shec
+add_executable(unittest_erasure_code_plugin_shec
+ TestErasureCodePluginShec.cc
+ $<TARGET_OBJECTS:unit-main>
+ )
+add_ceph_unittest(unittest_erasure_code_plugin_shec)
+target_link_libraries(unittest_erasure_code_plugin_shec
+ global
+ ${CMAKE_DL_LIBS}
+ ceph-common)
+add_dependencies(unittest_erasure_code_plugin_shec
+ ec_shec)
+
+# unittest_erasure_code_example
+add_executable(unittest_erasure_code_example
+ ${CMAKE_SOURCE_DIR}/src/erasure-code/ErasureCode.cc
+ TestErasureCodeExample.cc
+ $<TARGET_OBJECTS:unit-main>
+)
+add_ceph_unittest(unittest_erasure_code_example)
+target_link_libraries(unittest_erasure_code_example
+ global
+ ${CMAKE_DL_LIBS}
+ ceph-common
+ erasure_code
+ ${UNITTEST_LIBS}
+ )
+
+include_directories(SYSTEM ${CMAKE_SOURCE_DIR}/src/erasure-code/jerasure/jerasure/include)
+include_directories(SYSTEM ${CMAKE_SOURCE_DIR}/src/erasure-code//jerasure/gf-complete/include)
+
+# unittest_erasure_code_jerasure
+add_executable(unittest_erasure_code_jerasure
+ TestErasureCodeJerasure.cc
+ $<TARGET_OBJECTS:unit-main>
+ )
+add_ceph_unittest(unittest_erasure_code_jerasure)
+target_link_libraries(unittest_erasure_code_jerasure
+ global
+ ceph-common
+ ec_jerasure
+ )
+
+include_directories(${CMAKE_SOURCE_DIR}/src/erasure-code/jerasure)
+include_directories(${CMAKE_SOURCE_DIR}/src/erasure-code/shec)
+
+# unittest_erasure_code_shec
+add_executable(unittest_erasure_code_shec
+ TestErasureCodeShec.cc
+ $<TARGET_OBJECTS:unit-main>
+ )
+add_ceph_unittest(unittest_erasure_code_shec)
+target_link_libraries(unittest_erasure_code_shec
+ global
+ ${CMAKE_DL_LIBS}
+ ceph-common
+ ec_shec
+ )
+
+# unittest_erasure_code_shec_all
+add_executable(unittest_erasure_code_shec_all
+ TestErasureCodeShec_all.cc
+ )
+add_ceph_unittest(unittest_erasure_code_shec_all PARALLEL)
+target_link_libraries(unittest_erasure_code_shec_all
+ global
+ ${CMAKE_DL_LIBS}
+ ceph-common
+ ec_shec
+ )
+
+# unittest_erasure_code_shec_thread
+add_executable(unittest_erasure_code_shec_thread
+ TestErasureCodeShec_thread.cc
+ $<TARGET_OBJECTS:unit-main>
+ )
+add_ceph_unittest(unittest_erasure_code_shec_thread)
+target_link_libraries(unittest_erasure_code_shec_thread
+ global
+ ${CMAKE_DL_LIBS}
+ ceph-common
+ ec_shec
+ )
+
+
+# unittest_erasure_code_shec_arguments
+add_executable(unittest_erasure_code_shec_arguments
+ TestErasureCodeShec_arguments.cc
+ )
+add_ceph_unittest(unittest_erasure_code_shec_arguments)
+target_link_libraries(unittest_erasure_code_shec_arguments
+ global
+ ${CMAKE_DL_LIBS}
+ ceph-common
+ ec_shec
+ )
+
+#unitest_erasure_code_clay
+add_executable(unittest_erasure_code_clay
+ TestErasureCodeClay.cc
+ $<TARGET_OBJECTS:unit-main>)
+add_ceph_unittest(unittest_erasure_code_clay)
+target_link_libraries(unittest_erasure_code_clay
+ global
+ ${CMAKE_DL_LIBS}
+ ${UNITTEST_LIBS}
+ ceph-common
+ ec_clay
+ )
+
+# unittest_erasure_code_plugin_clay
+add_executable(unittest_erasure_code_plugin_clay
+ TestErasureCodePluginClay.cc
+ $<TARGET_OBJECTS:unit-main>)
+add_ceph_unittest(unittest_erasure_code_plugin_clay)
+add_dependencies(unittest_erasure_code_plugin_clay
+ ec_clay)
+target_link_libraries(unittest_erasure_code_plugin_clay
+ GTest::Main
+ global
+ ${CMAKE_DL_LIBS}
+ ${UNITTEST_LIBS}
+ ceph-common)
+
diff --git a/src/test/erasure-code/ErasureCodeExample.h b/src/test/erasure-code/ErasureCodeExample.h
new file mode 100644
index 000000000..4226361c4
--- /dev/null
+++ b/src/test/erasure-code/ErasureCodeExample.h
@@ -0,0 +1,195 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph distributed storage system
+ *
+ * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com>
+ *
+ * Author: Loic Dachary <loic@dachary.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#ifndef CEPH_ERASURE_CODE_EXAMPLE_H
+#define CEPH_ERASURE_CODE_EXAMPLE_H
+
+#include <unistd.h>
+#include <errno.h>
+#include <algorithm>
+#include <sstream>
+
+#include "crush/CrushWrapper.h"
+#include "osd/osd_types.h"
+#include "erasure-code/ErasureCode.h"
+
+#define FIRST_DATA_CHUNK 0
+#define SECOND_DATA_CHUNK 1
+#define DATA_CHUNKS 2u
+
+#define CODING_CHUNK 2
+#define CODING_CHUNKS 1u
+
+#define MINIMUM_TO_RECOVER 2u
+
+class ErasureCodeExample final : public ErasureCode {
+public:
+ ~ErasureCodeExample() override {}
+
+ int create_rule(const std::string &name,
+ CrushWrapper &crush,
+ std::ostream *ss) const override {
+ return crush.add_simple_rule(name, "default", "host", "",
+ "indep", pg_pool_t::TYPE_ERASURE, ss);
+ }
+
+ int minimum_to_decode_with_cost(const std::set<int> &want_to_read,
+ const std::map<int, int> &available,
+ std::set<int> *minimum) override {
+ //
+ // If one chunk is more expensive to fetch than the others,
+ // recover it instead. For instance, if the cost reflects the
+ // time it takes for a chunk to be retrieved from a remote
+ // OSD and if CPU is cheap, it could make sense to recover
+ // instead of fetching the chunk.
+ //
+ std::map<int, int> c2c(available);
+ if (c2c.size() > DATA_CHUNKS) {
+ if (c2c[FIRST_DATA_CHUNK] > c2c[SECOND_DATA_CHUNK] &&
+ c2c[FIRST_DATA_CHUNK] > c2c[CODING_CHUNK])
+ c2c.erase(FIRST_DATA_CHUNK);
+ else if(c2c[SECOND_DATA_CHUNK] > c2c[FIRST_DATA_CHUNK] &&
+ c2c[SECOND_DATA_CHUNK] > c2c[CODING_CHUNK])
+ c2c.erase(SECOND_DATA_CHUNK);
+ else if(c2c[CODING_CHUNK] > c2c[FIRST_DATA_CHUNK] &&
+ c2c[CODING_CHUNK] > c2c[SECOND_DATA_CHUNK])
+ c2c.erase(CODING_CHUNK);
+ }
+ std::set <int> available_chunks;
+ for (std::map<int, int>::const_iterator i = c2c.begin();
+ i != c2c.end();
+ ++i)
+ available_chunks.insert(i->first);
+ return _minimum_to_decode(want_to_read, available_chunks, minimum);
+ }
+
+ unsigned int get_chunk_count() const override {
+ return DATA_CHUNKS + CODING_CHUNKS;
+ }
+
+ unsigned int get_data_chunk_count() const override {
+ return DATA_CHUNKS;
+ }
+
+ unsigned int get_chunk_size(unsigned int object_size) const override {
+ return ( object_size / DATA_CHUNKS ) + 1;
+ }
+
+ int encode(const std::set<int> &want_to_encode,
+ const bufferlist &in,
+ std::map<int, bufferlist> *encoded) override {
+ //
+ // make sure all data chunks have the same length, allocating
+ // padding if necessary.
+ //
+ unsigned int chunk_length = get_chunk_size(in.length());
+ bufferlist out(in);
+ unsigned int width = get_chunk_count() * get_chunk_size(in.length());
+ bufferptr pad(width - in.length());
+ pad.zero(0, get_data_chunk_count());
+ out.push_back(pad);
+ //
+ // compute the coding chunk with first chunk ^ second chunk
+ //
+ char *p = out.c_str();
+ for (unsigned i = 0; i < chunk_length; i++)
+ p[i + CODING_CHUNK * chunk_length] =
+ p[i + FIRST_DATA_CHUNK * chunk_length] ^
+ p[i + SECOND_DATA_CHUNK * chunk_length];
+ //
+ // populate the bufferlist with bufferptr pointing
+ // to chunk boundaries
+ //
+ const bufferptr &ptr = out.front();
+ for (auto j = want_to_encode.begin();
+ j != want_to_encode.end();
+ ++j) {
+ bufferlist tmp;
+ bufferptr chunk(ptr, (*j) * chunk_length, chunk_length);
+ tmp.push_back(chunk);
+ tmp.claim_append((*encoded)[*j]);
+ (*encoded)[*j].swap(tmp);
+ }
+ return 0;
+ }
+
+ int encode_chunks(const std::set<int> &want_to_encode,
+ std::map<int, bufferlist> *encoded) override {
+ ceph_abort();
+ return 0;
+ }
+
+ int _decode(const std::set<int> &want_to_read,
+ const std::map<int, bufferlist> &chunks,
+ std::map<int, bufferlist> *decoded) override {
+ //
+ // All chunks have the same size
+ //
+ unsigned chunk_length = (*chunks.begin()).second.length();
+ for (std::set<int>::iterator i = want_to_read.begin();
+ i != want_to_read.end();
+ ++i) {
+ if (chunks.find(*i) != chunks.end()) {
+ //
+ // If the chunk is available, just copy the bufferptr pointer
+ // to the decoded argument.
+ //
+ (*decoded)[*i] = chunks.find(*i)->second;
+ } else if(chunks.size() != 2) {
+ //
+ // If a chunk is missing and there are not enough chunks
+ // to recover, abort.
+ //
+ return -ERANGE;
+ } else {
+ //
+ // No matter what the missing chunk is, XOR of the other
+ // two recovers it.
+ //
+ std::map<int, bufferlist>::const_iterator k = chunks.begin();
+ const char *a = k->second.front().c_str();
+ ++k;
+ const char *b = k->second.front().c_str();
+ bufferptr chunk(chunk_length);
+ char *c = chunk.c_str();
+ for (unsigned j = 0; j < chunk_length; j++) {
+ c[j] = a[j] ^ b[j];
+ }
+
+ bufferlist tmp;
+ tmp.append(chunk);
+ tmp.claim_append((*decoded)[*i]);
+ (*decoded)[*i].swap(tmp);
+ }
+ }
+ return 0;
+ }
+
+ int decode_chunks(const std::set<int> &want_to_read,
+ const std::map<int, bufferlist> &chunks,
+ std::map<int, bufferlist> *decoded) override {
+ ceph_abort();
+ return 0;
+ }
+
+ const std::vector<int> &get_chunk_mapping() const override {
+ static std::vector<int> mapping;
+ return mapping;
+ }
+
+};
+
+#endif
diff --git a/src/test/erasure-code/ErasureCodePluginExample.cc b/src/test/erasure-code/ErasureCodePluginExample.cc
new file mode 100644
index 000000000..697b77d94
--- /dev/null
+++ b/src/test/erasure-code/ErasureCodePluginExample.cc
@@ -0,0 +1,45 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph distributed storage system
+ *
+ * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com>
+ * Copyright (C) 2014 Red Hat <contact@redhat.com>
+ *
+ * Author: Loic Dachary <loic@dachary.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#include <unistd.h>
+
+#include "ceph_ver.h"
+#include "erasure-code/ErasureCodePlugin.h"
+#include "ErasureCodeExample.h"
+
+using namespace std;
+
+class ErasureCodePluginExample : public ErasureCodePlugin {
+public:
+ int factory(const std::string &directory,
+ ErasureCodeProfile &profile,
+ ErasureCodeInterfaceRef *erasure_code,
+ ostream *ss) override
+ {
+ *erasure_code = ErasureCodeInterfaceRef(new ErasureCodeExample());
+ (*erasure_code)->init(profile, ss);
+ return 0;
+ }
+};
+
+const char *__erasure_code_version() { return CEPH_GIT_NICE_VER; }
+
+int __erasure_code_init(char *plugin_name, char *directory)
+{
+ ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
+ return instance.add(plugin_name, new ErasureCodePluginExample());
+}
diff --git a/src/test/erasure-code/ErasureCodePluginFailToInitialize.cc b/src/test/erasure-code/ErasureCodePluginFailToInitialize.cc
new file mode 100644
index 000000000..f1219fab4
--- /dev/null
+++ b/src/test/erasure-code/ErasureCodePluginFailToInitialize.cc
@@ -0,0 +1,26 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph distributed storage system
+ *
+ * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com>
+ * Copyright (C) 2014 Red Hat <contact@redhat.com>
+ *
+ * Author: Loic Dachary <loic@dachary.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#include <errno.h>
+#include "ceph_ver.h"
+
+extern "C" const char *__erasure_code_version() { return CEPH_GIT_NICE_VER; }
+
+extern "C" int __erasure_code_init(char *plugin_name, char *directory)
+{
+ return -ESRCH;
+}
diff --git a/src/test/erasure-code/ErasureCodePluginFailToRegister.cc b/src/test/erasure-code/ErasureCodePluginFailToRegister.cc
new file mode 100644
index 000000000..9e8e0161e
--- /dev/null
+++ b/src/test/erasure-code/ErasureCodePluginFailToRegister.cc
@@ -0,0 +1,25 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph distributed storage system
+ *
+ * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com>
+ * Copyright (C) 2014 Red Hat <contact@redhat.com>
+ *
+ * Author: Loic Dachary <loic@dachary.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#include "ceph_ver.h"
+
+extern "C" const char *__erasure_code_version() { return CEPH_GIT_NICE_VER; }
+
+extern "C" int __erasure_code_init(char *plugin_name, char *directory)
+{
+ return 0;
+}
diff --git a/src/test/erasure-code/ErasureCodePluginHangs.cc b/src/test/erasure-code/ErasureCodePluginHangs.cc
new file mode 100644
index 000000000..037fff1bb
--- /dev/null
+++ b/src/test/erasure-code/ErasureCodePluginHangs.cc
@@ -0,0 +1,27 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph distributed storage system
+ *
+ * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com>
+ * Copyright (C) 2014 Red Hat <contact@redhat.com>
+ *
+ * Author: Loic Dachary <loic@dachary.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#include <unistd.h>
+#include "ceph_ver.h"
+
+extern "C" const char *__erasure_code_version() { return CEPH_GIT_NICE_VER; }
+
+extern "C" int __erasure_code_init(char *plugin_name, char *directory)
+{
+ sleep(10);
+ return 0;
+}
diff --git a/src/test/erasure-code/ErasureCodePluginMissingEntryPoint.cc b/src/test/erasure-code/ErasureCodePluginMissingEntryPoint.cc
new file mode 100644
index 000000000..8a55214b1
--- /dev/null
+++ b/src/test/erasure-code/ErasureCodePluginMissingEntryPoint.cc
@@ -0,0 +1,6 @@
+#include "ceph_ver.h"
+
+// missing int __erasure_code_init(char *plugin_name, char *directory) {}
+
+extern "C" const char *__erasure_code_version() { return CEPH_GIT_NICE_VER; }
+
diff --git a/src/test/erasure-code/ErasureCodePluginMissingVersion.cc b/src/test/erasure-code/ErasureCodePluginMissingVersion.cc
new file mode 100644
index 000000000..da4ed0e48
--- /dev/null
+++ b/src/test/erasure-code/ErasureCodePluginMissingVersion.cc
@@ -0,0 +1,3 @@
+// missing __erasure_code_version
+
+int __this_is_an_used_variable_to_avoid_warnings;
diff --git a/src/test/erasure-code/TestErasureCode.cc b/src/test/erasure-code/TestErasureCode.cc
new file mode 100644
index 000000000..05b95ded4
--- /dev/null
+++ b/src/test/erasure-code/TestErasureCode.cc
@@ -0,0 +1,169 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph distributed storage system
+ *
+ * Copyright (C) 2014 Red Hat <contact@redhat.com>
+ *
+ * Author: Loic Dachary <loic@dachary.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#include <errno.h>
+#include <stdlib.h>
+
+#include "erasure-code/ErasureCode.h"
+#include "global/global_context.h"
+#include "common/config.h"
+#include "gtest/gtest.h"
+
+using namespace std;
+
+class ErasureCodeTest : public ErasureCode {
+public:
+ map<int, bufferlist> encode_chunks_encoded;
+ unsigned int k;
+ unsigned int m;
+ unsigned int chunk_size;
+
+ ErasureCodeTest(unsigned int _k, unsigned int _m, unsigned int _chunk_size) :
+ k(_k), m(_m), chunk_size(_chunk_size) {}
+ ~ErasureCodeTest() override {}
+
+ int init(ErasureCodeProfile &profile, ostream *ss) override {
+ return 0;
+ }
+
+ unsigned int get_chunk_count() const override { return k + m; }
+ unsigned int get_data_chunk_count() const override { return k; }
+ unsigned int get_chunk_size(unsigned int object_size) const override {
+ return chunk_size;
+ }
+ int encode_chunks(const set<int> &want_to_encode,
+ map<int, bufferlist> *encoded) override {
+ encode_chunks_encoded = *encoded;
+ return 0;
+ }
+ int decode_chunks(const set<int> &want_to_read,
+ const map<int, bufferlist> &chunks,
+ map<int, bufferlist> *decoded) override {
+ ceph_abort_msg("ErasureCode::decode_chunks not implemented");
+ }
+
+ int create_rule(const string &name,
+ CrushWrapper &crush,
+ ostream *ss) const override { return 0; }
+};
+
+/*
+ * If we have a buffer of 5 bytes (X below) and a chunk size of 3
+ * bytes, for k=3, m=1 an additional 7 bytes (P and C below) will
+ * need to be allocated for padding (P) and the 3 coding bytes (C).
+ *
+ * X -+ +----------+ +-X
+ * X | | data 0 | | X
+ * X | +----------+ | X
+ * X | +----------+ | X -> +-X
+ * X -+ | data 1 | +-X -> | X
+ * P -+ +----------+ | P
+ * P | +----------+ | P
+ * P | | data 2 | | P
+ * P | +----------+ | P
+ * C | +----------+ | C
+ * C | | coding 3 | | C
+ * C -+ +----------+ +-C
+ *
+ * The data chunks 1 and 2 (data 1 and data 2 above) overflow the
+ * original buffer because it needs padding. A new buffer will
+ * be allocated to contain the chunk that overflows and all other
+ * chunks after it, including the coding chunk(s).
+ *
+ * The following test creates a siguation where the buffer provided
+ * for encoding is not memory aligned. After encoding it asserts that:
+ *
+ * a) each chunk is SIMD aligned
+ * b) the data 1 chunk content is as expected which implies that its
+ * content has been copied over.
+ *
+ * It is possible for a flawed implementation to pas the test because the
+ * underlying allocation function enforces it.
+ */
+TEST(ErasureCodeTest, encode_memory_align)
+{
+ int k = 3;
+ int m = 1;
+ unsigned chunk_size = ErasureCode::SIMD_ALIGN * 7;
+ ErasureCodeTest erasure_code(k, m, chunk_size);
+
+ set<int> want_to_encode;
+ for (unsigned int i = 0; i < erasure_code.get_chunk_count(); i++)
+ want_to_encode.insert(i);
+ string data(chunk_size + chunk_size / 2, 'X'); // uses 1.5 chunks out of 3
+ // make sure nothing is memory aligned
+ bufferptr ptr(buffer::create_aligned(data.length() + 1, ErasureCode::SIMD_ALIGN));
+ ptr.copy_in(1, data.length(), data.c_str());
+ ptr.set_offset(1);
+ ptr.set_length(data.length());
+ bufferlist in;
+ in.append(ptr);
+ map<int, bufferlist> encoded;
+
+ ASSERT_FALSE(in.is_aligned(ErasureCode::SIMD_ALIGN));
+ ASSERT_EQ(0, erasure_code.encode(want_to_encode, in, &encoded));
+ for (unsigned int i = 0; i < erasure_code.get_chunk_count(); i++)
+ ASSERT_TRUE(encoded[i].is_aligned(ErasureCode::SIMD_ALIGN));
+ for (unsigned i = 0; i < chunk_size / 2; i++)
+ ASSERT_EQ(encoded[1][i], 'X');
+ ASSERT_NE(encoded[1][chunk_size / 2], 'X');
+}
+
+TEST(ErasureCodeTest, encode_misaligned_non_contiguous)
+{
+ int k = 3;
+ int m = 1;
+ unsigned chunk_size = ErasureCode::SIMD_ALIGN * 7;
+ ErasureCodeTest erasure_code(k, m, chunk_size);
+
+ set<int> want_to_encode;
+ for (unsigned int i = 0; i < erasure_code.get_chunk_count(); i++)
+ want_to_encode.insert(i);
+ string data(chunk_size, 'X');
+ // create a non contiguous bufferlist where the frist and the second
+ // bufferptr are not size aligned although they are memory aligned
+ bufferlist in;
+ {
+ bufferptr ptr(buffer::create_aligned(data.length() - 1, ErasureCode::SIMD_ALIGN));
+ in.append(ptr);
+ }
+ {
+ bufferptr ptr(buffer::create_aligned(data.length() + 1, ErasureCode::SIMD_ALIGN));
+ in.append(ptr);
+ }
+ map<int, bufferlist> encoded;
+
+ ASSERT_FALSE(in.is_contiguous());
+ ASSERT_TRUE(in.front().is_aligned(ErasureCode::SIMD_ALIGN));
+ ASSERT_FALSE(in.front().is_n_align_sized(chunk_size));
+ ASSERT_TRUE(in.back().is_aligned(ErasureCode::SIMD_ALIGN));
+ ASSERT_FALSE(in.back().is_n_align_sized(chunk_size));
+ ASSERT_EQ(0, erasure_code.encode(want_to_encode, in, &encoded));
+ for (unsigned int i = 0; i < erasure_code.get_chunk_count(); i++) {
+ ASSERT_TRUE(encoded[i].is_aligned(ErasureCode::SIMD_ALIGN));
+ ASSERT_TRUE(encoded[i].is_n_align_sized(chunk_size));
+ }
+}
+
+/*
+ * Local Variables:
+ * compile-command: "cd ../.. ;
+ * make -j4 unittest_erasure_code &&
+ * valgrind --tool=memcheck --leak-check=full \
+ * ./unittest_erasure_code \
+ * --gtest_filter=*.* --log-to-stderr=true"
+ * End:
+ */
diff --git a/src/test/erasure-code/TestErasureCodeClay.cc b/src/test/erasure-code/TestErasureCodeClay.cc
new file mode 100644
index 000000000..cb4740948
--- /dev/null
+++ b/src/test/erasure-code/TestErasureCodeClay.cc
@@ -0,0 +1,596 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph distributed storage system
+ *
+ * Copyright (C) 2018 Indian Institute of Science <office.ece@iisc.ac.in>
+ *
+ * Author: Myna Vajha <mynaramana@gmail.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#include <errno.h>
+#include <stdlib.h>
+
+#include "crush/CrushWrapper.h"
+#include "include/stringify.h"
+#include "erasure-code/clay/ErasureCodeClay.h"
+#include "global/global_context.h"
+#include "common/config_proxy.h"
+#include "gtest/gtest.h"
+
+using namespace std;
+
+TEST(ErasureCodeClay, sanity_check_k)
+{
+ ErasureCodeClay clay(g_conf().get_val<std::string>("erasure_code_dir"));
+ ErasureCodeProfile profile;
+ profile["k"] = "1";
+ profile["m"] = "1";
+ ostringstream errors;
+ EXPECT_EQ(-EINVAL, clay.init(profile, &errors));
+ EXPECT_NE(std::string::npos, errors.str().find("must be >= 2"));
+}
+
+TEST(ErasureCodeClay, encode_decode)
+{
+ ostringstream errors;
+ ErasureCodeClay clay(g_conf().get_val<std::string>("erasure_code_dir"));
+ ErasureCodeProfile profile;
+ profile["k"] = "2";
+ profile["m"] = "2";
+ int r= clay.init(profile, &cerr);
+ EXPECT_EQ(0, r);
+
+#define LARGE_ENOUGH 2048
+ bufferptr in_ptr(buffer::create_page_aligned(LARGE_ENOUGH));
+ in_ptr.zero();
+ in_ptr.set_length(0);
+ const char *payload =
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+ in_ptr.append(payload, strlen(payload));
+ bufferlist in;
+ in.push_back(in_ptr);
+ int want_to_encode[] = { 0, 1, 2, 3 };
+ map<int, bufferlist> encoded;
+ EXPECT_EQ(0, clay.encode(set<int>(want_to_encode, want_to_encode+4),
+ in,
+ &encoded));
+ EXPECT_EQ(4u, encoded.size());
+ unsigned length = encoded[0].length();
+ EXPECT_EQ(0, memcmp(encoded[0].c_str(), in.c_str(), length));
+ EXPECT_EQ(0, memcmp(encoded[1].c_str(), in.c_str() + length,
+ in.length() - length));
+
+
+ // all chunks are available
+ {
+ int want_to_decode[] = { 0, 1 };
+ map<int, bufferlist> decoded;
+ EXPECT_EQ(0, clay._decode(set<int>(want_to_decode, want_to_decode+2),
+ encoded,
+ &decoded));
+ EXPECT_EQ(2u, decoded.size());
+ EXPECT_EQ(length, decoded[0].length());
+ EXPECT_EQ(0, memcmp(decoded[0].c_str(), in.c_str(), length));
+ EXPECT_EQ(0, memcmp(decoded[1].c_str(), in.c_str() + length,
+ in.length() - length));
+ }
+
+ // check all two chunks missing possibilities and recover them
+ for (int i=1; i<4; i++) {
+ for (int j=0; j<i; j++) {
+ map<int, bufferlist> degraded = encoded;
+ degraded.erase(j);
+ degraded.erase(i);
+ EXPECT_EQ(2u, degraded.size());
+ int want_to_decode[] = {j,i};
+ map<int, bufferlist> decoded;
+ EXPECT_EQ(0, clay._decode(set<int>(want_to_decode, want_to_decode+2),
+ degraded,
+ &decoded));
+ EXPECT_EQ(4u, decoded.size());
+ EXPECT_EQ(length, decoded[j].length());
+ EXPECT_EQ(0, memcmp(decoded[j].c_str(), encoded[j].c_str(), length));
+ EXPECT_EQ(0, memcmp(decoded[i].c_str(), encoded[i].c_str(), length));
+ }
+ }
+ //check for all one chunk missing possibilities
+ int sc_size = length/clay.sub_chunk_no;
+ int avail[] = {0,1,2,3};
+ for (int i=0; i < 4; i++) {
+ set<int> want_to_read;
+ want_to_read.insert(i);
+ set<int> available(avail, avail+4);
+ available.erase(i);
+ map<int, vector<pair<int,int>>> minimum;
+ EXPECT_EQ(0, clay.minimum_to_decode(want_to_read, available, &minimum));
+ map<int, bufferlist> helper;
+ for (map<int, vector<pair<int,int>>>::iterator h=minimum.begin(); h!= minimum.end(); ++h) {
+ for(vector<pair<int,int>>::iterator ind=h->second.begin(); ind != h->second.end(); ++ind) {
+ bufferlist temp;
+ temp.substr_of(encoded[h->first], ind->first*sc_size, ind->second*sc_size);
+ helper[h->first].append(temp);
+ }
+ }
+ for (map<int, vector<pair<int,int>>>::iterator h=minimum.begin(); h!= minimum.end(); ++h) {
+ EXPECT_EQ(length/clay.q, helper[h->first].length());
+ }
+ EXPECT_EQ(3u, helper.size());
+ map<int, bufferlist> decoded;
+ EXPECT_EQ(0, clay.decode(want_to_read, helper, &decoded, length));
+ EXPECT_EQ(1u, decoded.size());
+ EXPECT_EQ(0, memcmp(decoded[i].c_str(), encoded[i].c_str(), length));
+ }
+}
+
+
+TEST(ErasureCodeClay, encode_decode_aloof_nodes)
+{
+ ostringstream errors;
+ ErasureCodeClay clay(g_conf().get_val<std::string>("erasure_code_dir"));
+ ErasureCodeProfile profile;
+ profile["k"] = "3";
+ profile["m"] = "3";
+ profile["d"] = "4";
+ int r= clay.init(profile, &cerr);
+ EXPECT_EQ(0, r);
+
+#define LARGE_ENOUGH 2048
+ bufferptr in_ptr(buffer::create_page_aligned(LARGE_ENOUGH));
+ in_ptr.zero();
+ in_ptr.set_length(0);
+ const char *payload =
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+ in_ptr.append(payload, strlen(payload));
+ bufferlist in;
+ in.push_back(in_ptr);
+ int want_to_encode[] = { 0, 1, 2, 3, 4, 5 };
+ map<int, bufferlist> encoded;
+ EXPECT_EQ(0, clay.encode(set<int>(want_to_encode, want_to_encode+6),
+ in,
+ &encoded));
+ EXPECT_EQ(6u, encoded.size());
+ unsigned length = encoded[0].length();
+ if (in.length() < length) {
+ EXPECT_EQ(0, memcmp(encoded[0].c_str(), in.c_str(), in.length()));
+ } else if (in.length() <= 2*length ) {
+ EXPECT_EQ(0, memcmp(encoded[0].c_str(), in.c_str(), in.length()));
+ EXPECT_EQ(0, memcmp(encoded[1].c_str(), in.c_str()+length, in.length()-length));
+ } else {
+ EXPECT_EQ(1, in.length() <= 3*length);
+ EXPECT_EQ(0, memcmp(encoded[0].c_str(), in.c_str(), in.length()));
+ EXPECT_EQ(0, memcmp(encoded[1].c_str(), in.c_str()+length, length));
+ EXPECT_EQ(0, memcmp(encoded[2].c_str(), in.c_str()+2*length, in.length()-2*length));
+ }
+
+ // all chunks are available
+ {
+ int want_to_decode[] = { 0, 1, 2 };
+ map<int, bufferlist> decoded;
+ EXPECT_EQ(0, clay._decode(set<int>(want_to_decode, want_to_decode+3),
+ encoded,
+ &decoded));
+ EXPECT_EQ(3u, decoded.size());
+ EXPECT_EQ(length, decoded[0].length());
+ EXPECT_EQ(0, memcmp(decoded[0].c_str(), encoded[0].c_str(), length));
+ EXPECT_EQ(0, memcmp(decoded[1].c_str(), encoded[1].c_str(), length));
+ EXPECT_EQ(0, memcmp(decoded[2].c_str(), encoded[2].c_str(), length));
+ }
+
+ // check all three chunks missing possibilities and recover them
+ for (int i=2; i<6; i++) {
+ for (int j=1; j<i; j++) {
+ for(int k=0; k<j; k++) {
+ map<int, bufferlist> degraded = encoded;
+ degraded.erase(k);
+ degraded.erase(j);
+ degraded.erase(i);
+ EXPECT_EQ(3u, degraded.size());
+ int want_to_decode[] = {k,j,i};
+ map<int, bufferlist> decoded;
+ EXPECT_EQ(0, clay._decode(set<int>(want_to_decode, want_to_decode+3),
+ degraded,
+ &decoded));
+ EXPECT_EQ(6u, decoded.size());
+ EXPECT_EQ(length, decoded[j].length());
+ EXPECT_EQ(0, memcmp(decoded[k].c_str(), encoded[k].c_str(), length));
+ EXPECT_EQ(0, memcmp(decoded[j].c_str(), encoded[j].c_str(), length));
+ EXPECT_EQ(0, memcmp(decoded[i].c_str(), encoded[i].c_str(), length));
+ }
+ }
+ }
+ //check for all one chunk missing possibilities
+ int sc_size = length/clay.sub_chunk_no;
+ int avail[] = {0,1,2,3,4,5};
+ for (int i=0; i < 6; i++) {
+ vector<pair<int,int>> repair_subchunks;
+ map<int, vector<pair<int,int>>> minimum;
+ set<int> want_to_read;
+ want_to_read.insert(i);
+ set<int> available(avail, avail+6);
+ available.erase(i);
+ clay.minimum_to_decode(want_to_read, available, &minimum);
+ map<int, bufferlist> helper;
+ for (map<int, vector<pair<int,int>>>::iterator h=minimum.begin(); h!= minimum.end(); ++h) {
+ for(vector<pair<int,int>>::iterator ind=h->second.begin(); ind != h->second.end(); ++ind) {
+ bufferlist temp;
+ temp.substr_of(encoded[h->first], ind->first*sc_size, ind->second*sc_size);
+ helper[h->first].append(temp);
+ }
+ }
+ for (map<int, vector<pair<int,int>>>::iterator h=minimum.begin(); h!= minimum.end(); ++h) {
+ EXPECT_EQ(length/clay.q, helper[h->first].length());
+ }
+ EXPECT_EQ((unsigned)clay.d, helper.size());
+ map<int, bufferlist> decoded;
+ EXPECT_EQ(0, clay.decode(want_to_read, helper, &decoded, length));
+ EXPECT_EQ(1u, decoded.size());
+ EXPECT_EQ(0, memcmp(decoded[i].c_str(), encoded[i].c_str(), length));
+ }
+}
+
+TEST(ErasureCodeClay, encode_decode_shortening_case)
+{
+ ostringstream errors;
+ ErasureCodeClay clay(g_conf().get_val<std::string>("erasure_code_dir"));
+ ErasureCodeProfile profile;
+ profile["k"] = "4";
+ profile["m"] = "3";
+ profile["d"] = "5";
+ int r= clay.init(profile, &cerr);
+ EXPECT_EQ(0, r);
+
+ EXPECT_EQ(2, clay.q);
+ EXPECT_EQ(4, clay.t);
+ EXPECT_EQ(1, clay.nu);
+
+ bufferptr in_ptr(buffer::create_page_aligned(LARGE_ENOUGH));
+ in_ptr.zero();
+ in_ptr.set_length(0);
+ const char *payload =
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+ in_ptr.append(payload, strlen(payload));
+ bufferlist in;
+ in.push_back(in_ptr);
+ int want_to_encode[] = { 0, 1, 2, 3, 4, 5, 6 };
+ map<int, bufferlist> encoded;
+ EXPECT_EQ(0, clay.encode(set<int>(want_to_encode, want_to_encode+7),
+ in,
+ &encoded));
+ EXPECT_EQ(7u, encoded.size());
+ unsigned length = encoded[0].length();
+ if (in.length() < length) {
+ EXPECT_EQ(0, memcmp(encoded[0].c_str(), in.c_str(), in.length()));
+ } else if (in.length() <= 2*length) {
+ EXPECT_EQ(0, memcmp(encoded[0].c_str(), in.c_str(), in.length()));
+ EXPECT_EQ(0, memcmp(encoded[1].c_str(), in.c_str()+length, in.length()-length));
+ } else if (in.length() <= 3*length) {
+ EXPECT_EQ(0, memcmp(encoded[0].c_str(), in.c_str(), in.length()));
+ EXPECT_EQ(0, memcmp(encoded[1].c_str(), in.c_str()+length, length));
+ EXPECT_EQ(0, memcmp(encoded[2].c_str(), in.c_str()+2*length, in.length()-2*length));
+ } else {
+ EXPECT_EQ(1, in.length() <= 4*length);
+ EXPECT_EQ(0, memcmp(encoded[0].c_str(), in.c_str(), in.length()));
+ EXPECT_EQ(0, memcmp(encoded[1].c_str(), in.c_str()+length, length));
+ EXPECT_EQ(0, memcmp(encoded[2].c_str(), in.c_str()+2*length, length));
+ EXPECT_EQ(0, memcmp(encoded[3].c_str(), in.c_str()+3*length, in.length()-3*length));
+ }
+
+ // all chunks are available
+ {
+ int want_to_decode[] = { 0, 1, 2, 3 };
+ map<int, bufferlist> decoded;
+ EXPECT_EQ(0, clay._decode(set<int>(want_to_decode, want_to_decode+4),
+ encoded,
+ &decoded));
+ EXPECT_EQ(4u, decoded.size());
+ EXPECT_EQ(length, decoded[0].length());
+ EXPECT_EQ(0, memcmp(decoded[0].c_str(), encoded[0].c_str(), length));
+ EXPECT_EQ(0, memcmp(decoded[1].c_str(), encoded[1].c_str(), length));
+ EXPECT_EQ(0, memcmp(decoded[2].c_str(), encoded[2].c_str(), length));
+ EXPECT_EQ(0, memcmp(decoded[3].c_str(), encoded[3].c_str(), length));
+ }
+
+ // check all three chunks missing possibilities and recover them
+ for (int i=2; i<7; i++) {
+ for (int j=1; j<i; j++) {
+ for(int k=0; k<j; k++) {
+ map<int, bufferlist> degraded = encoded;
+ degraded.erase(k);
+ degraded.erase(j);
+ degraded.erase(i);
+ EXPECT_EQ(4u, degraded.size());
+ int want_to_decode[] = {k,j,i};
+ map<int, bufferlist> decoded;
+ EXPECT_EQ(0, clay._decode(set<int>(want_to_decode, want_to_decode+3),
+ degraded,
+ &decoded));
+ EXPECT_EQ(7u, decoded.size());
+ EXPECT_EQ(length, decoded[j].length());
+ EXPECT_EQ(0, memcmp(decoded[k].c_str(), encoded[k].c_str(), length));
+ EXPECT_EQ(0, memcmp(decoded[j].c_str(), encoded[j].c_str(), length));
+ EXPECT_EQ(0, memcmp(decoded[i].c_str(), encoded[i].c_str(), length));
+ }
+ }
+ }
+ //check for all one chunk missing possibilities
+ int sc_size = length/clay.sub_chunk_no;
+ int avail[] = {0,1,2,3,4,5,6};
+ for (int i=0; i < 7; i++) {
+ vector<pair<int,int>> repair_subchunks;
+ map<int, vector<pair<int,int>>> minimum;
+ set<int> want_to_read;
+ want_to_read.insert(i);
+ set<int> available(avail, avail+7);
+ available.erase(i);
+ clay.minimum_to_decode(want_to_read, available, &minimum);
+ map<int, bufferlist> helper;
+ for (map<int, vector<pair<int,int>>>::iterator h=minimum.begin(); h!= minimum.end(); ++h) {
+ for(vector<pair<int,int>>::iterator ind=h->second.begin(); ind != h->second.end(); ++ind) {
+ bufferlist temp;
+ temp.substr_of(encoded[h->first], ind->first*sc_size, ind->second*sc_size);
+ helper[h->first].append(temp);
+ }
+ }
+ for (map<int, vector<pair<int,int>>>::iterator h=minimum.begin(); h!= minimum.end(); ++h) {
+ EXPECT_EQ(length/clay.q, helper[h->first].length());
+ }
+ EXPECT_EQ(static_cast<size_t>(clay.d), helper.size());
+ map<int, bufferlist> decoded;
+ EXPECT_EQ(0, clay.decode(want_to_read, helper, &decoded, length));
+ EXPECT_EQ(1u, decoded.size());
+ EXPECT_EQ(length, decoded[i].length());
+ EXPECT_EQ(0, memcmp(decoded[i].c_str(), encoded[i].c_str(), length));
+ }
+}
+
+TEST(ErasureCodeClay, minimum_to_decode)
+{
+ ErasureCodeClay clay(g_conf().get_val<std::string>("erasure_code_dir"));
+ ErasureCodeProfile profile;
+ profile["k"] = "2";
+ profile["m"] = "2";
+ EXPECT_EQ(0, clay.init(profile, &cerr));
+
+ //
+ // If trying to read nothing, the minimum is empty.
+ //
+ {
+ set<int> want_to_read;
+ set<int> available_chunks;
+ set<int> minimum;
+
+ EXPECT_EQ(0, clay._minimum_to_decode(want_to_read,
+ available_chunks,
+ &minimum));
+ EXPECT_TRUE(minimum.empty());
+ }
+ //
+ // There is no way to read a chunk if none are available.
+ //
+ {
+ set<int> want_to_read;
+ set<int> available_chunks;
+ set<int> minimum;
+
+ want_to_read.insert(0);
+
+ EXPECT_EQ(-EIO, clay._minimum_to_decode(want_to_read,
+ available_chunks,
+ &minimum));
+ }
+ //
+ // Reading a subset of the available chunks is always possible.
+ //
+ {
+ set<int> want_to_read;
+ set<int> available_chunks;
+ set<int> minimum;
+
+ want_to_read.insert(0);
+ available_chunks.insert(0);
+
+ EXPECT_EQ(0, clay._minimum_to_decode(want_to_read,
+ available_chunks,
+ &minimum));
+ EXPECT_EQ(want_to_read, minimum);
+ }
+ //
+ // There is no way to read a missing chunk if there is less than k
+ // chunks available.
+ //
+ {
+ set<int> want_to_read;
+ set<int> available_chunks;
+ set<int> minimum;
+
+ want_to_read.insert(0);
+ want_to_read.insert(1);
+ available_chunks.insert(0);
+
+ EXPECT_EQ(-EIO, clay._minimum_to_decode(want_to_read,
+ available_chunks,
+ &minimum));
+ }
+ //
+ // When chunks are not available, the minimum can be made of any
+ // chunks. For instance, to read 1 and 3 below the minimum could be
+ // 2 and 3 which may seem better because it contains one of the
+ // chunks to be read. But it won't be more efficient than retrieving
+ // 0 and 2 instead because, in both cases, the decode function will
+ // need to run the same recovery operation and use the same amount
+ // of CPU and memory.
+ //
+ {
+ set<int> want_to_read;
+ set<int> available_chunks;
+ set<int> minimum;
+
+ want_to_read.insert(1);
+ want_to_read.insert(3);
+ available_chunks.insert(0);
+ available_chunks.insert(2);
+ available_chunks.insert(3);
+
+ EXPECT_EQ(0, clay._minimum_to_decode(want_to_read,
+ available_chunks,
+ &minimum));
+ EXPECT_EQ(2u, minimum.size());
+ EXPECT_EQ(0u, minimum.count(3));
+ }
+}
+
+TEST(ErasureCodeClay, encode)
+{
+ ErasureCodeClay clay(g_conf().get_val<std::string>("erasure_code_dir"));
+ ErasureCodeProfile profile;
+ profile["k"] = "2";
+ profile["m"] = "2";
+ EXPECT_EQ(0, clay.init(profile, &cerr));
+
+ unsigned aligned_object_size = clay.get_chunk_size(1) * 2 * 2;
+ {
+ //
+ // When the input bufferlist needs to be padded because
+ // it is not properly aligned, it is padded with zeros.
+ //
+ bufferlist in;
+ map<int,bufferlist> encoded;
+ int want_to_encode[] = { 0, 1, 2, 3 };
+ int trail_length = 1;
+ in.append(string(aligned_object_size + trail_length, 'X'));
+ EXPECT_EQ(0, clay.encode(set<int>(want_to_encode, want_to_encode+4),
+ in,
+ &encoded));
+ EXPECT_EQ(4u, encoded.size());
+ char *last_chunk = encoded[1].c_str();
+ int length =encoded[1].length();
+ EXPECT_EQ('X', last_chunk[0]);
+ EXPECT_EQ('\0', last_chunk[length - trail_length]);
+ }
+
+ {
+ //
+ // When only the first chunk is required, the encoded map only
+ // contains the first chunk. Although the clay encode
+ // internally allocated a buffer because of padding requirements
+ // and also computes the coding chunks, they are released before
+ // the return of the method, as shown when running the tests thru
+ // valgrind (there is no leak).
+ //
+ bufferlist in;
+ map<int,bufferlist> encoded;
+ set<int> want_to_encode;
+ want_to_encode.insert(0);
+ int trail_length = 1;
+ in.append(string(aligned_object_size + trail_length, 'X'));
+ EXPECT_EQ(0, clay.encode(want_to_encode, in, &encoded));
+ EXPECT_EQ(1u, encoded.size());
+ }
+}
+
+TEST(ErasureCodeClay, create_rule)
+{
+ std::unique_ptr<CrushWrapper> c = std::make_unique<CrushWrapper>();
+ c->create();
+ int root_type = 2;
+ c->set_type_name(root_type, "root");
+ int host_type = 1;
+ c->set_type_name(host_type, "host");
+ int osd_type = 0;
+ c->set_type_name(osd_type, "osd");
+
+ int rootno;
+ c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+ root_type, 0, NULL, NULL, &rootno);
+ c->set_item_name(rootno, "default");
+
+ map<string,string> loc;
+ loc["root"] = "default";
+
+ int num_host = 4;
+ int num_osd = 5;
+ int osd = 0;
+ for (int h=0; h<num_host; ++h) {
+ loc["host"] = string("host-") + stringify(h);
+ for (int o=0; o<num_osd; ++o, ++osd) {
+ c->insert_item(g_ceph_context, osd, 1.0, string("osd.") + stringify(osd), loc);
+ }
+ }
+
+ c->finalize();
+
+ {
+ stringstream ss;
+ ErasureCodeClay clay(g_conf().get_val<std::string>("erasure_code_dir"));
+ ErasureCodeProfile profile;
+ profile["k"] = "2";
+ profile["m"] = "2";
+ EXPECT_EQ(0, clay.init(profile, &cerr));
+ int ruleid = clay.create_rule("myrule", *c, &ss);
+ EXPECT_EQ(0, ruleid);
+ EXPECT_EQ(-EEXIST, clay.create_rule("myrule", *c, &ss));
+ //
+ // the minimum that is expected from the created rule is to
+ // successfully map get_chunk_count() devices from the crushmap,
+ // at least once.
+ //
+ vector<__u32> weight(c->get_max_devices(), 0x10000);
+ vector<int> out;
+ int x = 0;
+ c->do_rule(ruleid, x, out, clay.get_chunk_count(), weight, 0);
+ ASSERT_EQ(out.size(), clay.get_chunk_count());
+ for (unsigned i=0; i<out.size(); ++i)
+ ASSERT_NE(CRUSH_ITEM_NONE, out[i]);
+ }
+ {
+ stringstream ss;
+ ErasureCodeClay clay(g_conf().get_val<std::string>("erasure_code_dir"));
+ ErasureCodeProfile profile;
+ profile["k"] = "2";
+ profile["m"] = "2";
+ profile["crush-root"] = "BAD";
+ EXPECT_EQ(0, clay.init(profile, &cerr));
+ EXPECT_EQ(-ENOENT, clay.create_rule("otherrule", *c, &ss));
+ EXPECT_EQ("root item BAD does not exist", ss.str());
+ }
+ {
+ stringstream ss;
+ ErasureCodeClay clay(g_conf().get_val<std::string>("erasure_code_dir"));
+ ErasureCodeProfile profile;
+ profile["k"] = "2";
+ profile["m"] = "2";
+ profile["crush-failure-domain"] = "WORSE";
+ EXPECT_EQ(0, clay.init(profile, &cerr));
+ EXPECT_EQ(-EINVAL, clay.create_rule("otherrule", *c, &ss));
+ EXPECT_EQ("unknown type WORSE", ss.str());
+ }
+}
+
+/*
+ * Local Variables:
+ * compile-command: "cd ../.. ;
+ * make -j4 unittest_erasure_code_clay &&
+ * valgrind --tool=memcheck \
+ * ./unittest_erasure_code_clay \
+ * --gtest_filter=*.* --log-to-stderr=true --debug-osd=20"
+ * End:
+ */
diff --git a/src/test/erasure-code/TestErasureCodeExample.cc b/src/test/erasure-code/TestErasureCodeExample.cc
new file mode 100644
index 000000000..b488a604b
--- /dev/null
+++ b/src/test/erasure-code/TestErasureCodeExample.cc
@@ -0,0 +1,248 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph distributed storage system
+ *
+ * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com>
+ *
+ * Author: Loic Dachary <loic@dachary.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+#include <stdlib.h>
+
+#include "include/stringify.h"
+#include "ErasureCodeExample.h"
+#include "global/global_context.h"
+#include "gtest/gtest.h"
+
+using namespace std;
+
+TEST(ErasureCodeExample, chunk_size)
+{
+ ErasureCodeExample example;
+ EXPECT_EQ(3u, example.get_chunk_count());
+ EXPECT_EQ(11u, example.get_chunk_size(20));
+}
+
+TEST(ErasureCodeExample, minimum_to_decode)
+{
+ ErasureCodeExample example;
+ set<int> available_chunks;
+ set<int> want_to_read;
+ want_to_read.insert(1);
+ {
+ set<int> minimum;
+ EXPECT_EQ(-EIO, example._minimum_to_decode(want_to_read,
+ available_chunks,
+ &minimum));
+ }
+ available_chunks.insert(0);
+ available_chunks.insert(2);
+ {
+ set<int> minimum;
+ EXPECT_EQ(0, example._minimum_to_decode(want_to_read,
+ available_chunks,
+ &minimum));
+ EXPECT_EQ(available_chunks, minimum);
+ EXPECT_EQ(2u, minimum.size());
+ EXPECT_EQ(1u, minimum.count(0));
+ EXPECT_EQ(1u, minimum.count(2));
+ }
+ {
+ set<int> minimum;
+ available_chunks.insert(1);
+ EXPECT_EQ(0, example._minimum_to_decode(want_to_read,
+ available_chunks,
+ &minimum));
+ EXPECT_EQ(1u, minimum.size());
+ EXPECT_EQ(1u, minimum.count(1));
+ }
+}
+
+TEST(ErasureCodeExample, minimum_to_decode_with_cost)
+{
+ ErasureCodeExample example;
+ map<int,int> available;
+ set<int> want_to_read;
+ want_to_read.insert(1);
+ {
+ set<int> minimum;
+ EXPECT_EQ(-EIO, example.minimum_to_decode_with_cost(want_to_read,
+ available,
+ &minimum));
+ }
+ available[0] = 1;
+ available[2] = 1;
+ {
+ set<int> minimum;
+ EXPECT_EQ(0, example.minimum_to_decode_with_cost(want_to_read,
+ available,
+ &minimum));
+ EXPECT_EQ(2u, minimum.size());
+ EXPECT_EQ(1u, minimum.count(0));
+ EXPECT_EQ(1u, minimum.count(2));
+ }
+ {
+ set<int> minimum;
+ available[1] = 1;
+ EXPECT_EQ(0, example.minimum_to_decode_with_cost(want_to_read,
+ available,
+ &minimum));
+ EXPECT_EQ(1u, minimum.size());
+ EXPECT_EQ(1u, minimum.count(1));
+ }
+ {
+ set<int> minimum;
+ available[1] = 2;
+ EXPECT_EQ(0, example.minimum_to_decode_with_cost(want_to_read,
+ available,
+ &minimum));
+ EXPECT_EQ(2u, minimum.size());
+ EXPECT_EQ(1u, minimum.count(0));
+ EXPECT_EQ(1u, minimum.count(2));
+ }
+}
+
+TEST(ErasureCodeExample, encode_decode)
+{
+ ErasureCodeExample example;
+
+ bufferlist in;
+ in.append("ABCDE");
+ set<int> want_to_encode;
+ for(unsigned int i = 0; i < example.get_chunk_count(); i++)
+ want_to_encode.insert(i);
+ map<int, bufferlist> encoded;
+ EXPECT_EQ(0, example.encode(want_to_encode, in, &encoded));
+ EXPECT_EQ(example.get_chunk_count(), encoded.size());
+ EXPECT_EQ(example.get_chunk_size(in.length()), encoded[0].length());
+ EXPECT_EQ('A', encoded[0][0]);
+ EXPECT_EQ('B', encoded[0][1]);
+ EXPECT_EQ('C', encoded[0][2]);
+ EXPECT_EQ('D', encoded[1][0]);
+ EXPECT_EQ('E', encoded[1][1]);
+ EXPECT_EQ('A'^'D', encoded[2][0]);
+ EXPECT_EQ('B'^'E', encoded[2][1]);
+ EXPECT_EQ('C'^0, encoded[2][2]);
+
+ // all chunks are available
+ {
+ int want_to_decode[] = { 0, 1 };
+ map<int, bufferlist> decoded;
+ EXPECT_EQ(0, example._decode(set<int>(want_to_decode, want_to_decode+2),
+ encoded,
+ &decoded));
+ EXPECT_EQ(2u, decoded.size());
+ EXPECT_EQ(3u, decoded[0].length());
+ EXPECT_EQ('A', decoded[0][0]);
+ EXPECT_EQ('B', decoded[0][1]);
+ EXPECT_EQ('C', decoded[0][2]);
+ EXPECT_EQ('D', decoded[1][0]);
+ EXPECT_EQ('E', decoded[1][1]);
+ }
+
+ // one chunk is missing
+ {
+ map<int, bufferlist> degraded = encoded;
+ degraded.erase(0);
+ EXPECT_EQ(2u, degraded.size());
+ int want_to_decode[] = { 0, 1 };
+ map<int, bufferlist> decoded;
+ EXPECT_EQ(0, example._decode(set<int>(want_to_decode, want_to_decode+2),
+ degraded,
+ &decoded));
+ EXPECT_EQ(2u, decoded.size());
+ EXPECT_EQ(3u, decoded[0].length());
+ EXPECT_EQ('A', decoded[0][0]);
+ EXPECT_EQ('B', decoded[0][1]);
+ EXPECT_EQ('C', decoded[0][2]);
+ EXPECT_EQ('D', decoded[1][0]);
+ EXPECT_EQ('E', decoded[1][1]);
+ }
+}
+
+TEST(ErasureCodeExample, decode)
+{
+ ErasureCodeExample example;
+
+#define LARGE_ENOUGH 2048
+ bufferptr in_ptr(buffer::create_page_aligned(LARGE_ENOUGH));
+ in_ptr.zero();
+ in_ptr.set_length(0);
+ const char *payload =
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+ in_ptr.append(payload, strlen(payload));
+ bufferlist in;
+ in.push_back(in_ptr);
+ int want_to_encode[] = { 0, 1, 2 };
+ map<int, bufferlist> encoded;
+ EXPECT_EQ(0, example.encode(set<int>(want_to_encode, want_to_encode+3),
+ in,
+ &encoded));
+ EXPECT_EQ(3u, encoded.size());
+
+ // successfull decode
+ bufferlist out;
+ EXPECT_EQ(0, example.decode_concat(encoded, &out));
+ bufferlist usable;
+ usable.substr_of(out, 0, in.length());
+ EXPECT_TRUE(usable == in);
+
+ // cannot recover
+ map<int, bufferlist> degraded;
+ degraded[0] = encoded[0];
+ EXPECT_EQ(-ERANGE, example.decode_concat(degraded, &out));
+}
+
+TEST(ErasureCodeExample, create_rule)
+{
+ std::unique_ptr<CrushWrapper> c = std::make_unique<CrushWrapper>();
+ c->create();
+ c->set_type_name(2, "root");
+ c->set_type_name(1, "host");
+ c->set_type_name(0, "osd");
+
+ int rootno;
+ c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+ 5, 0, NULL, NULL, &rootno);
+ c->set_item_name(rootno, "default");
+
+ map<string,string> loc;
+ loc["root"] = "default";
+
+ int num_host = 2;
+ int num_osd = 5;
+ int osd = 0;
+ for (int h=0; h<num_host; ++h) {
+ loc["host"] = string("host-") + stringify(h);
+ for (int o=0; o<num_osd; ++o, ++osd) {
+ c->insert_item(g_ceph_context, osd, 1.0, string("osd.") + stringify(osd), loc);
+ }
+ }
+
+ stringstream ss;
+ ErasureCodeExample example;
+ EXPECT_EQ(0, example.create_rule("myrule", *c, &ss));
+}
+
+/*
+ * Local Variables:
+ * compile-command: "cd ../.. ;
+ * make -j4 &&
+ * make unittest_erasure_code_example &&
+ * valgrind --leak-check=full --tool=memcheck \
+ * ./unittest_erasure_code_example --gtest_filter=*.* \
+ * --log-to-stderr=true --debug-osd=20
+ * "
+ * End:
+ */
+
diff --git a/src/test/erasure-code/TestErasureCodeIsa.cc b/src/test/erasure-code/TestErasureCodeIsa.cc
new file mode 100644
index 000000000..bbd4441fc
--- /dev/null
+++ b/src/test/erasure-code/TestErasureCodeIsa.cc
@@ -0,0 +1,967 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2014 CERN (Switzerland)
+ * Copyright (C) 2014 Red Hat <contact@redhat.com>
+ *
+ * Author: Andreas-Joachim Peters <Andreas.Joachim.Peters@cern.ch>
+ * Author: Loic Dachary <loic@dachary.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#include <errno.h>
+#include <stdlib.h>
+
+#include "crush/CrushWrapper.h"
+#include "include/stringify.h"
+#include "erasure-code/isa/ErasureCodeIsa.h"
+#include "erasure-code/isa/xor_op.h"
+#include "global/global_context.h"
+#include "common/config.h"
+#include "gtest/gtest.h"
+
+using namespace std;
+
+ErasureCodeIsaTableCache tcache;
+
+class IsaErasureCodeTest : public ::testing::Test {
+public:
+ void compare_chunks(bufferlist &in, map<int, bufferlist> &encoded);
+ void encode_decode(unsigned object_size);
+};
+
+void IsaErasureCodeTest::compare_chunks(bufferlist &in, map<int, bufferlist> &encoded)
+{
+ unsigned object_size = in.length();
+ unsigned chunk_size = encoded[0].length();
+ for (unsigned i = 0; i < encoded.size(); i++) {
+ if (i * chunk_size >= object_size)
+ break;
+ int chunk_length = object_size > (i + 1) * chunk_size ? chunk_size : object_size - i * chunk_size;
+ EXPECT_EQ(0, memcmp(encoded[i].c_str(), in.c_str() + i * chunk_size, chunk_length));
+ }
+}
+
+void IsaErasureCodeTest::encode_decode(unsigned object_size)
+{
+ ErasureCodeIsaDefault Isa(tcache);
+
+ ErasureCodeProfile profile;
+ profile["k"] = "2";
+ profile["m"] = "2";
+ Isa.init(profile, &cerr);
+
+ string payload(object_size, 'X');
+ bufferlist in;
+ // may be multiple bufferptr if object_size is larger than CEPH_PAGE_SIZE
+ in.append(payload.c_str(), payload.length());
+ int want_to_encode[] = {0, 1, 2, 3};
+ map<int, bufferlist> encoded;
+ EXPECT_EQ(0, Isa.encode(set<int>(want_to_encode, want_to_encode + 4),
+ in,
+ &encoded));
+ EXPECT_EQ(4u, encoded.size());
+ unsigned chunk_size = encoded[0].length();
+ EXPECT_EQ(chunk_size, Isa.get_chunk_size(object_size));
+ compare_chunks(in, encoded);
+
+ // all chunks are available
+ {
+ int want_to_decode[] = {0, 1};
+ map<int, bufferlist> decoded;
+ EXPECT_EQ(0, Isa._decode(set<int>(want_to_decode, want_to_decode + 2),
+ encoded,
+ &decoded));
+ EXPECT_EQ(2u, decoded.size());
+ EXPECT_EQ(chunk_size, decoded[0].length());
+ compare_chunks(in, decoded);
+ }
+
+ // one data chunk is missing
+ {
+ map<int, bufferlist> degraded = encoded;
+
+ string enc1(encoded[1].c_str(), chunk_size);
+
+ degraded.erase(1);
+ EXPECT_EQ(3u, degraded.size());
+ int want_to_decode[] = {1};
+ map<int, bufferlist> decoded;
+ EXPECT_EQ(0, Isa._decode(set<int>(want_to_decode, want_to_decode + 1),
+ degraded,
+ &decoded));
+ // always decode all, regardless of want_to_decode
+ EXPECT_EQ(4u, decoded.size());
+ EXPECT_EQ(chunk_size, decoded[1].length());
+ EXPECT_EQ(0, memcmp(decoded[1].c_str(), enc1.c_str(), chunk_size));
+ }
+
+ // non-xor coding chunk is missing
+ {
+ map<int, bufferlist> degraded = encoded;
+
+ string enc3(encoded[3].c_str(), chunk_size);
+
+ degraded.erase(3);
+ EXPECT_EQ(3u, degraded.size());
+ int want_to_decode[] = {3};
+ map<int, bufferlist> decoded;
+ EXPECT_EQ(0, Isa._decode(set<int>(want_to_decode, want_to_decode + 1),
+ degraded,
+ &decoded));
+ // always decode all, regardless of want_to_decode
+ EXPECT_EQ(4u, decoded.size());
+ EXPECT_EQ(chunk_size, decoded[3].length());
+ EXPECT_EQ(0, memcmp(decoded[3].c_str(), enc3.c_str(), chunk_size));
+ }
+
+ // xor coding chunk is missing
+ {
+ map<int, bufferlist> degraded = encoded;
+
+ string enc2(encoded[2].c_str(), chunk_size);
+
+ degraded.erase(2);
+ EXPECT_EQ(3u, degraded.size());
+ int want_to_decode[] = {2};
+ map<int, bufferlist> decoded;
+ EXPECT_EQ(0, Isa._decode(set<int>(want_to_decode, want_to_decode + 1),
+ degraded,
+ &decoded));
+ // always decode all, regardless of want_to_decode
+ EXPECT_EQ(4u, decoded.size());
+ EXPECT_EQ(chunk_size, decoded[2].length());
+ EXPECT_EQ(0, memcmp(decoded[2].c_str(), enc2.c_str(), chunk_size));
+ }
+
+ // one data and one coding chunk is missing
+ {
+ map<int, bufferlist> degraded = encoded;
+
+ string enc3(encoded[3].c_str(), chunk_size);
+
+ degraded.erase(1);
+ degraded.erase(3);
+ EXPECT_EQ(2u, degraded.size());
+ int want_to_decode[] = {1, 3};
+ map<int, bufferlist> decoded;
+ EXPECT_EQ(0, Isa._decode(set<int>(want_to_decode, want_to_decode + 2),
+ degraded,
+ &decoded));
+ // always decode all, regardless of want_to_decode
+ EXPECT_EQ(4u, decoded.size());
+ EXPECT_EQ(chunk_size, decoded[1].length());
+ EXPECT_EQ(0, memcmp(decoded[3].c_str(), enc3.c_str(), chunk_size));
+ }
+
+ // two data chunks are missing
+ {
+ map<int, bufferlist> degraded = encoded;
+ degraded.erase(0);
+ degraded.erase(1);
+ EXPECT_EQ(2u, degraded.size());
+ int want_to_decode[] = {0, 1};
+ map<int, bufferlist> decoded;
+ EXPECT_EQ(0, Isa._decode(set<int>(want_to_decode, want_to_decode + 2),
+ degraded,
+ &decoded));
+ // always decode all, regardless of want_to_decode
+ EXPECT_EQ(4u, decoded.size());
+ EXPECT_EQ(chunk_size, decoded[0].length());
+ compare_chunks(in, decoded);
+ }
+
+}
+
+TEST_F(IsaErasureCodeTest, encode_decode)
+{
+ encode_decode(1);
+ encode_decode(EC_ISA_ADDRESS_ALIGNMENT);
+ encode_decode(EC_ISA_ADDRESS_ALIGNMENT + 1);
+ encode_decode(2048);
+ encode_decode(4096);
+ encode_decode(4096 + 1);
+}
+
+TEST_F(IsaErasureCodeTest, minimum_to_decode)
+{
+ ErasureCodeIsaDefault Isa(tcache);
+ ErasureCodeProfile profile;
+ profile["k"] = "2";
+ profile["m"] = "2";
+ Isa.init(profile, &cerr);
+
+ //
+ // If trying to read nothing, the minimum is empty.
+ //
+ {
+ set<int> want_to_read;
+ set<int> available_chunks;
+ set<int> minimum;
+
+ EXPECT_EQ(0, Isa._minimum_to_decode(want_to_read,
+ available_chunks,
+ &minimum));
+ EXPECT_TRUE(minimum.empty());
+ }
+ //
+ // There is no way to read a chunk if none are available.
+ //
+ {
+ set<int> want_to_read;
+ set<int> available_chunks;
+ set<int> minimum;
+
+ want_to_read.insert(0);
+
+ EXPECT_EQ(-EIO, Isa._minimum_to_decode(want_to_read,
+ available_chunks,
+ &minimum));
+ }
+ //
+ // Reading a subset of the available chunks is always possible.
+ //
+ {
+ set<int> want_to_read;
+ set<int> available_chunks;
+ set<int> minimum;
+
+ want_to_read.insert(0);
+ available_chunks.insert(0);
+
+ EXPECT_EQ(0, Isa._minimum_to_decode(want_to_read,
+ available_chunks,
+ &minimum));
+ EXPECT_EQ(want_to_read, minimum);
+ }
+ //
+ // There is no way to read a missing chunk if there is less than k
+ // chunks available.
+ //
+ {
+ set<int> want_to_read;
+ set<int> available_chunks;
+ set<int> minimum;
+
+ want_to_read.insert(0);
+ want_to_read.insert(1);
+ available_chunks.insert(0);
+
+ EXPECT_EQ(-EIO, Isa._minimum_to_decode(want_to_read,
+ available_chunks,
+ &minimum));
+ }
+ //
+ // When chunks are not available, the minimum can be made of any
+ // chunks. For instance, to read 1 and 3 below the minimum could be
+ // 2 and 3 which may seem better because it contains one of the
+ // chunks to be read. But it won't be more efficient than retrieving
+ // 0 and 2 instead because, in both cases, the decode function will
+ // need to run the same recovery operation and use the same amount
+ // of CPU and memory.
+ //
+ {
+ set<int> want_to_read;
+ set<int> available_chunks;
+ set<int> minimum;
+
+ want_to_read.insert(1);
+ want_to_read.insert(3);
+ available_chunks.insert(0);
+ available_chunks.insert(2);
+ available_chunks.insert(3);
+
+ EXPECT_EQ(0, Isa._minimum_to_decode(want_to_read,
+ available_chunks,
+ &minimum));
+ EXPECT_EQ(2u, minimum.size());
+ EXPECT_EQ(0u, minimum.count(3));
+ }
+}
+
+TEST_F(IsaErasureCodeTest, chunk_size)
+{
+ {
+ ErasureCodeIsaDefault Isa(tcache);
+ ErasureCodeProfile profile;
+ profile["k"] = "2";
+ profile["m"] = "1";
+ Isa.init(profile, &cerr);
+ const int k = 2;
+
+ ASSERT_EQ(EC_ISA_ADDRESS_ALIGNMENT, Isa.get_chunk_size(1));
+ ASSERT_EQ(EC_ISA_ADDRESS_ALIGNMENT, Isa.get_chunk_size(EC_ISA_ADDRESS_ALIGNMENT * k - 1));
+ ASSERT_EQ(EC_ISA_ADDRESS_ALIGNMENT * 2, Isa.get_chunk_size(EC_ISA_ADDRESS_ALIGNMENT * k + 1));
+ }
+ {
+ ErasureCodeIsaDefault Isa(tcache);
+ ErasureCodeProfile profile;
+ profile["k"] = "3";
+ profile["m"] = "1";
+ Isa.init(profile, &cerr);
+ const int k = 3;
+
+ ASSERT_EQ(EC_ISA_ADDRESS_ALIGNMENT, Isa.get_chunk_size(1));
+ ASSERT_EQ(EC_ISA_ADDRESS_ALIGNMENT, Isa.get_chunk_size(EC_ISA_ADDRESS_ALIGNMENT * k - 1));
+ ASSERT_EQ(EC_ISA_ADDRESS_ALIGNMENT * 2, Isa.get_chunk_size(EC_ISA_ADDRESS_ALIGNMENT * k + 1));
+ unsigned object_size = EC_ISA_ADDRESS_ALIGNMENT * k * 1024 + 1;
+ ASSERT_NE(0u, object_size % k);
+ ASSERT_NE(0u, object_size % EC_ISA_ADDRESS_ALIGNMENT);
+ unsigned chunk_size = Isa.get_chunk_size(object_size);
+ ASSERT_EQ(0u, chunk_size % EC_ISA_ADDRESS_ALIGNMENT);
+ ASSERT_GT(chunk_size, (chunk_size * k) - object_size);
+ }
+}
+
+TEST_F(IsaErasureCodeTest, encode)
+{
+ ErasureCodeIsaDefault Isa(tcache);
+ ErasureCodeProfile profile;
+ profile["k"] = "2";
+ profile["m"] = "2";
+ Isa.init(profile, &cerr);
+
+ unsigned aligned_object_size = Isa.get_alignment() * 2;
+ {
+ //
+ // When the input bufferlist needs to be padded because
+ // it is not properly aligned, it is padded with zeros.
+ //
+ bufferlist in;
+ map<int,bufferlist> encoded;
+ int want_to_encode[] = { 0, 1, 2, 3 };
+ int trail_length = 1;
+ in.append(string(aligned_object_size + trail_length, 'X'));
+ EXPECT_EQ(0, Isa.encode(set<int>(want_to_encode, want_to_encode+4),
+ in,
+ &encoded));
+ EXPECT_EQ(4u, encoded.size());
+ char *last_chunk = encoded[1].c_str();
+ int length =encoded[1].length();
+ EXPECT_EQ('X', last_chunk[0]);
+ EXPECT_EQ('\0', last_chunk[length - trail_length]);
+ }
+
+ {
+ //
+ // When only the first chunk is required, the encoded map only
+ // contains the first chunk. Although the Isa encode
+ // internally allocated a buffer because of padding requirements
+ // and also computes the coding chunks, they are released before
+ // the return of the method, as shown when running the tests thru
+ // valgrind (there is no leak).
+ //
+ bufferlist in;
+ map<int,bufferlist> encoded;
+ set<int> want_to_encode;
+ want_to_encode.insert(0);
+ int trail_length = 1;
+ in.append(string(aligned_object_size + trail_length, 'X'));
+ EXPECT_EQ(0, Isa.encode(want_to_encode, in, &encoded));
+ EXPECT_EQ(1u, encoded.size());
+ }
+}
+
+TEST_F(IsaErasureCodeTest, sanity_check_k)
+{
+ ErasureCodeIsaDefault Isa(tcache);
+ ErasureCodeProfile profile;
+ profile["k"] = "1";
+ profile["m"] = "1";
+ ostringstream errors;
+ EXPECT_EQ(-EINVAL, Isa.init(profile, &errors));
+ EXPECT_NE(std::string::npos, errors.str().find("must be >= 2"));
+}
+
+bool
+DecodeAndVerify(ErasureCodeIsaDefault& Isa, map<int, bufferlist> &degraded, set<int> want_to_decode, buffer::ptr* enc, int length)
+{
+ map<int, bufferlist> decoded;
+ bool ok;
+
+ // decode as requested
+ ok = Isa._decode(want_to_decode,
+ degraded,
+ &decoded);
+
+ for (int i = 0; i < (int) decoded.size(); i++) {
+ // compare all the buffers with their original
+ ok |= memcmp(decoded[i].c_str(), enc[i].c_str(), length);
+ }
+
+ return ok;
+}
+
+TEST_F(IsaErasureCodeTest, isa_vandermonde_exhaustive)
+{
+ // Test all possible failure scenarios and reconstruction cases for
+ // a (12,4) configuration using the vandermonde matrix
+
+ ErasureCodeIsaDefault Isa(tcache);
+ ErasureCodeProfile profile;
+ profile["k"] = "12";
+ profile["m"] = "4";
+ Isa.init(profile, &cerr);
+
+ const int k = 12;
+ const int m = 4;
+
+#define LARGE_ENOUGH 2048
+ bufferptr in_ptr(buffer::create_page_aligned(LARGE_ENOUGH));
+ in_ptr.zero();
+ in_ptr.set_length(0);
+ const char *payload =
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+ in_ptr.append(payload, strlen(payload));
+ bufferlist in;
+ in.push_back(in_ptr);
+
+ set<int>want_to_encode;
+
+ map<int, bufferlist> encoded;
+ for (int i = 0; i < (k + m); i++) {
+ want_to_encode.insert(i);
+ }
+
+
+ EXPECT_EQ(0, Isa.encode(want_to_encode,
+ in,
+ &encoded));
+
+ EXPECT_EQ((unsigned) (k + m), encoded.size());
+
+ unsigned length = encoded[0].length();
+
+ for (int i = 0; i < k; i++) {
+ EXPECT_EQ(0, memcmp(encoded[i].c_str(), in.c_str() + (i * length), length));
+ }
+
+ buffer::ptr enc[k + m];
+ // create buffers with a copy of the original data to be able to compare it after decoding
+ {
+ for (int i = 0; i < (k + m); i++) {
+ buffer::ptr newenc(buffer::create_page_aligned(LARGE_ENOUGH));
+ enc[i] = newenc;
+ enc[i].zero();
+ enc[i].set_length(0);
+ enc[i].append(encoded[i].c_str(), length);
+ }
+ }
+
+ // loop through all possible loss scenarios
+ int cnt_cf = 0;
+
+ for (int l1 = 0; l1 < (k + m); l1++) {
+ map<int, bufferlist> degraded = encoded;
+ set<int> want_to_decode;
+ bool err;
+ degraded.erase(l1);
+ want_to_decode.insert(l1);
+ err = DecodeAndVerify(Isa, degraded, want_to_decode, enc, length);
+ EXPECT_EQ(0, err);
+ cnt_cf++;
+ for (int l2 = l1 + 1; l2 < (k + m); l2++) {
+ degraded.erase(l2);
+ want_to_decode.insert(l2);
+ err = DecodeAndVerify(Isa, degraded, want_to_decode, enc, length);
+ EXPECT_EQ(0, err);
+ cnt_cf++;
+ for (int l3 = l2 + 1; l3 < (k + m); l3++) {
+ degraded.erase(l3);
+ want_to_decode.insert(l3);
+ err = DecodeAndVerify(Isa, degraded, want_to_decode, enc, length);
+ EXPECT_EQ(0, err);
+ cnt_cf++;
+ for (int l4 = l3 + 1; l4 < (k + m); l4++) {
+ degraded.erase(l4);
+ want_to_decode.insert(l4);
+ err = DecodeAndVerify(Isa, degraded, want_to_decode, enc, length);
+ EXPECT_EQ(0, err);
+ degraded[l4] = encoded[l4];
+ want_to_decode.erase(l4);
+ cnt_cf++;
+ }
+ degraded[l3] = encoded[l3];
+ want_to_decode.erase(l3);
+ }
+ degraded[l2] = encoded[l2];
+ want_to_decode.erase(l2);
+ }
+ degraded[l1] = encoded[l1];
+ want_to_decode.erase(l1);
+ }
+ EXPECT_EQ(2516, cnt_cf);
+ EXPECT_EQ(2506, tcache.getDecodingTableCacheSize()); // 3 entries from (2,2) test and 2503 from (12,4)
+}
+
+TEST_F(IsaErasureCodeTest, isa_cauchy_exhaustive)
+{
+ // Test all possible failure scenarios and reconstruction cases for
+ // a (12,4) configuration using the cauchy matrix
+ ErasureCodeIsaDefault Isa(tcache,ErasureCodeIsaDefault::kCauchy);
+ ErasureCodeProfile profile;
+ profile["k"] = "12";
+ profile["m"] = "4";
+ profile["technique"] = "cauchy";
+
+ Isa.init(profile, &cerr);
+
+ const int k = 12;
+ const int m = 4;
+
+#define LARGE_ENOUGH 2048
+ bufferptr in_ptr(buffer::create_page_aligned(LARGE_ENOUGH));
+ in_ptr.zero();
+ in_ptr.set_length(0);
+ const char *payload =
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+ in_ptr.append(payload, strlen(payload));
+ bufferlist in;
+ in.push_back(in_ptr);
+
+ set<int>want_to_encode;
+
+ map<int, bufferlist> encoded;
+ for (int i = 0; i < (k + m); i++) {
+ want_to_encode.insert(i);
+ }
+
+
+ EXPECT_EQ(0, Isa.encode(want_to_encode,
+ in,
+ &encoded));
+
+ EXPECT_EQ((unsigned) (k + m), encoded.size());
+
+ unsigned length = encoded[0].length();
+
+ for (int i = 0; i < k; i++) {
+ EXPECT_EQ(0, memcmp(encoded[i].c_str(), in.c_str() + (i * length), length));
+ }
+
+ buffer::ptr enc[k + m];
+ // create buffers with a copy of the original data to be able to compare it after decoding
+ {
+ for (int i = 0; i < (k + m); i++) {
+ buffer::ptr newenc(buffer::create_page_aligned(LARGE_ENOUGH));
+ enc[i] = newenc;
+ enc[i].zero();
+ enc[i].set_length(0);
+ enc[i].append(encoded[i].c_str(), length);
+ }
+ }
+
+ // loop through all possible loss scenarios
+ int cnt_cf = 0;
+
+ for (int l1 = 0; l1 < (k + m); l1++) {
+ map<int, bufferlist> degraded = encoded;
+ set<int> want_to_decode;
+ bool err;
+ degraded.erase(l1);
+ want_to_decode.insert(l1);
+ err = DecodeAndVerify(Isa, degraded, want_to_decode, enc, length);
+ EXPECT_EQ(0, err);
+ cnt_cf++;
+ for (int l2 = l1 + 1; l2 < (k + m); l2++) {
+ degraded.erase(l2);
+ want_to_decode.insert(l2);
+ err = DecodeAndVerify(Isa, degraded, want_to_decode, enc, length);
+ EXPECT_EQ(0, err);
+ cnt_cf++;
+ for (int l3 = l2 + 1; l3 < (k + m); l3++) {
+ degraded.erase(l3);
+ want_to_decode.insert(l3);
+ err = DecodeAndVerify(Isa, degraded, want_to_decode, enc, length);
+ EXPECT_EQ(0, err);
+ cnt_cf++;
+ for (int l4 = l3 + 1; l4 < (k + m); l4++) {
+ degraded.erase(l4);
+ want_to_decode.insert(l4);
+ err = DecodeAndVerify(Isa, degraded, want_to_decode, enc, length);
+ EXPECT_EQ(0, err);
+ degraded[l4] = encoded[l4];
+ want_to_decode.erase(l4);
+ cnt_cf++;
+ }
+ degraded[l3] = encoded[l3];
+ want_to_decode.erase(l3);
+ }
+ degraded[l2] = encoded[l2];
+ want_to_decode.erase(l2);
+ }
+ degraded[l1] = encoded[l1];
+ want_to_decode.erase(l1);
+ }
+ EXPECT_EQ(2516, cnt_cf);
+ EXPECT_EQ(2516, tcache.getDecodingTableCacheSize(ErasureCodeIsaDefault::kCauchy));
+}
+
+TEST_F(IsaErasureCodeTest, isa_cauchy_cache_trash)
+{
+ // Test all possible failure scenarios and reconstruction cases for
+ // a (12,4) configuration using the cauchy matrix
+ ErasureCodeIsaDefault Isa(tcache,ErasureCodeIsaDefault::kCauchy);
+ ErasureCodeProfile profile;
+ profile["k"] = "16";
+ profile["m"] = "4";
+ profile["technique"] = "cauchy";
+
+ Isa.init(profile, &cerr);
+
+ const int k = 16;
+ const int m = 4;
+
+#define LARGE_ENOUGH 2048
+ bufferptr in_ptr(buffer::create_page_aligned(LARGE_ENOUGH));
+ in_ptr.zero();
+ in_ptr.set_length(0);
+ const char *payload =
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+ in_ptr.append(payload, strlen(payload));
+ bufferlist in;
+ in.push_back(in_ptr);
+
+ set<int>want_to_encode;
+
+ map<int, bufferlist> encoded;
+ for (int i = 0; i < (k + m); i++) {
+ want_to_encode.insert(i);
+ }
+
+
+ EXPECT_EQ(0, Isa.encode(want_to_encode,
+ in,
+ &encoded));
+
+ EXPECT_EQ((unsigned) (k + m), encoded.size());
+
+ unsigned length = encoded[0].length();
+
+ for (int i = 0; i < k; i++) {
+ EXPECT_EQ(0, memcmp(encoded[i].c_str(), in.c_str() + (i * length), length));
+ }
+
+ buffer::ptr enc[k + m];
+ // create buffers with a copy of the original data to be able to compare it after decoding
+ {
+ for (int i = 0; i < (k + m); i++) {
+ buffer::ptr newenc(buffer::create_page_aligned(LARGE_ENOUGH));
+ enc[i] = newenc;
+ enc[i].zero();
+ enc[i].set_length(0);
+ enc[i].append(encoded[i].c_str(), length);
+ }
+ }
+
+ // loop through all possible loss scenarios
+ int cnt_cf = 0;
+
+ for (int l1 = 0; l1 < (k + m); l1++) {
+ map<int, bufferlist> degraded = encoded;
+ set<int> want_to_decode;
+ bool err;
+ degraded.erase(l1);
+ want_to_decode.insert(l1);
+ err = DecodeAndVerify(Isa, degraded, want_to_decode, enc, length);
+ EXPECT_EQ(0, err);
+ cnt_cf++;
+ for (int l2 = l1 + 1; l2 < (k + m); l2++) {
+ degraded.erase(l2);
+ want_to_decode.insert(l2);
+ err = DecodeAndVerify(Isa, degraded, want_to_decode, enc, length);
+ EXPECT_EQ(0, err);
+ cnt_cf++;
+ for (int l3 = l2 + 1; l3 < (k + m); l3++) {
+ degraded.erase(l3);
+ want_to_decode.insert(l3);
+ err = DecodeAndVerify(Isa, degraded, want_to_decode, enc, length);
+ EXPECT_EQ(0, err);
+ cnt_cf++;
+ for (int l4 = l3 + 1; l4 < (k + m); l4++) {
+ degraded.erase(l4);
+ want_to_decode.insert(l4);
+ err = DecodeAndVerify(Isa, degraded, want_to_decode, enc, length);
+ EXPECT_EQ(0, err);
+ degraded[l4] = encoded[l4];
+ want_to_decode.erase(l4);
+ cnt_cf++;
+ }
+ degraded[l3] = encoded[l3];
+ want_to_decode.erase(l3);
+ }
+ degraded[l2] = encoded[l2];
+ want_to_decode.erase(l2);
+ }
+ degraded[l1] = encoded[l1];
+ want_to_decode.erase(l1);
+ }
+ EXPECT_EQ(6195, cnt_cf);
+ EXPECT_EQ(2516, tcache.getDecodingTableCacheSize(ErasureCodeIsaDefault::kCauchy));
+}
+
+TEST_F(IsaErasureCodeTest, isa_xor_codec)
+{
+ // Test all possible failure scenarios and reconstruction cases for
+ // a (4,1) RAID-5 like configuration
+
+ ErasureCodeIsaDefault Isa(tcache);
+ ErasureCodeProfile profile;
+ profile["k"] = "4";
+ profile["m"] = "1";
+ Isa.init(profile, &cerr);
+
+ const int k = 4;
+ const int m = 1;
+
+#define LARGE_ENOUGH 2048
+ bufferptr in_ptr(buffer::create_page_aligned(LARGE_ENOUGH));
+ in_ptr.zero();
+ in_ptr.set_length(0);
+ const char *payload =
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+ in_ptr.append(payload, strlen(payload));
+ bufferlist in;
+ in.push_back(in_ptr);
+
+ set<int>want_to_encode;
+
+ map<int, bufferlist> encoded;
+ for (int i = 0; i < (k + m); i++) {
+ want_to_encode.insert(i);
+ }
+
+
+ EXPECT_EQ(0, Isa.encode(want_to_encode,
+ in,
+ &encoded));
+
+ EXPECT_EQ((unsigned) (k + m), encoded.size());
+
+ unsigned length = encoded[0].length();
+
+ for (int i = 0; i < k; i++) {
+ EXPECT_EQ(0, memcmp(encoded[i].c_str(), in.c_str() + (i * length), length));
+ }
+
+ buffer::ptr enc[k + m];
+ // create buffers with a copy of the original data to be able to compare it after decoding
+ {
+ for (int i = 0; i < (k + m); i++) {
+ buffer::ptr newenc(buffer::create_page_aligned(LARGE_ENOUGH));
+ enc[i] = newenc;
+ enc[i].zero();
+ enc[i].set_length(0);
+ enc[i].append(encoded[i].c_str(), length);
+ }
+ }
+
+ // loop through all possible loss scenarios
+ int cnt_cf = 0;
+
+ for (int l1 = 0; l1 < (k + m); l1++) {
+ map<int, bufferlist> degraded = encoded;
+ set<int> want_to_decode;
+ bool err;
+ degraded.erase(l1);
+ want_to_decode.insert(l1);
+ err = DecodeAndVerify(Isa, degraded, want_to_decode, enc, length);
+ EXPECT_EQ(0, err);
+ cnt_cf++;
+ degraded[l1] = encoded[l1];
+ want_to_decode.erase(l1);
+ }
+ EXPECT_EQ(5, cnt_cf);
+}
+
+TEST_F(IsaErasureCodeTest, create_rule)
+{
+ std::unique_ptr<CrushWrapper> c = std::make_unique<CrushWrapper>();
+ c->create();
+ int root_type = 2;
+ c->set_type_name(root_type, "root");
+ int host_type = 1;
+ c->set_type_name(host_type, "host");
+ int osd_type = 0;
+ c->set_type_name(osd_type, "osd");
+
+ int rootno;
+ c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+ root_type, 0, NULL, NULL, &rootno);
+ c->set_item_name(rootno, "default");
+
+ map<string,string> loc;
+ loc["root"] = "default";
+
+ int num_host = 4;
+ int num_osd = 5;
+ int osd = 0;
+ for (int h=0; h<num_host; ++h) {
+ loc["host"] = string("host-") + stringify(h);
+ for (int o=0; o<num_osd; ++o, ++osd) {
+ c->insert_item(g_ceph_context, osd, 1.0, string("osd.") + stringify(osd), loc);
+ }
+ }
+
+ c->finalize();
+
+ {
+ stringstream ss;
+ ErasureCodeIsaDefault isa(tcache);
+ ErasureCodeProfile profile;
+ profile["k"] = "2";
+ profile["m"] = "2";
+ profile["w"] = "8";
+ isa.init(profile, &cerr);
+ int rule = isa.create_rule("myrule", *c, &ss);
+ EXPECT_EQ(0, rule);
+ EXPECT_EQ(-EEXIST, isa.create_rule("myrule", *c, &ss));
+ //
+ // the minimum that is expected from the created rule is to
+ // successfully map get_chunk_count() devices from the crushmap,
+ // at least once.
+ //
+ vector<__u32> weight(c->get_max_devices(), 0x10000);
+ vector<int> out;
+ int x = 0;
+ c->do_rule(rule, x, out, isa.get_chunk_count(), weight, 0);
+ ASSERT_EQ(out.size(), isa.get_chunk_count());
+ for (unsigned i=0; i<out.size(); ++i)
+ ASSERT_NE(CRUSH_ITEM_NONE, out[i]);
+ }
+ {
+ stringstream ss;
+ ErasureCodeIsaDefault isa(tcache);
+ ErasureCodeProfile profile;
+ profile["k"] = "2";
+ profile["m"] = "2";
+ profile["w"] = "8";
+ profile["crush-root"] = "BAD";
+ isa.init(profile, &cerr);
+ EXPECT_EQ(-ENOENT, isa.create_rule("otherrule", *c, &ss));
+ EXPECT_EQ("root item BAD does not exist", ss.str());
+ }
+ {
+ stringstream ss;
+ ErasureCodeIsaDefault isa(tcache);
+ ErasureCodeProfile profile;
+ profile["k"] = "2";
+ profile["m"] = "2";
+ profile["w"] = "8";
+ profile["crush-failure-domain"] = "WORSE";
+ isa.init(profile, &cerr);
+ EXPECT_EQ(-EINVAL, isa.create_rule("otherrule", *c, &ss));
+ EXPECT_EQ("unknown type WORSE", ss.str());
+ }
+}
+
+/*
+ * Local Variables:
+ * compile-command: "cd ../.. ; make -j4 unittest_erasure_code_isa &&
+ * libtool --mode=execute valgrind --tool=memcheck \
+ * ./unittest_erasure_code_isa \
+ * --gtest_filter=*.* --log-to-stderr=true --debug-osd=20"
+ * End:
+ */
diff --git a/src/test/erasure-code/TestErasureCodeJerasure.cc b/src/test/erasure-code/TestErasureCodeJerasure.cc
new file mode 100644
index 000000000..835f3c7b6
--- /dev/null
+++ b/src/test/erasure-code/TestErasureCodeJerasure.cc
@@ -0,0 +1,370 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph distributed storage system
+ *
+ * Copyright (C) 2013,2014 Cloudwatt <libre.licensing@cloudwatt.com>
+ * Copyright (C) 2014 Red Hat <contact@redhat.com>
+ *
+ * Author: Loic Dachary <loic@dachary.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#include <errno.h>
+#include <stdlib.h>
+
+#include "crush/CrushWrapper.h"
+#include "include/stringify.h"
+#include "erasure-code/jerasure/ErasureCodeJerasure.h"
+#include "global/global_context.h"
+#include "common/config.h"
+#include "gtest/gtest.h"
+
+using namespace std;
+
+template <typename T>
+class ErasureCodeTest : public ::testing::Test {
+ public:
+};
+
+typedef ::testing::Types<
+ ErasureCodeJerasureReedSolomonVandermonde,
+ ErasureCodeJerasureReedSolomonRAID6,
+ ErasureCodeJerasureCauchyOrig,
+ ErasureCodeJerasureCauchyGood,
+ ErasureCodeJerasureLiberation,
+ ErasureCodeJerasureBlaumRoth,
+ ErasureCodeJerasureLiber8tion
+> JerasureTypes;
+TYPED_TEST_SUITE(ErasureCodeTest, JerasureTypes);
+
+TYPED_TEST(ErasureCodeTest, sanity_check_k)
+{
+ TypeParam jerasure;
+ ErasureCodeProfile profile;
+ profile["k"] = "1";
+ profile["m"] = "1";
+ profile["packetsize"] = "8";
+ ostringstream errors;
+ EXPECT_EQ(-EINVAL, jerasure.init(profile, &errors));
+ EXPECT_NE(std::string::npos, errors.str().find("must be >= 2"));
+}
+
+TYPED_TEST(ErasureCodeTest, encode_decode)
+{
+ const char *per_chunk_alignments[] = { "false", "true" };
+ for (int per_chunk_alignment = 0 ;
+ per_chunk_alignment < 2;
+ per_chunk_alignment++) {
+ TypeParam jerasure;
+ ErasureCodeProfile profile;
+ profile["k"] = "2";
+ profile["m"] = "2";
+ profile["packetsize"] = "8";
+ profile["jerasure-per-chunk-alignment"] =
+ per_chunk_alignments[per_chunk_alignment];
+ jerasure.init(profile, &cerr);
+
+#define LARGE_ENOUGH 2048
+ bufferptr in_ptr(buffer::create_page_aligned(LARGE_ENOUGH));
+ in_ptr.zero();
+ in_ptr.set_length(0);
+ const char *payload =
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+ in_ptr.append(payload, strlen(payload));
+ bufferlist in;
+ in.push_back(in_ptr);
+ int want_to_encode[] = { 0, 1, 2, 3 };
+ map<int, bufferlist> encoded;
+ EXPECT_EQ(0, jerasure.encode(set<int>(want_to_encode, want_to_encode+4),
+ in,
+ &encoded));
+ EXPECT_EQ(4u, encoded.size());
+ unsigned length = encoded[0].length();
+ EXPECT_EQ(0, memcmp(encoded[0].c_str(), in.c_str(), length));
+ EXPECT_EQ(0, memcmp(encoded[1].c_str(), in.c_str() + length,
+ in.length() - length));
+
+
+ // all chunks are available
+ {
+ int want_to_decode[] = { 0, 1 };
+ map<int, bufferlist> decoded;
+ EXPECT_EQ(0, jerasure._decode(set<int>(want_to_decode, want_to_decode+2),
+ encoded,
+ &decoded));
+ EXPECT_EQ(2u, decoded.size());
+ EXPECT_EQ(length, decoded[0].length());
+ EXPECT_EQ(0, memcmp(decoded[0].c_str(), in.c_str(), length));
+ EXPECT_EQ(0, memcmp(decoded[1].c_str(), in.c_str() + length,
+ in.length() - length));
+ }
+
+ // two chunks are missing
+ {
+ map<int, bufferlist> degraded = encoded;
+ degraded.erase(0);
+ degraded.erase(1);
+ EXPECT_EQ(2u, degraded.size());
+ int want_to_decode[] = { 0, 1 };
+ map<int, bufferlist> decoded;
+ EXPECT_EQ(0, jerasure._decode(set<int>(want_to_decode, want_to_decode+2),
+ degraded,
+ &decoded));
+ // always decode all, regardless of want_to_decode
+ EXPECT_EQ(4u, decoded.size());
+ EXPECT_EQ(length, decoded[0].length());
+ EXPECT_EQ(0, memcmp(decoded[0].c_str(), in.c_str(), length));
+ EXPECT_EQ(0, memcmp(decoded[1].c_str(), in.c_str() + length,
+ in.length() - length));
+ }
+ }
+}
+
+TYPED_TEST(ErasureCodeTest, minimum_to_decode)
+{
+ TypeParam jerasure;
+ ErasureCodeProfile profile;
+ profile["k"] = "2";
+ profile["m"] = "2";
+ profile["w"] = "7";
+ profile["packetsize"] = "8";
+ jerasure.init(profile, &cerr);
+
+ //
+ // If trying to read nothing, the minimum is empty.
+ //
+ {
+ set<int> want_to_read;
+ set<int> available_chunks;
+ set<int> minimum;
+
+ EXPECT_EQ(0, jerasure._minimum_to_decode(want_to_read,
+ available_chunks,
+ &minimum));
+ EXPECT_TRUE(minimum.empty());
+ }
+ //
+ // There is no way to read a chunk if none are available.
+ //
+ {
+ set<int> want_to_read;
+ set<int> available_chunks;
+ set<int> minimum;
+
+ want_to_read.insert(0);
+
+ EXPECT_EQ(-EIO, jerasure._minimum_to_decode(want_to_read,
+ available_chunks,
+ &minimum));
+ }
+ //
+ // Reading a subset of the available chunks is always possible.
+ //
+ {
+ set<int> want_to_read;
+ set<int> available_chunks;
+ set<int> minimum;
+
+ want_to_read.insert(0);
+ available_chunks.insert(0);
+
+ EXPECT_EQ(0, jerasure._minimum_to_decode(want_to_read,
+ available_chunks,
+ &minimum));
+ EXPECT_EQ(want_to_read, minimum);
+ }
+ //
+ // There is no way to read a missing chunk if there is less than k
+ // chunks available.
+ //
+ {
+ set<int> want_to_read;
+ set<int> available_chunks;
+ set<int> minimum;
+
+ want_to_read.insert(0);
+ want_to_read.insert(1);
+ available_chunks.insert(0);
+
+ EXPECT_EQ(-EIO, jerasure._minimum_to_decode(want_to_read,
+ available_chunks,
+ &minimum));
+ }
+ //
+ // When chunks are not available, the minimum can be made of any
+ // chunks. For instance, to read 1 and 3 below the minimum could be
+ // 2 and 3 which may seem better because it contains one of the
+ // chunks to be read. But it won't be more efficient than retrieving
+ // 0 and 2 instead because, in both cases, the decode function will
+ // need to run the same recovery operation and use the same amount
+ // of CPU and memory.
+ //
+ {
+ set<int> want_to_read;
+ set<int> available_chunks;
+ set<int> minimum;
+
+ want_to_read.insert(1);
+ want_to_read.insert(3);
+ available_chunks.insert(0);
+ available_chunks.insert(2);
+ available_chunks.insert(3);
+
+ EXPECT_EQ(0, jerasure._minimum_to_decode(want_to_read,
+ available_chunks,
+ &minimum));
+ EXPECT_EQ(2u, minimum.size());
+ EXPECT_EQ(0u, minimum.count(3));
+ }
+}
+
+TEST(ErasureCodeTest, encode)
+{
+ ErasureCodeJerasureReedSolomonVandermonde jerasure;
+ ErasureCodeProfile profile;
+ profile["k"] = "2";
+ profile["m"] = "2";
+ profile["w"] = "8";
+ jerasure.init(profile, &cerr);
+
+ unsigned aligned_object_size = jerasure.get_alignment() * 2;
+ {
+ //
+ // When the input bufferlist needs to be padded because
+ // it is not properly aligned, it is padded with zeros.
+ //
+ bufferlist in;
+ map<int,bufferlist> encoded;
+ int want_to_encode[] = { 0, 1, 2, 3 };
+ int trail_length = 1;
+ in.append(string(aligned_object_size + trail_length, 'X'));
+ EXPECT_EQ(0, jerasure.encode(set<int>(want_to_encode, want_to_encode+4),
+ in,
+ &encoded));
+ EXPECT_EQ(4u, encoded.size());
+ char *last_chunk = encoded[1].c_str();
+ int length =encoded[1].length();
+ EXPECT_EQ('X', last_chunk[0]);
+ EXPECT_EQ('\0', last_chunk[length - trail_length]);
+ }
+
+ {
+ //
+ // When only the first chunk is required, the encoded map only
+ // contains the first chunk. Although the jerasure encode
+ // internally allocated a buffer because of padding requirements
+ // and also computes the coding chunks, they are released before
+ // the return of the method, as shown when running the tests thru
+ // valgrind (there is no leak).
+ //
+ bufferlist in;
+ map<int,bufferlist> encoded;
+ set<int> want_to_encode;
+ want_to_encode.insert(0);
+ int trail_length = 1;
+ in.append(string(aligned_object_size + trail_length, 'X'));
+ EXPECT_EQ(0, jerasure.encode(want_to_encode, in, &encoded));
+ EXPECT_EQ(1u, encoded.size());
+ }
+}
+
+TEST(ErasureCodeTest, create_rule)
+{
+ std::unique_ptr<CrushWrapper> c = std::make_unique<CrushWrapper>();
+ c->create();
+ int root_type = 2;
+ c->set_type_name(root_type, "root");
+ int host_type = 1;
+ c->set_type_name(host_type, "host");
+ int osd_type = 0;
+ c->set_type_name(osd_type, "osd");
+
+ int rootno;
+ c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+ root_type, 0, NULL, NULL, &rootno);
+ c->set_item_name(rootno, "default");
+
+ map<string,string> loc;
+ loc["root"] = "default";
+
+ int num_host = 4;
+ int num_osd = 5;
+ int osd = 0;
+ for (int h=0; h<num_host; ++h) {
+ loc["host"] = string("host-") + stringify(h);
+ for (int o=0; o<num_osd; ++o, ++osd) {
+ c->insert_item(g_ceph_context, osd, 1.0, string("osd.") + stringify(osd), loc);
+ }
+ }
+
+ c->finalize();
+
+ {
+ stringstream ss;
+ ErasureCodeJerasureReedSolomonVandermonde jerasure;
+ ErasureCodeProfile profile;
+ profile["k"] = "2";
+ profile["m"] = "2";
+ profile["w"] = "8";
+ jerasure.init(profile, &cerr);
+ int rule = jerasure.create_rule("myrule", *c, &ss);
+ EXPECT_EQ(0, rule);
+ EXPECT_EQ(-EEXIST, jerasure.create_rule("myrule", *c, &ss));
+ //
+ // the minimum that is expected from the created rule is to
+ // successfully map get_chunk_count() devices from the crushmap,
+ // at least once.
+ //
+ vector<__u32> weight(c->get_max_devices(), 0x10000);
+ vector<int> out;
+ int x = 0;
+ c->do_rule(rule, x, out, jerasure.get_chunk_count(), weight, 0);
+ ASSERT_EQ(out.size(), jerasure.get_chunk_count());
+ for (unsigned i=0; i<out.size(); ++i)
+ ASSERT_NE(CRUSH_ITEM_NONE, out[i]);
+ }
+ {
+ stringstream ss;
+ ErasureCodeJerasureReedSolomonVandermonde jerasure;
+ ErasureCodeProfile profile;
+ profile["k"] = "2";
+ profile["m"] = "2";
+ profile["w"] = "8";
+ profile["crush-root"] = "BAD";
+ jerasure.init(profile, &cerr);
+ EXPECT_EQ(-ENOENT, jerasure.create_rule("otherrule", *c, &ss));
+ EXPECT_EQ("root item BAD does not exist", ss.str());
+ }
+ {
+ stringstream ss;
+ ErasureCodeJerasureReedSolomonVandermonde jerasure;
+ ErasureCodeProfile profile;
+ profile["k"] = "2";
+ profile["m"] = "2";
+ profile["w"] = "8";
+ profile["crush-failure-domain"] = "WORSE";
+ jerasure.init(profile, &cerr);
+ EXPECT_EQ(-EINVAL, jerasure.create_rule("otherrule", *c, &ss));
+ EXPECT_EQ("unknown type WORSE", ss.str());
+ }
+}
+
+/*
+ * Local Variables:
+ * compile-command: "cd ../.. ;
+ * make -j4 unittest_erasure_code_jerasure &&
+ * valgrind --tool=memcheck \
+ * ./unittest_erasure_code_jerasure \
+ * --gtest_filter=*.* --log-to-stderr=true --debug-osd=20"
+ * End:
+ */
diff --git a/src/test/erasure-code/TestErasureCodeLrc.cc b/src/test/erasure-code/TestErasureCodeLrc.cc
new file mode 100644
index 000000000..aca6ccae9
--- /dev/null
+++ b/src/test/erasure-code/TestErasureCodeLrc.cc
@@ -0,0 +1,925 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph distributed storage system
+ *
+ * Copyright (C) 2014 Cloudwatt <libre.licensing@cloudwatt.com>
+ * Copyright (C) 2014 Red Hat <contact@redhat.com>
+ *
+ * Author: Loic Dachary <loic@dachary.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#include <errno.h>
+#include <stdlib.h>
+
+#include "crush/CrushWrapper.h"
+#include "include/stringify.h"
+#include "erasure-code/lrc/ErasureCodeLrc.h"
+#include "global/global_context.h"
+#include "common/config_proxy.h"
+#include "gtest/gtest.h"
+
+using namespace std;
+
+TEST(ErasureCodeLrc, parse_rule)
+{
+ ErasureCodeLrc lrc(g_conf().get_val<std::string>("erasure_code_dir"));
+ EXPECT_EQ("default", lrc.rule_root);
+ EXPECT_EQ("host", lrc.rule_steps.front().type);
+
+ ErasureCodeProfile profile;
+ profile["crush-root"] = "other";
+ EXPECT_EQ(0, lrc.parse_rule(profile, &cerr));
+ EXPECT_EQ("other", lrc.rule_root);
+
+ profile["crush-steps"] = "[]";
+ EXPECT_EQ(0, lrc.parse_rule(profile, &cerr));
+ EXPECT_TRUE(lrc.rule_steps.empty());
+
+ profile["crush-steps"] = "0";
+ EXPECT_EQ(ERROR_LRC_ARRAY, lrc.parse_rule(profile, &cerr));
+
+ profile["crush-steps"] = "{";
+ EXPECT_EQ(ERROR_LRC_PARSE_JSON, lrc.parse_rule(profile, &cerr));
+
+ profile["crush-steps"] = "[0]";
+ EXPECT_EQ(ERROR_LRC_ARRAY, lrc.parse_rule(profile, &cerr));
+
+ profile["crush-steps"] = "[[0]]";
+ EXPECT_EQ(ERROR_LRC_RULE_OP, lrc.parse_rule(profile, &cerr));
+
+ profile["crush-steps"] = "[[\"choose\", 0]]";
+ EXPECT_EQ(ERROR_LRC_RULE_TYPE, lrc.parse_rule(profile, &cerr));
+
+ profile["crush-steps"] = "[[\"choose\", \"host\", []]]";
+ EXPECT_EQ(ERROR_LRC_RULE_N, lrc.parse_rule(profile, &cerr));
+
+ profile["crush-steps"] = "[[\"choose\", \"host\", 2]]";
+ EXPECT_EQ(0, lrc.parse_rule(profile, &cerr));
+
+ const ErasureCodeLrc::Step &step = lrc.rule_steps.front();
+ EXPECT_EQ("choose", step.op);
+ EXPECT_EQ("host", step.type);
+ EXPECT_EQ(2, step.n);
+
+ profile["crush-steps"] =
+ "["
+ " [\"choose\", \"rack\", 2], "
+ " [\"chooseleaf\", \"host\", 5], "
+ "]";
+ EXPECT_EQ(0, lrc.parse_rule(profile, &cerr));
+ EXPECT_EQ(2U, lrc.rule_steps.size());
+ {
+ const ErasureCodeLrc::Step &step = lrc.rule_steps[0];
+ EXPECT_EQ("choose", step.op);
+ EXPECT_EQ("rack", step.type);
+ EXPECT_EQ(2, step.n);
+ }
+ {
+ const ErasureCodeLrc::Step &step = lrc.rule_steps[1];
+ EXPECT_EQ("chooseleaf", step.op);
+ EXPECT_EQ("host", step.type);
+ EXPECT_EQ(5, step.n);
+ }
+}
+
+TEST(ErasureCodeTest, create_rule)
+{
+ CrushWrapper *c = new CrushWrapper;
+ c->create();
+ int root_type = 3;
+ c->set_type_name(root_type, "root");
+ int rack_type = 2;
+ c->set_type_name(rack_type, "rack");
+ int host_type = 1;
+ c->set_type_name(host_type, "host");
+ int osd_type = 0;
+ c->set_type_name(osd_type, "osd");
+
+ int rootno;
+ c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+ root_type, 0, NULL, NULL, &rootno);
+ c->set_item_name(rootno, "default");
+
+ map<string,string> loc;
+ loc["root"] = "default";
+
+ //
+ // Set all to 10 so that the item number it trivial to decompose
+ // into rack/host/osd.
+ //
+ int num_rack;
+ int num_host;
+ int num_osd;
+ num_rack = num_host = num_osd = 10;
+ int osd = 0;
+ for (int r=0; r<num_rack; ++r) {
+ loc["rack"] = string("rack-") + stringify(r);
+ for (int h=0; h<num_host; ++h) {
+ loc["host"] = string("host-") + stringify(r) + string("-") + stringify(h);
+ for (int o=0; o<num_osd; ++o, ++osd) {
+ c->insert_item(g_ceph_context, osd, 1.0, string("osd.") + stringify(osd), loc);
+ }
+ }
+ }
+
+ c->finalize();
+
+ ErasureCodeLrc lrc(g_conf().get_val<std::string>("erasure_code_dir"));
+ EXPECT_EQ(0, lrc.create_rule("rule1", *c, &cerr));
+
+ ErasureCodeProfile profile;
+ unsigned int racks = 2;
+ unsigned int hosts = 5;
+ profile["crush-steps"] =
+ "["
+ " [\"choose\", \"rack\", " + stringify(racks) + "], "
+ " [\"chooseleaf\", \"host\", " + stringify(hosts) + "], "
+ "]";
+ const char *rule_name = "rule2";
+ EXPECT_EQ(0, lrc.parse_rule(profile, &cerr));
+ EXPECT_EQ(1, lrc.create_rule(rule_name, *c, &cerr));
+
+ vector<__u32> weight;
+ for (int o = 0; o < c->get_max_devices(); o++)
+ weight.push_back(0x10000);
+ int rule = c->get_rule_id(rule_name);
+ vector<int> out;
+ unsigned int n = racks * hosts;
+ c->do_rule(rule, 1, out, n, weight, 0);
+ EXPECT_EQ(n, out.size());
+ //
+ // check that the first five are in the same rack and the next five
+ // in the same rack
+ //
+ int first_rack = out[0] / num_host / num_osd;
+ EXPECT_EQ(first_rack, out[1] / num_host / num_osd);
+ EXPECT_EQ(first_rack, out[2] / num_host / num_osd);
+ EXPECT_EQ(first_rack, out[3] / num_host / num_osd);
+ EXPECT_EQ(first_rack, out[4] / num_host / num_osd);
+ int second_rack = out[5] / num_host / num_osd;
+ EXPECT_EQ(second_rack, out[6] / num_host / num_osd);
+ EXPECT_EQ(second_rack, out[7] / num_host / num_osd);
+ EXPECT_EQ(second_rack, out[8] / num_host / num_osd);
+ EXPECT_EQ(second_rack, out[9] / num_host / num_osd);
+}
+
+TEST(ErasureCodeLrc, parse_kml)
+{
+ ErasureCodeLrc lrc(g_conf().get_val<std::string>("erasure_code_dir"));
+ ErasureCodeProfile profile;
+ EXPECT_EQ(0, lrc.parse_kml(profile, &cerr));
+ profile["k"] = "4";
+ EXPECT_EQ(ERROR_LRC_ALL_OR_NOTHING, lrc.parse_kml(profile, &cerr));
+ const char *generated[] = { "mapping",
+ "layers",
+ "crush-steps" };
+ profile["m"] = "2";
+ profile["l"] = "3";
+
+ for (int i = 0; i < 3; i++) {
+ profile[generated[i]] = "SET";
+ EXPECT_EQ(ERROR_LRC_GENERATED, lrc.parse_kml(profile, &cerr));
+ profile.erase(profile.find(generated[i]));
+ }
+
+ profile["k"] = "4";
+ profile["m"] = "2";
+ profile["l"] = "7";
+ EXPECT_EQ(ERROR_LRC_K_M_MODULO, lrc.parse_kml(profile, &cerr));
+
+ profile["k"] = "3";
+ profile["m"] = "3";
+ profile["l"] = "3";
+ EXPECT_EQ(ERROR_LRC_K_MODULO, lrc.parse_kml(profile, &cerr));
+
+ profile["k"] = "4";
+ profile["m"] = "2";
+ profile["l"] = "3";
+ EXPECT_EQ(0, lrc.parse_kml(profile, &cerr));
+ EXPECT_EQ("[ "
+ " [ \"DDc_DDc_\", \"\" ],"
+ " [ \"DDDc____\", \"\" ],"
+ " [ \"____DDDc\", \"\" ],"
+ "]", profile["layers"]);
+ EXPECT_EQ("DD__DD__", profile["mapping"]);
+ EXPECT_EQ("chooseleaf", lrc.rule_steps[0].op);
+ EXPECT_EQ("host", lrc.rule_steps[0].type);
+ EXPECT_EQ(0, lrc.rule_steps[0].n);
+ EXPECT_EQ(1U, lrc.rule_steps.size());
+ profile.erase(profile.find("mapping"));
+ profile.erase(profile.find("layers"));
+
+ profile["k"] = "4";
+ profile["m"] = "2";
+ profile["l"] = "3";
+ profile["crush-failure-domain"] = "osd";
+ EXPECT_EQ(0, lrc.parse_kml(profile, &cerr));
+ EXPECT_EQ("chooseleaf", lrc.rule_steps[0].op);
+ EXPECT_EQ("osd", lrc.rule_steps[0].type);
+ EXPECT_EQ(0, lrc.rule_steps[0].n);
+ EXPECT_EQ(1U, lrc.rule_steps.size());
+ profile.erase(profile.find("mapping"));
+ profile.erase(profile.find("layers"));
+
+ profile["k"] = "4";
+ profile["m"] = "2";
+ profile["l"] = "3";
+ profile["crush-failure-domain"] = "osd";
+ profile["crush-locality"] = "rack";
+ EXPECT_EQ(0, lrc.parse_kml(profile, &cerr));
+ EXPECT_EQ("choose", lrc.rule_steps[0].op);
+ EXPECT_EQ("rack", lrc.rule_steps[0].type);
+ EXPECT_EQ(2, lrc.rule_steps[0].n);
+ EXPECT_EQ("chooseleaf", lrc.rule_steps[1].op);
+ EXPECT_EQ("osd", lrc.rule_steps[1].type);
+ EXPECT_EQ(4, lrc.rule_steps[1].n);
+ EXPECT_EQ(2U, lrc.rule_steps.size());
+ profile.erase(profile.find("mapping"));
+ profile.erase(profile.find("layers"));
+}
+
+TEST(ErasureCodeLrc, layers_description)
+{
+ ErasureCodeLrc lrc(g_conf().get_val<std::string>("erasure_code_dir"));
+ ErasureCodeProfile profile;
+
+ json_spirit::mArray description;
+ EXPECT_EQ(ERROR_LRC_DESCRIPTION,
+ lrc.layers_description(profile, &description, &cerr));
+
+ {
+ const char *description_string = "\"not an array\"";
+ profile["layers"] = description_string;
+ EXPECT_EQ(ERROR_LRC_ARRAY,
+ lrc.layers_description(profile, &description, &cerr));
+ }
+ {
+ const char *description_string = "invalid json";
+ profile["layers"] = description_string;
+ EXPECT_EQ(ERROR_LRC_PARSE_JSON,
+ lrc.layers_description(profile, &description, &cerr));
+ }
+ {
+ const char *description_string = "[]";
+ profile["layers"] = description_string;
+ EXPECT_EQ(0, lrc.layers_description(profile, &description, &cerr));
+ }
+}
+
+TEST(ErasureCodeLrc, layers_parse)
+{
+ {
+ ErasureCodeLrc lrc(g_conf().get_val<std::string>("erasure_code_dir"));
+ ErasureCodeProfile profile;
+
+ const char *description_string ="[ 0 ]";
+ profile["layers"] = description_string;
+ json_spirit::mArray description;
+ EXPECT_EQ(0, lrc.layers_description(profile, &description, &cerr));
+ EXPECT_EQ(ERROR_LRC_ARRAY,
+ lrc.layers_parse(description_string, description, &cerr));
+ }
+
+ {
+ ErasureCodeLrc lrc(g_conf().get_val<std::string>("erasure_code_dir"));
+ ErasureCodeProfile profile;
+
+ const char *description_string ="[ [ 0 ] ]";
+ profile["layers"] = description_string;
+ json_spirit::mArray description;
+ EXPECT_EQ(0, lrc.layers_description(profile, &description, &cerr));
+ EXPECT_EQ(ERROR_LRC_STR,
+ lrc.layers_parse(description_string, description, &cerr));
+ }
+
+ {
+ ErasureCodeLrc lrc(g_conf().get_val<std::string>("erasure_code_dir"));
+ ErasureCodeProfile profile;
+
+ const char *description_string ="[ [ \"\", 0 ] ]";
+ profile["layers"] = description_string;
+ json_spirit::mArray description;
+ EXPECT_EQ(0, lrc.layers_description(profile, &description, &cerr));
+ EXPECT_EQ(ERROR_LRC_CONFIG_OPTIONS,
+ lrc.layers_parse(description_string, description, &cerr));
+ }
+
+ //
+ // The second element can be an object describing the plugin
+ // profile.
+ //
+ {
+ ErasureCodeLrc lrc(g_conf().get_val<std::string>("erasure_code_dir"));
+ ErasureCodeProfile profile;
+
+ const char *description_string ="[ [ \"\", { \"a\": \"b\" }, \"ignored\" ] ]";
+ profile["layers"] = description_string;
+ json_spirit::mArray description;
+ EXPECT_EQ(0, lrc.layers_description(profile, &description, &cerr));
+ EXPECT_EQ(0, lrc.layers_parse(description_string, description, &cerr));
+ EXPECT_EQ("b", lrc.layers.front().profile["a"]);
+ }
+
+ //
+ // The second element can be a str_map parseable string describing the plugin
+ // profile.
+ //
+ {
+ ErasureCodeLrc lrc(g_conf().get_val<std::string>("erasure_code_dir"));
+ ErasureCodeProfile profile;
+
+ const char *description_string ="[ [ \"\", \"a=b c=d\" ] ]";
+ profile["layers"] = description_string;
+ json_spirit::mArray description;
+ EXPECT_EQ(0, lrc.layers_description(profile, &description, &cerr));
+ EXPECT_EQ(0, lrc.layers_parse(description_string, description, &cerr));
+ EXPECT_EQ("b", lrc.layers.front().profile["a"]);
+ EXPECT_EQ("d", lrc.layers.front().profile["c"]);
+ }
+
+}
+
+TEST(ErasureCodeLrc, layers_sanity_checks)
+{
+ {
+ ErasureCodeLrc lrc(g_conf().get_val<std::string>("erasure_code_dir"));
+ ErasureCodeProfile profile;
+ profile["mapping"] =
+ "__DDD__DD";
+ const char *description_string =
+ "[ "
+ " [ \"_cDDD_cDD\", \"\" ],"
+ " [ \"c_DDD____\", \"\" ],"
+ " [ \"_____cDDD\", \"\" ],"
+ "]";
+ profile["layers"] = description_string;
+ EXPECT_EQ(0, lrc.init(profile, &cerr));
+ }
+ {
+ ErasureCodeLrc lrc(g_conf().get_val<std::string>("erasure_code_dir"));
+ ErasureCodeProfile profile;
+ const char *description_string =
+ "[ "
+ "]";
+ profile["layers"] = description_string;
+ EXPECT_EQ(ERROR_LRC_MAPPING, lrc.init(profile, &cerr));
+ }
+ {
+ ErasureCodeLrc lrc(g_conf().get_val<std::string>("erasure_code_dir"));
+ ErasureCodeProfile profile;
+ profile["mapping"] = "";
+ const char *description_string =
+ "[ "
+ "]";
+ profile["layers"] = description_string;
+ EXPECT_EQ(ERROR_LRC_LAYERS_COUNT, lrc.init(profile, &cerr));
+ }
+ {
+ ErasureCodeLrc lrc(g_conf().get_val<std::string>("erasure_code_dir"));
+ ErasureCodeProfile profile;
+ profile["mapping"] =
+ "DD";
+ const char *description_string =
+ "[ "
+ " [ \"DD??\", \"\" ], "
+ " [ \"DD\", \"\" ], "
+ " [ \"DD\", \"\" ], "
+ "]";
+ profile["layers"] = description_string;
+ EXPECT_EQ(-EINVAL, lrc.init(profile, &cerr));
+ }
+}
+
+TEST(ErasureCodeLrc, layers_init)
+{
+ {
+ ErasureCodeLrc lrc(g_conf().get_val<std::string>("erasure_code_dir"));
+ ErasureCodeProfile profile;
+
+ const char* env = getenv("CEPH_LIB");
+ string directory(env ? env : "lib");
+ string description_string =
+ "[ "
+ " [ \"_cDDD_cDD_\", \"directory=" + directory + "\" ],"
+ "]";
+ profile["layers"] = description_string;
+ json_spirit::mArray description;
+ EXPECT_EQ(0, lrc.layers_description(profile, &description, &cerr));
+ EXPECT_EQ(0, lrc.layers_parse(description_string, description, &cerr));
+ EXPECT_EQ(0, lrc.layers_init(&cerr));
+ EXPECT_EQ("5", lrc.layers.front().profile["k"]);
+ EXPECT_EQ("2", lrc.layers.front().profile["m"]);
+ EXPECT_EQ("jerasure", lrc.layers.front().profile["plugin"]);
+ EXPECT_EQ("reed_sol_van", lrc.layers.front().profile["technique"]);
+ }
+}
+
+TEST(ErasureCodeLrc, init)
+{
+ ErasureCodeLrc lrc(g_conf().get_val<std::string>("erasure_code_dir"));
+ ErasureCodeProfile profile;
+ profile["mapping"] =
+ "__DDD__DD";
+ const char *description_string =
+ "[ "
+ " [ \"_cDDD_cDD\", \"\" ],"
+ " [ \"c_DDD____\", \"\" ],"
+ " [ \"_____cDDD\", \"\" ],"
+ "]";
+ profile["layers"] = description_string;
+ EXPECT_EQ(0, lrc.init(profile, &cerr));
+}
+
+TEST(ErasureCodeLrc, init_kml)
+{
+ ErasureCodeLrc lrc(g_conf().get_val<std::string>("erasure_code_dir"));
+ ErasureCodeProfile profile;
+ profile["k"] = "4";
+ profile["m"] = "2";
+ profile["l"] = "3";
+ EXPECT_EQ(0, lrc.init(profile, &cerr));
+ EXPECT_EQ((unsigned int)(4 + 2 + (4 + 2) / 3), lrc.get_chunk_count());
+}
+
+TEST(ErasureCodeLrc, minimum_to_decode)
+{
+ // trivial : no erasures, the minimum is want_to_read
+ {
+ ErasureCodeLrc lrc(g_conf().get_val<std::string>("erasure_code_dir"));
+ ErasureCodeProfile profile;
+ profile["mapping"] =
+ "__DDD__DD";
+ const char *description_string =
+ "[ "
+ " [ \"_cDDD_cDD\", \"\" ],"
+ " [ \"c_DDD____\", \"\" ],"
+ " [ \"_____cDDD\", \"\" ],"
+ "]";
+ profile["layers"] = description_string;
+ EXPECT_EQ(0, lrc.init(profile, &cerr));
+ set<int> want_to_read;
+ want_to_read.insert(1);
+ set<int> available_chunks;
+ available_chunks.insert(1);
+ available_chunks.insert(2);
+ set<int> minimum;
+ EXPECT_EQ(0, lrc._minimum_to_decode(want_to_read, available_chunks, &minimum));
+ EXPECT_EQ(want_to_read, minimum);
+ }
+ // locally repairable erasure
+ {
+ ErasureCodeLrc lrc(g_conf().get_val<std::string>("erasure_code_dir"));
+ ErasureCodeProfile profile;
+ profile["mapping"] =
+ "__DDD__DD_";
+ const char *description_string =
+ "[ "
+ " [ \"_cDDD_cDD_\", \"\" ],"
+ " [ \"c_DDD_____\", \"\" ],"
+ " [ \"_____cDDD_\", \"\" ],"
+ " [ \"_____DDDDc\", \"\" ],"
+ "]";
+ profile["layers"] = description_string;
+ EXPECT_EQ(0, lrc.init(profile, &cerr));
+ EXPECT_EQ(profile["mapping"].length(),
+ lrc.get_chunk_count());
+ {
+ // want to read the last chunk
+ set<int> want_to_read;
+ want_to_read.insert(lrc.get_chunk_count() - 1);
+ // all chunks are available except the last chunk
+ set<int> available_chunks;
+ for (int i = 0; i < (int)lrc.get_chunk_count() - 1; i++)
+ available_chunks.insert(i);
+ // _____DDDDc can recover c
+ set<int> minimum;
+ EXPECT_EQ(0, lrc._minimum_to_decode(want_to_read, available_chunks, &minimum));
+ set<int> expected_minimum;
+ expected_minimum.insert(5);
+ expected_minimum.insert(6);
+ expected_minimum.insert(7);
+ expected_minimum.insert(8);
+ EXPECT_EQ(expected_minimum, minimum);
+ }
+ {
+ set<int> want_to_read;
+ want_to_read.insert(0);
+ set<int> available_chunks;
+ for (int i = 1; i < (int)lrc.get_chunk_count(); i++)
+ available_chunks.insert(i);
+ set<int> minimum;
+ EXPECT_EQ(0, lrc._minimum_to_decode(want_to_read, available_chunks, &minimum));
+ set<int> expected_minimum;
+ expected_minimum.insert(2);
+ expected_minimum.insert(3);
+ expected_minimum.insert(4);
+ EXPECT_EQ(expected_minimum, minimum);
+ }
+ }
+ // implicit parity required
+ {
+ ErasureCodeLrc lrc(g_conf().get_val<std::string>("erasure_code_dir"));
+ ErasureCodeProfile profile;
+ profile["mapping"] =
+ "__DDD__DD";
+ const char *description_string =
+ "[ "
+ " [ \"_cDDD_cDD\", \"\" ],"
+ " [ \"c_DDD____\", \"\" ],"
+ " [ \"_____cDDD\", \"\" ],"
+ "]";
+ profile["layers"] = description_string;
+ EXPECT_EQ(0, lrc.init(profile, &cerr));
+ EXPECT_EQ(profile["mapping"].length(),
+ lrc.get_chunk_count());
+ set<int> want_to_read;
+ want_to_read.insert(8);
+ //
+ // unable to recover, too many chunks missing
+ //
+ {
+ set<int> available_chunks;
+ available_chunks.insert(0);
+ available_chunks.insert(1);
+ // missing (2)
+ // missing (3)
+ available_chunks.insert(4);
+ available_chunks.insert(5);
+ available_chunks.insert(6);
+ // missing (7)
+ // missing (8)
+ set<int> minimum;
+ EXPECT_EQ(-EIO, lrc._minimum_to_decode(want_to_read, available_chunks, &minimum));
+ }
+ //
+ // We want to read chunk 8 and encoding was done with
+ //
+ // _cDDD_cDD
+ // c_DDD____
+ // _____cDDD
+ //
+ // First strategy fails:
+ //
+ // 012345678
+ // xxXXXxxXX initial chunks
+ // xx.XXxx.. missing (2, 7, 8)
+ // _____cDDD fail : can recover 1 but 2 are missing
+ // c_DDD____ ignored because 8 is not used (i.e. _)
+ // _cDDD_cDD fail : can recover 2 but 3 are missing
+ //
+ // Second strategy succeeds:
+ //
+ // 012345678
+ // xxXXXxxXX initial chunks
+ // xx.XXxx.. missing (2, 7, 8)
+ // _____cDDD fail : can recover 1 but 2 are missing
+ // c_DDD____ success: recovers chunk 2
+ // _cDDD_cDD success: recovers chunk 7, 8
+ //
+ {
+ set<int> available_chunks;
+ available_chunks.insert(0);
+ available_chunks.insert(1);
+ // missing (2)
+ available_chunks.insert(3);
+ available_chunks.insert(4);
+ available_chunks.insert(5);
+ available_chunks.insert(6);
+ // missing (7)
+ // missing (8)
+ set<int> minimum;
+ EXPECT_EQ(0, lrc._minimum_to_decode(want_to_read, available_chunks, &minimum));
+ EXPECT_EQ(available_chunks, minimum);
+ }
+ }
+}
+
+TEST(ErasureCodeLrc, encode_decode)
+{
+ ErasureCodeLrc lrc(g_conf().get_val<std::string>("erasure_code_dir"));
+ ErasureCodeProfile profile;
+ profile["mapping"] =
+ "__DD__DD";
+ const char *description_string =
+ "[ "
+ " [ \"_cDD_cDD\", \"\" ]," // global layer
+ " [ \"c_DD____\", \"\" ]," // first local layer
+ " [ \"____cDDD\", \"\" ]," // second local layer
+ "]";
+ profile["layers"] = description_string;
+ EXPECT_EQ(0, lrc.init(profile, &cerr));
+ EXPECT_EQ(4U, lrc.get_data_chunk_count());
+ unsigned int chunk_size = g_conf().get_val<Option::size_t>("osd_pool_erasure_code_stripe_unit");
+ unsigned int stripe_width = lrc.get_data_chunk_count() * chunk_size;
+ EXPECT_EQ(chunk_size, lrc.get_chunk_size(stripe_width));
+ set<int> want_to_encode;
+ map<int, bufferlist> encoded;
+ for (unsigned int i = 0; i < lrc.get_chunk_count(); ++i) {
+ want_to_encode.insert(i);
+ bufferptr ptr(buffer::create_page_aligned(chunk_size));
+ bufferlist tmp;
+ tmp.push_back(ptr);
+ tmp.claim_append(encoded[i]);
+ encoded[i].swap(tmp);
+ }
+ const vector<int> &mapping = lrc.get_chunk_mapping();
+ char c = 'A';
+ for (unsigned int i = 0; i < lrc.get_data_chunk_count(); i++) {
+ int j = mapping[i];
+ string s(chunk_size, c);
+ encoded[j].clear();
+ encoded[j].append(s);
+ c++;
+ }
+ EXPECT_EQ(0, lrc.encode_chunks(want_to_encode, &encoded));
+
+ {
+ map<int, bufferlist> chunks;
+ chunks[4] = encoded[4];
+ chunks[5] = encoded[5];
+ chunks[6] = encoded[6];
+ set<int> want_to_read;
+ want_to_read.insert(7);
+ set<int> available_chunks;
+ available_chunks.insert(4);
+ available_chunks.insert(5);
+ available_chunks.insert(6);
+ set<int> minimum;
+ EXPECT_EQ(0, lrc._minimum_to_decode(want_to_read, available_chunks, &minimum));
+ // only need three chunks from the second local layer
+ EXPECT_EQ(3U, minimum.size());
+ EXPECT_EQ(1U, minimum.count(4));
+ EXPECT_EQ(1U, minimum.count(5));
+ EXPECT_EQ(1U, minimum.count(6));
+ map<int, bufferlist> decoded;
+ EXPECT_EQ(0, lrc._decode(want_to_read, chunks, &decoded));
+ string s(chunk_size, 'D');
+ EXPECT_EQ(s, string(decoded[7].c_str(), chunk_size));
+ }
+ {
+ set<int> want_to_read;
+ want_to_read.insert(2);
+ map<int, bufferlist> chunks;
+ chunks[1] = encoded[1];
+ chunks[3] = encoded[3];
+ chunks[5] = encoded[5];
+ chunks[6] = encoded[6];
+ chunks[7] = encoded[7];
+ set<int> available_chunks;
+ available_chunks.insert(1);
+ available_chunks.insert(3);
+ available_chunks.insert(5);
+ available_chunks.insert(6);
+ available_chunks.insert(7);
+ set<int> minimum;
+ EXPECT_EQ(0, lrc._minimum_to_decode(want_to_read, available_chunks, &minimum));
+ EXPECT_EQ(5U, minimum.size());
+ EXPECT_EQ(available_chunks, minimum);
+
+ map<int, bufferlist> decoded;
+ EXPECT_EQ(0, lrc._decode(want_to_read, encoded, &decoded));
+ string s(chunk_size, 'A');
+ EXPECT_EQ(s, string(decoded[2].c_str(), chunk_size));
+ }
+ {
+ set<int> want_to_read;
+ want_to_read.insert(3);
+ want_to_read.insert(6);
+ want_to_read.insert(7);
+ set<int> available_chunks;
+ available_chunks.insert(0);
+ available_chunks.insert(1);
+ available_chunks.insert(2);
+ // available_chunks.insert(3);
+ available_chunks.insert(4);
+ available_chunks.insert(5);
+ // available_chunks.insert(6);
+ // available_chunks.insert(7);
+ encoded.erase(3);
+ encoded.erase(6);
+ set<int> minimum;
+ EXPECT_EQ(0, lrc._minimum_to_decode(want_to_read, available_chunks, &minimum));
+ EXPECT_EQ(4U, minimum.size());
+ // only need two chunks from the first local layer
+ EXPECT_EQ(1U, minimum.count(0));
+ EXPECT_EQ(1U, minimum.count(2));
+ // the above chunks will rebuild chunk 3 and the global layer only needs
+ // three more chunks to reach the required amount of chunks (4) to recover
+ // the last two
+ EXPECT_EQ(1U, minimum.count(1));
+ EXPECT_EQ(1U, minimum.count(2));
+ EXPECT_EQ(1U, minimum.count(5));
+
+ map<int, bufferlist> decoded;
+ EXPECT_EQ(0, lrc._decode(want_to_read, encoded, &decoded));
+ {
+ string s(chunk_size, 'B');
+ EXPECT_EQ(s, string(decoded[3].c_str(), chunk_size));
+ }
+ {
+ string s(chunk_size, 'C');
+ EXPECT_EQ(s, string(decoded[6].c_str(), chunk_size));
+ }
+ {
+ string s(chunk_size, 'D');
+ EXPECT_EQ(s, string(decoded[7].c_str(), chunk_size));
+ }
+ }
+}
+
+TEST(ErasureCodeLrc, encode_decode_2)
+{
+ ErasureCodeLrc lrc(g_conf().get_val<std::string>("erasure_code_dir"));
+ ErasureCodeProfile profile;
+ profile["mapping"] =
+ "DD__DD__";
+ const char *description_string =
+ "[ "
+ " [ \"DDc_DDc_\", \"\" ],"
+ " [ \"DDDc____\", \"\" ],"
+ " [ \"____DDDc\", \"\" ],"
+ "]";
+ profile["layers"] = description_string;
+ EXPECT_EQ(0, lrc.init(profile, &cerr));
+ EXPECT_EQ(4U, lrc.get_data_chunk_count());
+ unsigned int chunk_size = g_conf().get_val<Option::size_t>("osd_pool_erasure_code_stripe_unit");
+ unsigned int stripe_width = lrc.get_data_chunk_count() * chunk_size;
+ EXPECT_EQ(chunk_size, lrc.get_chunk_size(stripe_width));
+ set<int> want_to_encode;
+ map<int, bufferlist> encoded;
+ for (unsigned int i = 0; i < lrc.get_chunk_count(); ++i) {
+ want_to_encode.insert(i);
+ bufferptr ptr(buffer::create_page_aligned(chunk_size));
+ bufferlist tmp;
+ tmp.push_back(ptr);
+ tmp.claim_append(encoded[i]);
+ encoded[i].swap(tmp);
+ }
+ const vector<int> &mapping = lrc.get_chunk_mapping();
+ char c = 'A';
+ for (unsigned int i = 0; i < lrc.get_data_chunk_count(); i++) {
+ int j = mapping[i];
+ string s(chunk_size, c);
+ encoded[j].clear();
+ encoded[j].append(s);
+ c++;
+ }
+ EXPECT_EQ(0, lrc.encode_chunks(want_to_encode, &encoded));
+
+ {
+ set<int> want_to_read;
+ want_to_read.insert(0);
+ map<int, bufferlist> chunks;
+ chunks[1] = encoded[1];
+ chunks[3] = encoded[3];
+ chunks[4] = encoded[4];
+ chunks[5] = encoded[5];
+ chunks[6] = encoded[6];
+ chunks[7] = encoded[7];
+ set<int> available_chunks;
+ available_chunks.insert(1);
+ available_chunks.insert(3);
+ available_chunks.insert(4);
+ available_chunks.insert(5);
+ available_chunks.insert(6);
+ available_chunks.insert(7);
+ set<int> minimum;
+ EXPECT_EQ(0, lrc._minimum_to_decode(want_to_read, available_chunks, &minimum));
+ EXPECT_EQ(4U, minimum.size());
+ EXPECT_EQ(1U, minimum.count(1));
+ EXPECT_EQ(1U, minimum.count(4));
+ EXPECT_EQ(1U, minimum.count(5));
+ EXPECT_EQ(1U, minimum.count(6));
+
+ map<int, bufferlist> decoded;
+ EXPECT_EQ(0, lrc._decode(want_to_read, chunks, &decoded));
+ string s(chunk_size, 'A');
+ EXPECT_EQ(s, string(decoded[0].c_str(), chunk_size));
+ }
+ {
+ set<int> want_to_read;
+ for (unsigned int i = 0; i < lrc.get_chunk_count(); i++)
+ want_to_read.insert(i);
+ map<int, bufferlist> chunks;
+ chunks[1] = encoded[1];
+ chunks[3] = encoded[3];
+ chunks[5] = encoded[5];
+ chunks[6] = encoded[6];
+ chunks[7] = encoded[7];
+ set<int> available_chunks;
+ available_chunks.insert(1);
+ available_chunks.insert(3);
+ available_chunks.insert(5);
+ available_chunks.insert(6);
+ available_chunks.insert(7);
+ set<int> minimum;
+ EXPECT_EQ(0, lrc._minimum_to_decode(want_to_read, available_chunks, &minimum));
+ EXPECT_EQ(5U, minimum.size());
+ EXPECT_EQ(1U, minimum.count(1));
+ EXPECT_EQ(1U, minimum.count(3));
+ EXPECT_EQ(1U, minimum.count(5));
+ EXPECT_EQ(1U, minimum.count(6));
+ EXPECT_EQ(1U, minimum.count(7));
+
+ map<int, bufferlist> decoded;
+ EXPECT_EQ(0, lrc._decode(want_to_read, chunks, &decoded));
+ {
+ string s(chunk_size, 'A');
+ EXPECT_EQ(s, string(decoded[0].c_str(), chunk_size));
+ }
+ {
+ string s(chunk_size, 'B');
+ EXPECT_EQ(s, string(decoded[1].c_str(), chunk_size));
+ }
+ {
+ string s(chunk_size, 'C');
+ EXPECT_EQ(s, string(decoded[4].c_str(), chunk_size));
+ }
+ {
+ string s(chunk_size, 'D');
+ EXPECT_EQ(s, string(decoded[5].c_str(), chunk_size));
+ }
+ }
+ {
+ set<int> want_to_read;
+ for (unsigned int i = 0; i < lrc.get_chunk_count(); i++)
+ want_to_read.insert(i);
+ map<int, bufferlist> chunks;
+ chunks[1] = encoded[1];
+ chunks[3] = encoded[3];
+ chunks[5] = encoded[5];
+ chunks[6] = encoded[6];
+ chunks[7] = encoded[7];
+ set<int> available_chunks;
+ available_chunks.insert(1);
+ available_chunks.insert(3);
+ available_chunks.insert(5);
+ available_chunks.insert(6);
+ available_chunks.insert(7);
+ set<int> minimum;
+ EXPECT_EQ(0, lrc._minimum_to_decode(want_to_read, available_chunks, &minimum));
+ EXPECT_EQ(5U, minimum.size());
+ EXPECT_EQ(1U, minimum.count(1));
+ EXPECT_EQ(1U, minimum.count(3));
+ EXPECT_EQ(1U, minimum.count(5));
+ EXPECT_EQ(1U, minimum.count(6));
+ EXPECT_EQ(1U, minimum.count(7));
+
+ map<int, bufferlist> decoded;
+ EXPECT_EQ(0, lrc._decode(want_to_read, chunks, &decoded));
+ {
+ string s(chunk_size, 'A');
+ EXPECT_EQ(s, string(decoded[0].c_str(), chunk_size));
+ }
+ {
+ string s(chunk_size, 'B');
+ EXPECT_EQ(s, string(decoded[1].c_str(), chunk_size));
+ }
+ {
+ string s(chunk_size, 'C');
+ EXPECT_EQ(s, string(decoded[4].c_str(), chunk_size));
+ }
+ {
+ string s(chunk_size, 'D');
+ EXPECT_EQ(s, string(decoded[5].c_str(), chunk_size));
+ }
+ }
+ {
+ set<int> want_to_read;
+ want_to_read.insert(6);
+ map<int, bufferlist> chunks;
+ chunks[0] = encoded[0];
+ chunks[1] = encoded[1];
+ chunks[3] = encoded[3];
+ chunks[5] = encoded[5];
+ chunks[7] = encoded[7];
+ set<int> available_chunks;
+ available_chunks.insert(0);
+ available_chunks.insert(1);
+ available_chunks.insert(3);
+ available_chunks.insert(5);
+ available_chunks.insert(7);
+ set<int> minimum;
+ EXPECT_EQ(0, lrc._minimum_to_decode(want_to_read, available_chunks, &minimum));
+ EXPECT_EQ(available_chunks, minimum);
+
+ map<int, bufferlist> decoded;
+ EXPECT_EQ(0, lrc._decode(want_to_read, chunks, &decoded));
+ }
+}
+
+/*
+ * Local Variables:
+ * compile-command: "cd ../.. ;
+ * make -j4 unittest_erasure_code_lrc && valgrind --tool=memcheck \
+ * ./unittest_erasure_code_lrc \
+ * --gtest_filter=*.* --log-to-stderr=true --debug-osd=20"
+ * End:
+ */
diff --git a/src/test/erasure-code/TestErasureCodePlugin.cc b/src/test/erasure-code/TestErasureCodePlugin.cc
new file mode 100644
index 000000000..e396d9467
--- /dev/null
+++ b/src/test/erasure-code/TestErasureCodePlugin.cc
@@ -0,0 +1,131 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph distributed storage system
+ *
+ * Copyright (C) 2013,2014 Cloudwatt <libre.licensing@cloudwatt.com>
+ * Copyright (C) 2014 Red Hat <contact@redhat.com>
+ *
+ * Author: Loic Dachary <loic@dachary.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#include <errno.h>
+#include <signal.h>
+#include <stdlib.h>
+#include "common/Thread.h"
+#include "erasure-code/ErasureCodePlugin.h"
+#include "global/global_context.h"
+#include "common/config_proxy.h"
+#include "gtest/gtest.h"
+
+using namespace std;
+
+class ErasureCodePluginRegistryTest : public ::testing::Test {};
+
+TEST_F(ErasureCodePluginRegistryTest, factory_mutex) {
+ ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
+
+ {
+ unique_lock l{instance.lock, std::try_to_lock};
+ EXPECT_TRUE(l.owns_lock());
+ }
+ //
+ // Test that the loading of a plugin is protected by a mutex.
+
+ std::thread sleep_for_10_secs([] {
+ ErasureCodeProfile profile;
+ ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
+ ErasureCodeInterfaceRef erasure_code;
+ instance.factory("hangs",
+ g_conf().get_val<std::string>("erasure_code_dir"),
+ profile, &erasure_code, &cerr);
+ });
+ auto wait_until = [&instance](bool loading, unsigned max_secs) {
+ auto delay = 0ms;
+ const auto DELAY_MAX = std::chrono::seconds(max_secs);
+ for (; delay < DELAY_MAX; delay = (delay + 1ms) * 2) {
+ cout << "Trying (1) with delay " << delay << "us\n";
+ if (delay.count() > 0) {
+ std::this_thread::sleep_for(delay);
+ }
+ if (instance.loading == loading) {
+ return true;
+ }
+ }
+ return false;
+ };
+ // should be loading in 5 seconds
+ ASSERT_TRUE(wait_until(true, 5));
+ {
+ unique_lock l{instance.lock, std::try_to_lock};
+ EXPECT_TRUE(!l.owns_lock());
+ }
+ // should finish loading in 15 seconds
+ ASSERT_TRUE(wait_until(false, 15));
+ {
+ unique_lock l{instance.lock, std::try_to_lock};
+ EXPECT_TRUE(l.owns_lock());
+ }
+ sleep_for_10_secs.join();
+}
+
+TEST_F(ErasureCodePluginRegistryTest, all)
+{
+ ErasureCodeProfile profile;
+ string directory = g_conf().get_val<std::string>("erasure_code_dir");
+ ErasureCodeInterfaceRef erasure_code;
+ ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
+ EXPECT_FALSE(erasure_code);
+ EXPECT_EQ(-EIO, instance.factory("invalid",
+ g_conf().get_val<std::string>("erasure_code_dir"),
+ profile, &erasure_code, &cerr));
+ EXPECT_FALSE(erasure_code);
+ EXPECT_EQ(-EXDEV, instance.factory("missing_version",
+ g_conf().get_val<std::string>("erasure_code_dir"),
+ profile,
+ &erasure_code, &cerr));
+ EXPECT_FALSE(erasure_code);
+ EXPECT_EQ(-ENOENT, instance.factory("missing_entry_point",
+ g_conf().get_val<std::string>("erasure_code_dir"),
+ profile,
+ &erasure_code, &cerr));
+ EXPECT_FALSE(erasure_code);
+ EXPECT_EQ(-ESRCH, instance.factory("fail_to_initialize",
+ g_conf().get_val<std::string>("erasure_code_dir"),
+ profile,
+ &erasure_code, &cerr));
+ EXPECT_FALSE(erasure_code);
+ EXPECT_EQ(-EBADF, instance.factory("fail_to_register",
+ g_conf().get_val<std::string>("erasure_code_dir"),
+ profile,
+ &erasure_code, &cerr));
+ EXPECT_FALSE(erasure_code);
+ EXPECT_EQ(0, instance.factory("example",
+ g_conf().get_val<std::string>("erasure_code_dir"),
+ profile, &erasure_code, &cerr));
+ EXPECT_TRUE(erasure_code.get());
+ ErasureCodePlugin *plugin = 0;
+ {
+ std::lock_guard l{instance.lock};
+ EXPECT_EQ(-EEXIST, instance.load("example", directory, &plugin, &cerr));
+ EXPECT_EQ(-ENOENT, instance.remove("does not exist"));
+ EXPECT_EQ(0, instance.remove("example"));
+ EXPECT_EQ(0, instance.load("example", directory, &plugin, &cerr));
+ }
+}
+
+/*
+ * Local Variables:
+ * compile-command: "cd ../../../build ; make -j4 &&
+ * make unittest_erasure_code_plugin &&
+ * valgrind --tool=memcheck \
+ * ./bin/unittest_erasure_code_plugin \
+ * --gtest_filter=*.* --log-to-stderr=true --debug-osd=20"
+ * End:
+ */
diff --git a/src/test/erasure-code/TestErasureCodePluginClay.cc b/src/test/erasure-code/TestErasureCodePluginClay.cc
new file mode 100644
index 000000000..cbf6566dd
--- /dev/null
+++ b/src/test/erasure-code/TestErasureCodePluginClay.cc
@@ -0,0 +1,114 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph distributed storage system
+ *
+ * Copyright (C) 2018 Indian Institute of Science <office.ece@iisc.ac.in>
+ *
+ * Author: Myna Vajha <mynaramana@gmail.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#include <errno.h>
+#include <stdlib.h>
+#include "erasure-code/ErasureCodePlugin.h"
+#include "log/Log.h"
+#include "global/global_context.h"
+#include "common/config_proxy.h"
+#include "gtest/gtest.h"
+
+using namespace std;
+
+TEST(ErasureCodePlugin, factory)
+{
+ ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
+ ErasureCodeProfile profile;
+ {
+ ErasureCodeInterfaceRef erasure_code;
+ EXPECT_FALSE(erasure_code);
+ EXPECT_EQ(0, instance.factory("clay",
+ g_conf().get_val<std::string>("erasure_code_dir"),
+ profile,
+ &erasure_code, &cerr));
+ EXPECT_TRUE(erasure_code);
+ }
+ //check clay plugin with scalar_mds=jerasure
+ {
+ const char *techniques[] = {
+ "reed_sol_van",
+ "reed_sol_r6_op",
+ "cauchy_orig",
+ "cauchy_good",
+ "liber8tion",
+ 0
+ };
+ for(const char **technique = techniques; *technique; technique++) {
+ ErasureCodeInterfaceRef erasure_code;
+ ErasureCodeProfile profile;
+ profile["scalar_mds"] = "jerasure";
+ profile["technique"] = *technique;
+ EXPECT_FALSE(erasure_code);
+ EXPECT_EQ(0, instance.factory("clay",
+ g_conf().get_val<std::string>("erasure_code_dir"),
+ profile,
+ &erasure_code, &cerr));
+ EXPECT_TRUE(erasure_code.get());
+ }
+ }
+#ifdef WITH_EC_ISA_PLUGIN
+ //check clay plugin with scalar_mds=isa
+ {
+ const char *techniques[] = {
+ "reed_sol_van",
+ "cauchy",
+ 0
+ };
+ for(const char **technique = techniques; *technique; technique++) {
+ ErasureCodeInterfaceRef erasure_code;
+ ErasureCodeProfile profile;
+ profile["scalar_mds"] = "isa";
+ profile["technique"] = *technique;
+ EXPECT_FALSE(erasure_code);
+ EXPECT_EQ(0, instance.factory("clay",
+ g_conf().get_val<std::string>("erasure_code_dir"),
+ profile,
+ &erasure_code, &cerr));
+ EXPECT_TRUE(erasure_code.get());
+ }
+ }
+#endif
+ //check clay plugin with scalar_mds=shec
+ {
+ const char *techniques[] = {
+ "single",
+ "multiple",
+ 0
+ };
+ for(const char **technique = techniques; *technique; technique++) {
+ ErasureCodeInterfaceRef erasure_code;
+ ErasureCodeProfile profile;
+ profile["scalar_mds"] = "shec";
+ profile["technique"] = *technique;
+ EXPECT_FALSE(erasure_code);
+ EXPECT_EQ(0, instance.factory("clay",
+ g_conf().get_val<std::string>("erasure_code_dir"),
+ profile,
+ &erasure_code, &cerr));
+ EXPECT_TRUE(erasure_code.get());
+ }
+ }
+}
+
+/*
+ * Local Variables:
+ * compile-command: "cd ../.. ; make -j4 &&
+ * make unittest_erasure_code_plugin_clay &&
+ * valgrind --tool=memcheck ./unittest_erasure_code_plugin_clay \
+ * --gtest_filter=*.* --log-to-stderr=true --debug-osd=20"
+ * End:
+ */
diff --git a/src/test/erasure-code/TestErasureCodePluginIsa.cc b/src/test/erasure-code/TestErasureCodePluginIsa.cc
new file mode 100644
index 000000000..86bac636d
--- /dev/null
+++ b/src/test/erasure-code/TestErasureCodePluginIsa.cc
@@ -0,0 +1,62 @@
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2014 CERN (Switzerland)
+ *
+ * Author: Andreas-Joachim Peters <Andreas.Joachim.Peters@cern.ch>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#include <errno.h>
+#include <stdlib.h>
+#include "arch/probe.h"
+#include "arch/intel.h"
+#include "erasure-code/ErasureCodePlugin.h"
+#include "global/global_context.h"
+#include "common/config_proxy.h"
+#include "gtest/gtest.h"
+
+using namespace std;
+
+TEST(ErasureCodePlugin, factory)
+{
+ ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
+ ErasureCodeProfile profile;
+ {
+ ErasureCodeInterfaceRef erasure_code;
+ EXPECT_FALSE(erasure_code);
+ EXPECT_EQ(-EIO, instance.factory("no-isa",
+ g_conf().get_val<std::string>("erasure_code_dir"),
+ profile,
+ &erasure_code, &cerr));
+ EXPECT_FALSE(erasure_code);
+ }
+ const char *techniques[] = {
+ "reed_sol_van",
+ 0
+ };
+ for(const char **technique = techniques; *technique; technique++) {
+ ErasureCodeInterfaceRef erasure_code;
+ profile["technique"] = *technique;
+ EXPECT_FALSE(erasure_code);
+ EXPECT_EQ(0, instance.factory("isa",
+ g_conf().get_val<std::string>("erasure_code_dir"),
+ profile,
+ &erasure_code, &cerr));
+ EXPECT_TRUE(erasure_code.get());
+ }
+}
+
+/*
+ * Local Variables:
+ * compile-command: "cd ../.. ; make -j4 &&
+ * make unittest_erasure_code_plugin_isa &&
+ * valgrind --tool=memcheck ./unittest_erasure_code_plugin_isa \
+ * --gtest_filter=*.* --log-to-stderr=true --debug-osd=20"
+ * End:
+ */
diff --git a/src/test/erasure-code/TestErasureCodePluginJerasure.cc b/src/test/erasure-code/TestErasureCodePluginJerasure.cc
new file mode 100644
index 000000000..ffe4183bf
--- /dev/null
+++ b/src/test/erasure-code/TestErasureCodePluginJerasure.cc
@@ -0,0 +1,71 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph distributed storage system
+ *
+ * Copyright (C) 2013,2014 Cloudwatt <libre.licensing@cloudwatt.com>
+ * Copyright (C) 2014 Red Hat <contact@redhat.com>
+ *
+ * Author: Loic Dachary <loic@dachary.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#include <errno.h>
+#include <stdlib.h>
+#include "erasure-code/ErasureCodePlugin.h"
+#include "log/Log.h"
+#include "global/global_context.h"
+#include "common/config_proxy.h"
+#include "gtest/gtest.h"
+
+using namespace std;
+
+TEST(ErasureCodePlugin, factory)
+{
+ ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
+ ErasureCodeProfile profile;
+ {
+ ErasureCodeInterfaceRef erasure_code;
+ EXPECT_FALSE(erasure_code);
+ EXPECT_EQ(-ENOENT, instance.factory("jerasure",
+ g_conf().get_val<std::string>("erasure_code_dir"),
+ profile,
+ &erasure_code, &cerr));
+ EXPECT_FALSE(erasure_code);
+ }
+ const char *techniques[] = {
+ "reed_sol_van",
+ "reed_sol_r6_op",
+ "cauchy_orig",
+ "cauchy_good",
+ "liberation",
+ "blaum_roth",
+ "liber8tion",
+ 0
+ };
+ for(const char **technique = techniques; *technique; technique++) {
+ ErasureCodeInterfaceRef erasure_code;
+ ErasureCodeProfile profile;
+ profile["technique"] = *technique;
+ EXPECT_FALSE(erasure_code);
+ EXPECT_EQ(0, instance.factory("jerasure",
+ g_conf().get_val<std::string>("erasure_code_dir"),
+ profile,
+ &erasure_code, &cerr));
+ EXPECT_TRUE(erasure_code.get());
+ }
+}
+
+/*
+ * Local Variables:
+ * compile-command: "cd ../.. ; make -j4 &&
+ * make unittest_erasure_code_plugin_jerasure &&
+ * valgrind --tool=memcheck ./unittest_erasure_code_plugin_jerasure \
+ * --gtest_filter=*.* --log-to-stderr=true --debug-osd=20"
+ * End:
+ */
diff --git a/src/test/erasure-code/TestErasureCodePluginLrc.cc b/src/test/erasure-code/TestErasureCodePluginLrc.cc
new file mode 100644
index 000000000..9be2c336d
--- /dev/null
+++ b/src/test/erasure-code/TestErasureCodePluginLrc.cc
@@ -0,0 +1,50 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph distributed storage system
+ *
+ * Copyright (C) 2014 Cloudwatt <libre.licensing@cloudwatt.com>
+ * Copyright (C) 2014 Red Hat <contact@redhat.com>
+ *
+ * Author: Loic Dachary <loic@dachary.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#include <errno.h>
+#include <stdlib.h>
+#include "arch/probe.h"
+#include "arch/intel.h"
+#include "erasure-code/ErasureCodePlugin.h"
+#include "global/global_context.h"
+#include "common/config_proxy.h"
+#include "gtest/gtest.h"
+
+using namespace std;
+
+TEST(ErasureCodePlugin, factory)
+{
+ ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
+ ErasureCodeProfile profile;
+ profile["mapping"] = "DD_";
+ profile["layers"] = "[ [ \"DDc\", \"\" ] ]";
+ ErasureCodeInterfaceRef erasure_code;
+ EXPECT_FALSE(erasure_code);
+ EXPECT_EQ(0, instance.factory("lrc",
+ g_conf().get_val<std::string>("erasure_code_dir"),
+ profile, &erasure_code, &cerr));
+ EXPECT_TRUE(erasure_code.get());
+}
+
+/*
+ * Local Variables:
+ * compile-command: "cd ../.. ; make -j4 &&
+ * make unittest_erasure_code_plugin_lrc &&
+ * valgrind --tool=memcheck ./unittest_erasure_code_plugin_lrc \
+ * --gtest_filter=*.* --log-to-stderr=true --debug-osd=20"
+ * End:
+ */
diff --git a/src/test/erasure-code/TestErasureCodePluginShec.cc b/src/test/erasure-code/TestErasureCodePluginShec.cc
new file mode 100644
index 000000000..fae1b559e
--- /dev/null
+++ b/src/test/erasure-code/TestErasureCodePluginShec.cc
@@ -0,0 +1,65 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph distributed storage system
+ *
+ * Copyright (C) 2015 FUJITSU LIMITED
+ *
+ * Author: Shotaro Kawaguchi <kawaguchi.s@jp.fujitsu.com>
+ * Author: Takanori Nakao <nakao.takanori@jp.fujitsu.com>
+ * Author: Takeshi Miyamae <miyamae.takeshi@jp.fujitsu.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#include <errno.h>
+#include <stdlib.h>
+#include "erasure-code/ErasureCodePlugin.h"
+#include "global/global_context.h"
+#include "gtest/gtest.h"
+#include "common/config_proxy.h"
+
+using namespace std;
+
+TEST(ErasureCodePlugin, factory)
+{
+ ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
+ map<std::string,std::string> profile;
+ {
+ ErasureCodeInterfaceRef erasure_code;
+ EXPECT_FALSE(erasure_code);
+ EXPECT_EQ(0, instance.factory("shec",
+ g_conf().get_val<std::string>("erasure_code_dir"),
+ profile,
+ &erasure_code, &cerr));
+ EXPECT_TRUE(erasure_code.get());
+ }
+ const char *techniques[] = {
+ "single",
+ "multiple",
+ 0
+ };
+ for(const char **technique = techniques; *technique; technique++) {
+ ErasureCodeInterfaceRef erasure_code;
+ profile["technique"] = *technique;
+ EXPECT_FALSE(erasure_code);
+ EXPECT_EQ(0, instance.factory("shec",
+ g_conf().get_val<std::string>("erasure_code_dir"),
+ profile,
+ &erasure_code, &cerr));
+ EXPECT_TRUE(erasure_code.get());
+ }
+}
+
+/*
+ * Local Variables:
+ * compile-command: "cd ../.. ; make -j4 &&
+ * make unittest_erasure_code_plugin_shec &&
+ * valgrind --tool=memcheck ./unittest_erasure_code_plugin_shec \
+ * --gtest_filter=*.* --log-to-stderr=true --debug-osd=20"
+ * End:
+ */
diff --git a/src/test/erasure-code/TestErasureCodeShec.cc b/src/test/erasure-code/TestErasureCodeShec.cc
new file mode 100644
index 000000000..6b901dc6f
--- /dev/null
+++ b/src/test/erasure-code/TestErasureCodeShec.cc
@@ -0,0 +1,2823 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2014,2015 FUJITSU LIMITED
+ *
+ * Author: Shotaro Kawaguchi <kawaguchi.s@jp.fujitsu.com>
+ * Author: Takanori Nakao <nakao.takanori@jp.fujitsu.com>
+ * Author: Takeshi Miyamae <miyamae.takeshi@jp.fujitsu.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+//SUMMARY: TestErasureCodeShec
+
+#include <errno.h>
+#include <pthread.h>
+#include <stdlib.h>
+
+#include "crush/CrushWrapper.h"
+#include "osd/osd_types.h"
+#include "include/stringify.h"
+#include "erasure-code/shec/ErasureCodeShec.h"
+#include "erasure-code/ErasureCodePlugin.h"
+#include "global/global_context.h"
+#include "gtest/gtest.h"
+
+using namespace std;
+
+void* thread1(void* pParam);
+void* thread2(void* pParam);
+void* thread3(void* pParam);
+void* thread4(void* pParam);
+void* thread5(void* pParam);
+
+static int g_flag = 0;
+
+TEST(ErasureCodeShec, init_1)
+{
+ //all parameters are normal values
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+
+ int r = shec->init(*profile, &cerr);
+
+ //check profile
+ EXPECT_EQ(4, shec->k);
+ EXPECT_EQ(3, shec->m);
+ EXPECT_EQ(2, shec->c);
+ EXPECT_EQ(8, shec->w);
+ EXPECT_EQ(ErasureCodeShec::MULTIPLE, shec->technique);
+ EXPECT_STREQ("default", shec->rule_root.c_str());
+ EXPECT_STREQ("osd", shec->rule_failure_domain.c_str());
+ EXPECT_TRUE(shec->matrix != NULL);
+ EXPECT_EQ(0, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, init_2)
+{
+ //all parameters are normal values
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-root"] = "test";
+ (*profile)["crush-failure-domain"] = "host";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ (*profile)["w"] = "8";
+
+ int r = shec->init(*profile, &cerr);
+
+ //check profile
+ EXPECT_EQ(4, shec->k);
+ EXPECT_EQ(3, shec->m);
+ EXPECT_EQ(2, shec->c);
+ EXPECT_EQ(8, shec->w);
+ EXPECT_EQ(ErasureCodeShec::MULTIPLE, shec->technique);
+ EXPECT_STREQ("test", shec->rule_root.c_str());
+ EXPECT_STREQ("host", shec->rule_failure_domain.c_str());
+ EXPECT_TRUE(shec->matrix != NULL);
+ EXPECT_EQ(0, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, init_3)
+{
+ //all parameters are normal values
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ (*profile)["w"] = "16";
+
+ int r = shec->init(*profile, &cerr);
+
+ //check profile
+ EXPECT_EQ(4, shec->k);
+ EXPECT_EQ(3, shec->m);
+ EXPECT_EQ(2, shec->c);
+ EXPECT_EQ(16, shec->w);
+ EXPECT_EQ(ErasureCodeShec::MULTIPLE, shec->technique);
+ EXPECT_STREQ("default", shec->rule_root.c_str());
+ EXPECT_STREQ("osd", shec->rule_failure_domain.c_str());
+ EXPECT_TRUE(shec->matrix != NULL);
+ EXPECT_EQ(0, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, init_4)
+{
+ //all parameters are normal values
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ (*profile)["w"] = "32";
+
+ int r = shec->init(*profile, &cerr);
+
+ //check profile
+ EXPECT_EQ(4, shec->k);
+ EXPECT_EQ(3, shec->m);
+ EXPECT_EQ(2, shec->c);
+ EXPECT_EQ(32, shec->w);
+ EXPECT_EQ(ErasureCodeShec::MULTIPLE, shec->technique);
+ EXPECT_STREQ("default", shec->rule_root.c_str());
+ EXPECT_STREQ("osd", shec->rule_failure_domain.c_str());
+ EXPECT_TRUE(shec->matrix != NULL);
+ EXPECT_EQ(0, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, init_5)
+{
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ //plugin is not specified
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+
+ int r = shec->init(*profile, &cerr);
+
+ EXPECT_TRUE(shec->matrix != NULL);
+ EXPECT_EQ(0, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, init_6)
+{
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "jerasure"; //unexpected value
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+
+ int r = shec->init(*profile, &cerr);
+
+ EXPECT_TRUE(shec->matrix != NULL);
+ EXPECT_EQ(0, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, init_7)
+{
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "abc"; //unexpected value
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+
+ int r = shec->init(*profile, &cerr);
+
+ EXPECT_TRUE(shec->matrix != NULL);
+ EXPECT_EQ(0, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, init_8)
+{
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+
+ int r = shec->init(*profile, &cerr);
+
+ EXPECT_TRUE(shec->matrix != NULL);
+ EXPECT_EQ(0, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, init_9)
+{
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-root"] = "abc"; //unexpected value
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+
+ int r = shec->init(*profile, &cerr);
+
+ EXPECT_TRUE(shec->matrix != NULL);
+ EXPECT_EQ(0, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, init_10)
+{
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "abc"; //unexpected value
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+
+ int r = shec->init(*profile, &cerr);
+
+ EXPECT_TRUE(shec->matrix != NULL);
+ EXPECT_EQ(0, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, init_11)
+{
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "abc"; //unexpected value
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+
+ int r = shec->init(*profile, &cerr);
+
+ EXPECT_TRUE(shec->matrix != NULL);
+ EXPECT_EQ(0, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, init_12)
+{
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "-1"; //unexpected value
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+
+ int r = shec->init(*profile, &cerr);
+
+ EXPECT_EQ(-EINVAL, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, init_13)
+{
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "abc";
+ (*profile)["k"] = "0.1"; //unexpected value
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+
+ int r = shec->init(*profile, &cerr);
+
+ EXPECT_EQ(-EINVAL, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, init_14)
+{
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "a"; //unexpected value
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+
+ int r = shec->init(*profile, &cerr);
+
+ EXPECT_EQ(-EINVAL, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, init_15)
+{
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ //k is not specified
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+
+ int r = shec->init(*profile, &cerr);
+
+ EXPECT_EQ(-EINVAL, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, init_16)
+{
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "-1"; //unexpected value
+ (*profile)["c"] = "2";
+
+ int r = shec->init(*profile, &cerr);
+
+ EXPECT_EQ(-EINVAL, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, init_17)
+{
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "0.1"; //unexpected value
+ (*profile)["c"] = "2";
+
+ int r = shec->init(*profile, &cerr);
+
+ EXPECT_EQ(-EINVAL, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, init_18)
+{
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "a"; //unexpected value
+ (*profile)["c"] = "2";
+
+ int r = shec->init(*profile, &cerr);
+
+ EXPECT_EQ(-EINVAL, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, init_19)
+{
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ //m is not specified
+ (*profile)["c"] = "2";
+
+ int r = shec->init(*profile, &cerr);
+
+ EXPECT_EQ(-EINVAL, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, init_20)
+{
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "-1"; //unexpected value
+
+ int r = shec->init(*profile, &cerr);
+
+ EXPECT_EQ(-EINVAL, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, init_21)
+{
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "0.1"; //unexpected value
+
+ int r = shec->init(*profile, &cerr);
+
+ EXPECT_EQ(-EINVAL, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, init_22)
+{
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "a"; //unexpected value
+
+ int r = shec->init(*profile, &cerr);
+
+ EXPECT_EQ(-EINVAL, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, init_23)
+{
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ //c is not specified
+
+ int r = shec->init(*profile, &cerr);
+
+ EXPECT_EQ(-EINVAL, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, init_24)
+{
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ (*profile)["w"] = "1"; //unexpected value
+
+ int r = shec->init(*profile, &cerr);
+
+ EXPECT_TRUE(shec->matrix != NULL);
+ EXPECT_EQ(0, r);
+ EXPECT_EQ(4, shec->k);
+ EXPECT_EQ(3, shec->m);
+ EXPECT_EQ(2, shec->c);
+ EXPECT_EQ(8, shec->w);
+ //w is default value
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, init_25)
+{
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ (*profile)["w"] = "-1"; //unexpected value
+
+ int r = shec->init(*profile, &cerr);
+
+ EXPECT_TRUE(shec->matrix != NULL);
+ EXPECT_EQ(0, r);
+ EXPECT_EQ(4, shec->k);
+ EXPECT_EQ(3, shec->m);
+ EXPECT_EQ(2, shec->c);
+ EXPECT_EQ(8, shec->w);
+ //w is default value
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, init_26)
+{
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ (*profile)["w"] = "0.1"; //unexpected value
+
+ int r = shec->init(*profile, &cerr);
+
+ EXPECT_TRUE(shec->matrix != NULL);
+ EXPECT_EQ(0, r);
+ EXPECT_EQ(4, shec->k);
+ EXPECT_EQ(3, shec->m);
+ EXPECT_EQ(2, shec->c);
+ EXPECT_EQ(8, shec->w);
+ //w is default value
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, init_27)
+{
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ (*profile)["w"] = "a"; //unexpected value
+
+ int r = shec->init(*profile, &cerr);
+
+ EXPECT_TRUE(shec->matrix != NULL);
+ EXPECT_EQ(0, r);
+ EXPECT_EQ(4, shec->k);
+ EXPECT_EQ(3, shec->m);
+ EXPECT_EQ(2, shec->c);
+ EXPECT_EQ(8, shec->w);
+ //w is default value
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, init_28)
+{
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "10"; //c > m
+
+ int r = shec->init(*profile, &cerr);
+
+ EXPECT_EQ(-EINVAL, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, init_29)
+{
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ //k is not specified
+ //m is not specified
+ //c is not specified
+
+ int r = shec->init(*profile, &cerr);
+
+ EXPECT_TRUE(shec->matrix != NULL);
+ EXPECT_EQ(0, r);
+ //k,m,c are default values
+ EXPECT_EQ(4, shec->k);
+ EXPECT_EQ(3, shec->m);
+ EXPECT_EQ(2, shec->c);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, init_30)
+{
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "12";
+ (*profile)["m"] = "8";
+ (*profile)["c"] = "8";
+
+ int r = shec->init(*profile, &cerr);
+
+ EXPECT_TRUE(shec->matrix != NULL);
+ EXPECT_EQ(0, r);
+ EXPECT_EQ(12, shec->k);
+ EXPECT_EQ(8, shec->m);
+ EXPECT_EQ(8, shec->c);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, init_31)
+{
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "13";
+ (*profile)["m"] = "7";
+ (*profile)["c"] = "7";
+
+ int r = shec->init(*profile, &cerr);
+
+ EXPECT_EQ(-EINVAL, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, init_32)
+{
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "7";
+ (*profile)["m"] = "13";
+ (*profile)["c"] = "13";
+
+ int r = shec->init(*profile, &cerr);
+
+ EXPECT_EQ(-EINVAL, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, init_33)
+{
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "12";
+ (*profile)["m"] = "9";
+ (*profile)["c"] = "8";
+
+ int r = shec->init(*profile, &cerr);
+
+ EXPECT_EQ(-EINVAL, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, init_34)
+{
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "8";
+ (*profile)["m"] = "12";
+ (*profile)["c"] = "12";
+
+ int r = shec->init(*profile, &cerr);
+
+ EXPECT_EQ(-EINVAL, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, init2_4)
+{
+ //all parameters are normal values
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ shec->init(*profile, &cerr);
+ int r = shec->init(*profile, &cerr); //init executed twice
+
+ //check profile
+ EXPECT_EQ(4, shec->k);
+ EXPECT_EQ(3, shec->m);
+ EXPECT_EQ(2, shec->c);
+ EXPECT_EQ(8, shec->w);
+ EXPECT_EQ(ErasureCodeShec::MULTIPLE, shec->technique);
+ EXPECT_STREQ("default", shec->rule_root.c_str());
+ EXPECT_STREQ("osd", shec->rule_failure_domain.c_str());
+ EXPECT_TRUE(shec->matrix != NULL);
+ EXPECT_EQ(0, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, init2_5)
+{
+ //all parameters are normal values
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ ErasureCodeProfile *profile2 = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "host";
+ (*profile)["k"] = "10";
+ (*profile)["m"] = "6";
+ (*profile)["c"] = "5";
+ (*profile)["w"] = "16";
+
+ int r = shec->init(*profile, &cerr);
+
+ //reexecute init
+ (*profile2)["plugin"] = "shec";
+ (*profile2)["technique"] = "";
+ (*profile2)["crush-failure-domain"] = "osd";
+ (*profile2)["k"] = "4";
+ (*profile2)["m"] = "3";
+ (*profile2)["c"] = "2";
+ shec->init(*profile2, &cerr);
+
+ EXPECT_EQ(4, shec->k);
+ EXPECT_EQ(3, shec->m);
+ EXPECT_EQ(2, shec->c);
+ EXPECT_EQ(8, shec->w);
+ EXPECT_EQ(ErasureCodeShec::MULTIPLE, shec->technique);
+ EXPECT_STREQ("default", shec->rule_root.c_str());
+ EXPECT_STREQ("osd", shec->rule_failure_domain.c_str());
+ EXPECT_TRUE(shec->matrix != NULL);
+ EXPECT_EQ(0, r);
+
+ delete shec;
+ delete profile;
+ delete profile2;
+}
+
+TEST(ErasureCodeShec, minimum_to_decode_8)
+{
+ //init
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ shec->init(*profile, &cerr);
+
+ //minimum_to_decode
+ set<int> want_to_decode;
+ set<int> available_chunks;
+ set<int> minimum_chunks;
+
+ for (int i = 0; i < 8; ++i) {
+ want_to_decode.insert(i);
+ }
+ for (int i = 0; i < 5; ++i) {
+ available_chunks.insert(i);
+ }
+
+ int r = shec->_minimum_to_decode(want_to_decode, available_chunks,
+ &minimum_chunks);
+ EXPECT_EQ(-EINVAL, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, minimum_to_decode_9)
+{
+ //init
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ shec->init(*profile, &cerr);
+
+ //minimum_to_decode
+ set<int> want_to_decode;
+ set<int> available_chunks;
+ set<int> minimum_chunks;
+
+ for (int i = 0; i < 4; ++i) {
+ want_to_decode.insert(i);
+ }
+ for (int i = 0; i < 8; ++i) {
+ available_chunks.insert(i);
+ }
+
+ int r = shec->_minimum_to_decode(want_to_decode, available_chunks,
+ &minimum_chunks);
+ EXPECT_EQ(-EINVAL, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, minimum_to_decode_10)
+{
+ //init
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ shec->init(*profile, &cerr);
+
+ //minimum_to_decode
+ set<int> want_to_decode;
+ set<int> available_chunks;
+ set<int> minimum_chunks;
+
+ for (int i = 0; i < 7; ++i) {
+ want_to_decode.insert(i);
+ }
+ for (int i = 4; i < 7; ++i) {
+ available_chunks.insert(i);
+ }
+
+ int r = shec->_minimum_to_decode(want_to_decode, available_chunks,
+ &minimum_chunks);
+ EXPECT_EQ(-EIO, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, minimum_to_decode_11)
+{
+ //init
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ shec->init(*profile, &cerr);
+
+ //minimum_to_decode
+ set<int> want_to_decode;
+ set<int> available_chunks;
+ set<int> minimum_chunks;
+
+ for (int i = 0; i < 5; ++i) {
+ want_to_decode.insert(i);
+ }
+ for (int i = 4; i < 7; ++i) {
+ available_chunks.insert(i);
+ }
+
+ int r = shec->_minimum_to_decode(want_to_decode, available_chunks,
+ &minimum_chunks);
+ EXPECT_EQ(-EIO, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, minimum_to_decode_12)
+{
+ //init
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ shec->init(*profile, &cerr);
+
+ //minimum_to_decode
+ set<int> want_to_decode;
+ set<int> available_chunks;
+ //minimum_chunks is NULL
+
+ for (int i = 0; i < 7; ++i) {
+ want_to_decode.insert(i);
+ available_chunks.insert(i);
+ }
+
+ int r = shec->_minimum_to_decode(want_to_decode, available_chunks, NULL);
+ EXPECT_EQ(-EINVAL, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, minimum_to_decode_13)
+{
+ //init
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ shec->init(*profile, &cerr);
+
+ //minimum_to_decode
+ set<int> want_to_decode;
+ set<int> available_chunks;
+ set<int> minimum_chunks, minimum;
+
+ for (int i = 0; i < 7; ++i) {
+ want_to_decode.insert(i);
+ available_chunks.insert(i);
+ }
+ shec->_minimum_to_decode(want_to_decode, available_chunks, &minimum_chunks);
+ minimum = minimum_chunks; //normal value
+ for (int i = 100; i < 120; ++i) {
+ minimum_chunks.insert(i); //insert extra data
+ }
+
+ int r = shec->_minimum_to_decode(want_to_decode, available_chunks,
+ &minimum_chunks);
+ EXPECT_TRUE(shec->matrix != NULL);
+ EXPECT_EQ(0, r);
+ EXPECT_EQ(minimum, minimum_chunks);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, minimum_to_decode2_1)
+{
+ //init
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ shec->init(*profile, &cerr);
+
+ //minimum_to_decode
+ set<int> want_to_decode;
+ set<int> available_chunks;
+ set<int> minimum_chunks;
+
+ want_to_decode.insert(0);
+ available_chunks.insert(0);
+ available_chunks.insert(1);
+ available_chunks.insert(2);
+
+ int r = shec->_minimum_to_decode(want_to_decode, available_chunks,
+ &minimum_chunks);
+ EXPECT_TRUE(shec->matrix != NULL);
+ EXPECT_EQ(0, r);
+ EXPECT_TRUE(minimum_chunks.size());
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, minimum_to_decode2_3)
+{
+ //init
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ shec->init(*profile, &cerr);
+
+ //minimum_to_decode
+ set<int> want_to_decode;
+ set<int> available_chunks;
+ set<int> minimum_chunks;
+
+ want_to_decode.insert(0);
+ want_to_decode.insert(2);
+ available_chunks.insert(0);
+ available_chunks.insert(1);
+ available_chunks.insert(2);
+ available_chunks.insert(3);
+
+ pthread_t tid;
+ g_flag = 0;
+ pthread_create(&tid, NULL, thread1, shec);
+ while (g_flag == 0) {
+ usleep(1);
+ }
+ sleep(1);
+ printf("*** test start ***\n");
+ int r = shec->_minimum_to_decode(want_to_decode, available_chunks,
+ &minimum_chunks);
+ EXPECT_TRUE(shec->matrix != NULL);
+ EXPECT_EQ(0, r);
+ EXPECT_EQ(want_to_decode, minimum_chunks);
+ printf("*** test end ***\n");
+ g_flag = 0;
+ pthread_join(tid, NULL);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, minimum_to_decode_with_cost_1)
+{
+ //init
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ shec->init(*profile, &cerr);
+
+ //minimum_to_decode_with_cost
+ set<int> want_to_decode;
+ map<int, int> available_chunks;
+ set<int> minimum_chunks;
+
+ for (int i = 0; i < 7; ++i) {
+ want_to_decode.insert(i);
+ available_chunks.insert(make_pair(i, i));
+ }
+
+ int r = shec->minimum_to_decode_with_cost(want_to_decode, available_chunks,
+ &minimum_chunks);
+ EXPECT_TRUE(shec->matrix != NULL);
+ EXPECT_EQ(0, r);
+ EXPECT_TRUE(minimum_chunks.size());
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, minimum_to_decode_with_cost_2_3)
+{
+ //init
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ shec->init(*profile, &cerr);
+
+ //minimum_to_decode_with_cost
+ set<int> want_to_decode;
+ map<int, int> available_chunks;
+ set<int> minimum_chunks;
+
+ want_to_decode.insert(0);
+ want_to_decode.insert(2);
+ available_chunks[0] = 0;
+ available_chunks[1] = 1;
+ available_chunks[2] = 2;
+ available_chunks[3] = 3;
+
+ pthread_t tid;
+ g_flag = 0;
+ pthread_create(&tid, NULL, thread2, shec);
+ while (g_flag == 0) {
+ usleep(1);
+ }
+ sleep(1);
+ printf("*** test start ***\n");
+ int r = shec->minimum_to_decode_with_cost(want_to_decode, available_chunks,
+ &minimum_chunks);
+ EXPECT_TRUE(shec->matrix != NULL);
+ EXPECT_EQ(0, r);
+ EXPECT_EQ(want_to_decode, minimum_chunks);
+ printf("*** test end ***\n");
+ g_flag = 0;
+ pthread_join(tid, NULL);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, encode_1)
+{
+ //init
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ shec->init(*profile, &cerr);
+
+ //encode
+ bufferlist in;
+ set<int> want_to_encode;
+ map<int, bufferlist> encoded;
+
+ in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124
+ "0123"//128
+ );
+ for (unsigned int i = 0; i < shec->get_chunk_count(); ++i) {
+ want_to_encode.insert(i);
+ }
+
+ int r = shec->encode(want_to_encode, in, &encoded);
+ EXPECT_EQ(0, r);
+ EXPECT_EQ(shec->get_chunk_count(), encoded.size());
+ EXPECT_EQ(shec->get_chunk_size(in.length()), encoded[0].length());
+
+ //decode
+ int want_to_decode[] = { 0, 1, 2, 3, 4, 5, 6 };
+ map<int, bufferlist> decoded;
+ decoded.clear();
+ r = shec->_decode(set<int>(want_to_decode, want_to_decode + 2),
+ encoded,
+ &decoded);
+ EXPECT_NE(nullptr, shec->matrix);
+ EXPECT_EQ(0, r);
+ EXPECT_EQ(2u, decoded.size());
+ EXPECT_EQ(32u, decoded[0].length());
+
+ bufferlist out1, out2, usable;
+ //out1 is "encoded"
+ for (unsigned int i = 0; i < encoded.size(); ++i) {
+ out1.append(encoded[i]);
+ }
+ //out2 is "decoded"
+ r = shec->decode_concat(encoded, &out2);
+ usable.substr_of(out2, 0, in.length());
+ EXPECT_FALSE(out1 == in);
+ EXPECT_TRUE(usable == in);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, encode_2)
+{
+ //init
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ shec->init(*profile, &cerr);
+
+ //encode
+ bufferlist in;
+ set<int> want_to_encode;
+ map<int, bufferlist> encoded;
+
+ in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124
+ );
+ for (unsigned int i = 0; i < shec->get_chunk_count(); ++i) {
+ want_to_encode.insert(i);
+ }
+
+ int r = shec->encode(want_to_encode, in, &encoded);
+ EXPECT_EQ(0, r);
+ EXPECT_EQ(shec->get_chunk_count(), encoded.size());
+ EXPECT_EQ(shec->get_chunk_size(in.length()), encoded[0].length());
+
+ //decode
+ int want_to_decode[] = { 0, 1, 2, 3, 4, 5, 6 };
+ map<int, bufferlist> decoded;
+ r = shec->_decode(set<int>(want_to_decode, want_to_decode + 2), encoded,
+ &decoded);
+ EXPECT_TRUE(shec->matrix != NULL);
+ EXPECT_EQ(0, r);
+ EXPECT_EQ(2u, decoded.size());
+ EXPECT_EQ(32u, decoded[0].length());
+
+ bufferlist out1, out2, usable;
+ //out1 is "encoded"
+ for (unsigned int i = 0; i < encoded.size(); ++i)
+ out1.append(encoded[i]);
+ //out2 is "decoded"
+ shec->decode_concat(encoded, &out2);
+ usable.substr_of(out2, 0, in.length());
+ EXPECT_FALSE(out1 == in);
+ EXPECT_TRUE(usable == in);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, encode_3)
+{
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ shec->init(*profile, &cerr);
+
+ bufferlist in;
+ in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124
+ );
+ set<int> want_to_encode;
+ for (unsigned int i = 0; i < shec->get_chunk_count(); ++i) {
+ want_to_encode.insert(i);
+ }
+ want_to_encode.insert(10);
+ want_to_encode.insert(11);
+ map<int, bufferlist> encoded;
+ int r = shec->encode(want_to_encode, in, &encoded);
+ EXPECT_EQ(0, r);
+ EXPECT_EQ(shec->get_chunk_count(), encoded.size());
+ EXPECT_EQ(shec->get_chunk_size(in.length()), encoded[0].length());
+
+ //decode
+ int want_to_decode[] = { 0, 1, 2, 3, 4, 5, 6 };
+ map<int, bufferlist> decoded;
+ r = shec->_decode(set<int>(want_to_decode, want_to_decode + 2), encoded,
+ &decoded);
+ EXPECT_TRUE(shec->matrix != NULL);
+ EXPECT_EQ(0, r);
+ EXPECT_EQ(2u, decoded.size());
+ EXPECT_EQ(shec->get_chunk_size(in.length()), decoded[0].length());
+
+ bufferlist out1, out2, usable;
+ //out1 is "encoded"
+ for (unsigned int i = 0; i < encoded.size(); ++i) {
+ out1.append(encoded[i]);
+ }
+ //out2 is "decoded"
+ shec->decode_concat(encoded, &out2);
+ usable.substr_of(out2, 0, in.length());
+ EXPECT_FALSE(out1 == in);
+ EXPECT_TRUE(usable == in);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, encode_4)
+{
+ //init
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ shec->init(*profile, &cerr);
+
+ //encode
+ bufferlist in;
+ set<int> want_to_encode;
+ map<int, bufferlist> encoded;
+
+ in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124
+ );
+ for (unsigned int i = 0; i < shec->get_chunk_count() - 1; ++i) {
+ want_to_encode.insert(i);
+ }
+ want_to_encode.insert(100);
+
+ int r = shec->encode(want_to_encode, in, &encoded);
+ EXPECT_EQ(0, r);
+ EXPECT_EQ(shec->get_chunk_count()-1, encoded.size());
+ EXPECT_EQ(shec->get_chunk_size(in.length()), encoded[0].length());
+
+ //decode
+ int want_to_decode[] = { 0, 1, 2, 3, 4, 5, 6 };
+ map<int, bufferlist> decoded;
+ r = shec->_decode(set<int>(want_to_decode, want_to_decode + 2), encoded,
+ &decoded);
+ EXPECT_TRUE(shec->matrix != NULL);
+ EXPECT_EQ(0, r);
+ EXPECT_EQ(2u, decoded.size());
+ EXPECT_EQ(shec->get_chunk_size(in.length()), decoded[0].length());
+
+ bufferlist out1, out2, usable;
+ //out1 is "encoded"
+ for (unsigned int i = 0; i < encoded.size(); ++i) {
+ out1.append(encoded[i]);
+ }
+ //out2 is "decoded"
+ shec->decode_concat(encoded, &out2);
+ usable.substr_of(out2, 0, in.length());
+ EXPECT_FALSE(out1 == in);
+ EXPECT_TRUE(usable == in);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, encode_8)
+{
+ //init
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ shec->init(*profile, &cerr);
+
+ //encode
+ bufferlist in;
+ set<int> want_to_encode;
+
+ in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124
+ );
+ for (unsigned int i = 0; i < shec->get_chunk_count(); ++i) {
+ want_to_encode.insert(i);
+ }
+
+ int r = shec->encode(want_to_encode, in, NULL); //encoded = NULL
+ EXPECT_EQ(-EINVAL, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, encode_9)
+{
+ //init
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ shec->init(*profile, &cerr);
+
+ //encode
+ bufferlist in;
+ set<int> want_to_encode;
+ map<int, bufferlist> encoded;
+
+ in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124
+ );
+ for (unsigned int i = 0; i < shec->get_chunk_count(); ++i) {
+ want_to_encode.insert(i);
+ }
+ for (int i = 0; i < 100; ++i) {
+ encoded[i].append("ABCDEFGHIJKLMNOPQRSTUVWXYZ");
+ }
+
+ int r = shec->encode(want_to_encode, in, &encoded);
+ EXPECT_EQ(-EINVAL, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, encode2_1)
+{
+ //init
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ shec->init(*profile, &cerr);
+
+ //encode
+ bufferlist in;
+ set<int> want_to_encode;
+ map<int, bufferlist> encoded;
+
+ in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124
+ "0123"//128
+ );
+ for (unsigned int i = 0; i < shec->get_chunk_count(); ++i) {
+ want_to_encode.insert(i);
+ }
+
+ int r = shec->encode(want_to_encode, in, &encoded);
+ EXPECT_EQ(0, r);
+ EXPECT_EQ(shec->get_chunk_count(), encoded.size());
+ EXPECT_EQ(shec->get_chunk_size(in.length()), encoded[0].length());
+
+ //decode
+ int want_to_decode[] = { 0, 1, 2, 3, 4, 5, 6 };
+ map<int, bufferlist> decoded;
+ r = shec->_decode(set<int>(want_to_decode, want_to_decode + 2), encoded,
+ &decoded);
+ EXPECT_TRUE(shec->matrix != NULL);
+ EXPECT_EQ(0, r);
+ EXPECT_EQ(2u, decoded.size());
+ EXPECT_EQ(32u, decoded[0].length());
+
+ bufferlist out1, out2, usable;
+ //out1 is "encoded"
+ for (unsigned int i = 0; i < encoded.size(); ++i) {
+ out1.append(encoded[i]);
+ }
+ //out2 is "decoded"
+ shec->decode_concat(encoded, &out2);
+ usable.substr_of(out2, 0, in.length());
+ EXPECT_FALSE(out1 == in);
+ EXPECT_TRUE(usable == in);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, encode2_3)
+{
+ //init
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ shec->init(*profile, &cerr);
+
+ //encode
+ bufferlist in;
+ set<int> want_to_encode;
+ map<int, bufferlist> encoded;
+
+ in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124
+ "0123"//128
+ );
+ for (unsigned int i = 0; i < shec->get_chunk_count(); ++i) {
+ want_to_encode.insert(i);
+ }
+
+ pthread_t tid;
+ g_flag = 0;
+ pthread_create(&tid, NULL, thread4, shec);
+ while (g_flag == 0) {
+ usleep(1);
+ }
+ sleep(1);
+ printf("*** test start ***\n");
+ int r = shec->encode(want_to_encode, in, &encoded);
+ EXPECT_EQ(0, r);
+ EXPECT_EQ(shec->get_chunk_count(), encoded.size());
+ EXPECT_EQ(shec->get_chunk_size(in.length()), encoded[0].length());
+ printf("*** test end ***\n");
+ g_flag = 0;
+ pthread_join(tid, NULL);
+
+ //decode
+ int want_to_decode[] = { 0, 1, 2, 3, 4, 5, 6 };
+ map<int, bufferlist> decoded;
+
+ r = shec->_decode(set<int>(want_to_decode, want_to_decode + 2), encoded,
+ &decoded);
+ EXPECT_TRUE(shec->matrix != NULL);
+ EXPECT_EQ(0, r);
+ EXPECT_EQ(2u, decoded.size());
+ EXPECT_EQ(32u, decoded[0].length());
+
+ bufferlist out1, out2, usable;
+ //out1 is "encoded"
+ for (unsigned int i = 0; i < encoded.size(); ++i) {
+ out1.append(encoded[i]);
+ }
+ //out2 is "decoded"
+ shec->decode_concat(encoded, &out2);
+ usable.substr_of(out2, 0, in.length());
+ EXPECT_FALSE(out1 == in);
+ EXPECT_TRUE(usable == in);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, decode_1)
+{
+ //init
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ shec->init(*profile, &cerr);
+
+ //encode
+ bufferlist in;
+ set<int> want_to_encode;
+ map<int, bufferlist> encoded;
+
+ in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//186
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//248
+ );
+ for (unsigned int i = 0; i < shec->get_chunk_count(); ++i) {
+ want_to_encode.insert(i);
+ }
+
+ int r = shec->encode(want_to_encode, in, &encoded);
+ EXPECT_EQ(0, r);
+ EXPECT_EQ(shec->get_chunk_count(), encoded.size());
+ EXPECT_EQ(shec->get_chunk_size(in.length()), encoded[0].length());
+
+ // all chunks are available
+ //decode
+ int want_to_decode[] = { 0, 1, 2, 3, 4, 5, 6 };
+ map<int, bufferlist> decoded;
+
+ r = shec->_decode(set<int>(want_to_decode, want_to_decode + 7), encoded,
+ &decoded);
+ EXPECT_TRUE(shec->matrix != NULL);
+ EXPECT_EQ(0, r);
+ EXPECT_EQ(7u, decoded.size());
+
+ bufferlist usable;
+ int cmp;
+ unsigned int c_size = shec->get_chunk_size(in.length());
+ for (unsigned int i = 0; i < shec->get_data_chunk_count(); ++i) {
+ usable.clear();
+ EXPECT_EQ(c_size, decoded[i].length());
+ if ( c_size * (i+1) <= in.length() ) {
+ usable.substr_of(in, c_size * i, c_size);
+ cmp = memcmp(decoded[i].c_str(), usable.c_str(), c_size);
+ } else {
+ usable.substr_of(in, c_size * i, in.length() % c_size);
+ cmp = memcmp(decoded[i].c_str(), usable.c_str(), in.length() % c_size);
+ }
+ EXPECT_EQ(0, cmp);
+ }
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, decode_8)
+{
+ //init
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ shec->init(*profile, &cerr);
+
+ //encode
+ bufferlist in;
+ set<int> want_to_encode;
+ map<int, bufferlist> encoded;
+
+ in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//186
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//248
+ );
+ for (unsigned int i = 0; i < shec->get_chunk_count(); ++i) {
+ want_to_encode.insert(i);
+ }
+
+ int r = shec->encode(want_to_encode, in, &encoded);
+ EXPECT_EQ(0, r);
+ EXPECT_EQ(shec->get_chunk_count(), encoded.size());
+ EXPECT_EQ(shec->get_chunk_size(in.length()), encoded[0].length());
+
+ // all chunks are available
+ //decode
+ int want_to_decode[] = { 0, 1, 2, 3, 4, 5, 6, 7 }; //more than k+m
+ map<int, bufferlist> decoded;
+
+ r = shec->_decode(set<int>(want_to_decode, want_to_decode + 8), encoded,
+ &decoded);
+ EXPECT_EQ(0, r);
+ EXPECT_EQ(7u, decoded.size());
+ EXPECT_EQ(shec->get_chunk_size(in.length()), encoded[0].length());
+
+ bufferlist usable;
+ int cmp;
+ unsigned int c_size = shec->get_chunk_size(in.length());
+ for (unsigned int i = 0; i < shec->get_data_chunk_count(); ++i) {
+ usable.clear();
+ EXPECT_EQ(c_size, decoded[i].length());
+ if ( c_size * (i+1) <= in.length() ) {
+ usable.substr_of(in, c_size * i, c_size);
+ cmp = memcmp(decoded[i].c_str(), usable.c_str(), c_size);
+ } else {
+ usable.substr_of(in, c_size * i, in.length() % c_size);
+ cmp = memcmp(decoded[i].c_str(), usable.c_str(), in.length() % c_size);
+ }
+ EXPECT_EQ(0, cmp);
+ }
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, decode_9)
+{
+ //init
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ shec->init(*profile, &cerr);
+
+ //encode
+ bufferlist in;
+ set<int> want_to_encode;
+ map<int, bufferlist> encoded;
+
+ in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//186
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//248
+ );
+ for (unsigned int i = 0; i < shec->get_chunk_count(); ++i) {
+ want_to_encode.insert(i);
+ }
+
+ int r = shec->encode(want_to_encode, in, &encoded);
+ EXPECT_EQ(0, r);
+ EXPECT_EQ(shec->get_chunk_count(), encoded.size());
+ EXPECT_EQ(shec->get_chunk_size(in.length()), encoded[0].length());
+
+ // all chunks are available
+ //decode
+ int want_to_decode[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
+ map<int, bufferlist> decoded;
+
+ //extra data
+ bufferlist buf;
+ buf.append("abc");
+ encoded[100] = buf;
+
+ r = shec->_decode(set<int>(want_to_decode, want_to_decode + 10), encoded,
+ &decoded);
+ EXPECT_TRUE(shec->matrix != NULL);
+ EXPECT_EQ(0, r);
+ EXPECT_EQ(7u, decoded.size());
+ EXPECT_EQ(shec->get_chunk_size(in.length()), decoded[0].length());
+
+ bufferlist out1, usable;
+ //out1 is "encoded"
+ for (unsigned int i = 0; i < encoded.size(); ++i) {
+ out1.append(encoded[i]);
+ }
+ EXPECT_FALSE(out1 == in);
+ //usable is "decoded"
+ int cmp;
+ unsigned int c_size = shec->get_chunk_size(in.length());
+ for (unsigned int i = 0; i < shec->get_data_chunk_count(); ++i) {
+ usable.clear();
+ EXPECT_EQ(c_size, decoded[i].length());
+ if ( c_size * (i+1) <= in.length() ) {
+ usable.substr_of(in, c_size * i, c_size);
+ cmp = memcmp(decoded[i].c_str(), usable.c_str(), c_size);
+ } else {
+ usable.substr_of(in, c_size * i, in.length() % c_size);
+ cmp = memcmp(decoded[i].c_str(), usable.c_str(), in.length() % c_size);
+ }
+ EXPECT_EQ(0, cmp);
+ }
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, decode_10)
+{
+ //init
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ shec->init(*profile, &cerr);
+
+ //encode
+ bufferlist in;
+ set<int> want_to_encode;
+ map<int, bufferlist> encoded;
+
+ in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//186
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//248
+ );
+ for (unsigned int i = 0; i < shec->get_chunk_count(); ++i) {
+ want_to_encode.insert(i);
+ }
+
+ int r = shec->encode(want_to_encode, in, &encoded);
+ EXPECT_EQ(0, r);
+ EXPECT_EQ(shec->get_chunk_count(), encoded.size());
+ EXPECT_EQ(shec->get_chunk_size(in.length()), encoded[0].length());
+
+ //decode
+ int want_to_decode[] = { 0, 1, 2, 3, 4, 5, 6 }; //more than k+m
+ map<int, bufferlist> decoded, inchunks;
+
+ for ( unsigned int i = 0; i < 3; ++i) {
+ inchunks.insert(make_pair(i, encoded[i]));
+ }
+
+ r = shec->_decode(set<int>(want_to_decode, want_to_decode + 7), inchunks,
+ &decoded);
+ EXPECT_EQ(-1, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, decode_11)
+{
+ //init
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ shec->init(*profile, &cerr);
+
+ //encode
+ bufferlist in;
+ set<int> want_to_encode;
+ map<int, bufferlist> encoded;
+
+ in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124
+ "ABCD"//128
+ );
+ for (unsigned int i = 0; i < shec->get_chunk_count(); ++i) {
+ want_to_encode.insert(i);
+ }
+
+ int r = shec->encode(want_to_encode, in, &encoded);
+ EXPECT_EQ(0, r);
+ EXPECT_EQ(shec->get_chunk_count(), encoded.size());
+ EXPECT_EQ(shec->get_chunk_size(in.length()), encoded[0].length());
+
+ //decode
+ int want_to_decode[] = { 0, 1, 2, 3, 4 };
+ map<int, bufferlist> decoded, inchunks;
+
+ for ( unsigned int i = 4; i < 7; ++i) {
+ inchunks.insert(make_pair(i, encoded[i]));
+ }
+
+ r = shec->_decode(set<int>(want_to_decode, want_to_decode + 5), inchunks,
+ &decoded);
+ EXPECT_EQ(-1, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, decode_12)
+{
+ //init
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ shec->init(*profile, &cerr);
+
+ //encode
+ bufferlist in;
+ set<int> want_to_encode;
+ map<int, bufferlist> encoded;
+
+ in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//186
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//248
+ );
+ for (unsigned int i = 0; i < shec->get_chunk_count(); ++i) {
+ want_to_encode.insert(i);
+ }
+
+ int r = shec->encode(want_to_encode, in, &encoded);
+ EXPECT_EQ(0, r);
+ EXPECT_EQ(shec->get_chunk_count(), encoded.size());
+ EXPECT_EQ(shec->get_chunk_size(in.length()), encoded[0].length());
+
+ // all chunks are available
+ //decode
+ int want_to_decode[] = { 0, 1, 2, 3, 4, 5, 6 };
+
+ //decoded = NULL
+ r = shec->_decode(set<int>(want_to_decode, want_to_decode + 7), encoded,
+ NULL);
+ EXPECT_NE(0, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, decode_13)
+{
+ //init
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ shec->init(*profile, &cerr);
+
+ //encode
+ bufferlist in;
+ set<int> want_to_encode;
+ map<int, bufferlist> encoded;
+
+ in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//186
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//248
+ );
+ for (unsigned int i = 0; i < shec->get_chunk_count(); ++i) {
+ want_to_encode.insert(i);
+ }
+
+ int r = shec->encode(want_to_encode, in, &encoded);
+ EXPECT_EQ(0, r);
+ EXPECT_EQ(shec->get_chunk_count(), encoded.size());
+ EXPECT_EQ(shec->get_chunk_size(in.length()), encoded[0].length());
+
+ // all chunks are available
+ //decode
+ int want_to_decode[] = { 0, 1, 2, 3, 4, 5, 6 };
+ map<int, bufferlist> decoded;
+
+ //extra data
+ bufferlist buf;
+ buf.append("a");
+ for (int i = 0; i < 100; ++i) {
+ decoded[i] = buf;
+ }
+
+ r = shec->_decode(set<int>(want_to_decode, want_to_decode + 7), encoded,
+ &decoded);
+ EXPECT_NE(0, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, decode2_1)
+{
+ //init
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ shec->init(*profile, &cerr);
+
+ //encode
+ bufferlist in;
+ set<int> want_to_encode;
+ map<int, bufferlist> encoded;
+
+ in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//186
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//248
+ );
+ for (unsigned int i = 0; i < shec->get_chunk_count(); ++i) {
+ want_to_encode.insert(i);
+ }
+
+ int r = shec->encode(want_to_encode, in, &encoded);
+ EXPECT_EQ(0, r);
+ EXPECT_EQ(shec->get_chunk_count(), encoded.size());
+ EXPECT_EQ(shec->get_chunk_size(in.length()), encoded[0].length());
+
+ // all chunks are available
+ //decode
+ int want_to_decode[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
+ map<int, bufferlist> decoded;
+
+ r = shec->_decode(set<int>(want_to_decode, want_to_decode + 2), encoded,
+ &decoded);
+ EXPECT_TRUE(shec->matrix != NULL);
+ EXPECT_EQ(0, r);
+ EXPECT_EQ(2u, decoded.size());
+
+ bufferlist out;
+ shec->decode_concat(encoded, &out);
+ bufferlist usable;
+ usable.substr_of(out, 0, in.length());
+ EXPECT_TRUE(usable == in);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, decode2_3)
+{
+ //init
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ shec->init(*profile, &cerr);
+
+ //encode
+ bufferlist in;
+ set<int> want_to_encode;
+ map<int, bufferlist> encoded;
+
+ in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//186
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//248
+ );
+ for (unsigned int i = 0; i < shec->get_chunk_count(); ++i) {
+ want_to_encode.insert(i);
+ }
+
+ int r = shec->encode(want_to_encode, in, &encoded);
+ EXPECT_EQ(0, r);
+ EXPECT_EQ(shec->get_chunk_count(), encoded.size());
+ EXPECT_EQ(shec->get_chunk_size(in.length()), encoded[0].length());
+
+ // all chunks are available
+ //decode
+ int want_to_decode[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
+ map<int, bufferlist> decoded;
+
+ pthread_t tid;
+ g_flag = 0;
+ pthread_create(&tid, NULL, thread4, shec);
+ while (g_flag == 0) {
+ usleep(1);
+ }
+ sleep(1);
+ printf("*** test start ***\n");
+ r = shec->_decode(set<int>(want_to_decode, want_to_decode + 2), encoded,
+ &decoded);
+ EXPECT_TRUE(shec->matrix != NULL);
+ EXPECT_EQ(0, r);
+ EXPECT_EQ(2u, decoded.size());
+ printf("*** test end ***\n");
+ g_flag = 0;
+ pthread_join(tid, NULL);
+
+ bufferlist out;
+ shec->decode_concat(encoded, &out);
+ bufferlist usable;
+ usable.substr_of(out, 0, in.length());
+ EXPECT_TRUE(usable == in);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, decode2_4)
+{
+ //init
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ shec->init(*profile, &cerr);
+
+ //encode
+ bufferlist in;
+ set<int> want_to_encode;
+ map<int, bufferlist> encoded;
+
+ in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//186
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//248
+ );
+ for (unsigned int i = 0; i < shec->get_chunk_count(); ++i) {
+ want_to_encode.insert(i);
+ }
+
+ int r = shec->encode(want_to_encode, in, &encoded);
+ EXPECT_EQ(0, r);
+ EXPECT_EQ(shec->get_chunk_count(), encoded.size());
+ EXPECT_EQ(shec->get_chunk_size(in.length()), encoded[0].length());
+
+ //decode
+ int want_to_decode[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
+ map<int, bufferlist> decoded;
+
+ // cannot recover
+ bufferlist out;
+ map<int, bufferlist> degraded;
+ degraded[0] = encoded[0];
+
+ r = shec->_decode(set<int>(want_to_decode, want_to_decode + 2), degraded,
+ &decoded);
+ EXPECT_EQ(-1, r);
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, create_rule_1_2)
+{
+ //create rule
+ CrushWrapper *crush = new CrushWrapper;
+ crush->create();
+ crush->set_type_name(2, "root");
+ crush->set_type_name(1, "host");
+ crush->set_type_name(0, "osd");
+
+ int rootno;
+ crush->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1, 2, 0, NULL,
+ NULL, &rootno);
+ crush->set_item_name(rootno, "default");
+
+ map < string, string > loc;
+ loc["root"] = "default";
+
+ int num_host = 2;
+ int num_osd = 5;
+ int osd = 0;
+ for (int h = 0; h < num_host; ++h) {
+ loc["host"] = string("host-") + stringify(h);
+ for (int o = 0; o < num_osd; ++o, ++osd) {
+ crush->insert_item(g_ceph_context, osd, 1.0,
+ string("osd.") + stringify(osd), loc);
+ }
+ }
+
+ //init
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ shec->init(*profile, &cerr);
+
+ //create_rule
+ stringstream ss;
+
+ int r = shec->create_rule("myrule", *crush, &ss);
+ EXPECT_EQ(0, r);
+ EXPECT_STREQ("myrule", crush->rule_name_map[0].c_str());
+
+ //reexecute create_rule
+ r = shec->create_rule("myrule", *crush, &ss);
+ EXPECT_EQ(-EEXIST, r);
+
+ delete shec;
+ delete profile;
+ delete crush;
+}
+
+TEST(ErasureCodeShec, create_rule_4)
+{
+ //create rule
+ CrushWrapper *crush = new CrushWrapper;
+ crush->create();
+ crush->set_type_name(2, "root");
+ crush->set_type_name(1, "host");
+ crush->set_type_name(0, "osd");
+
+ int rootno;
+ crush->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1, 2, 0, NULL,
+ NULL, &rootno);
+ crush->set_item_name(rootno, "default");
+
+ map < string, string > loc;
+ loc["root"] = "default";
+
+ int num_host = 2;
+ int num_osd = 5;
+ int osd = 0;
+ for (int h = 0; h < num_host; ++h) {
+ loc["host"] = string("host-") + stringify(h);
+ for (int o = 0; o < num_osd; ++o, ++osd) {
+ crush->insert_item(g_ceph_context, osd, 1.0,
+ string("osd.") + stringify(osd), loc);
+ }
+ }
+
+ //init
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ shec->init(*profile, &cerr);
+
+ //create_rule
+ int r = shec->create_rule("myrule", *crush, NULL); //ss = NULL
+ EXPECT_EQ(0, r);
+
+ delete shec;
+ delete profile;
+ delete crush;
+}
+
+TEST(ErasureCodeShec, create_rule2_1)
+{
+ //create rule
+ CrushWrapper *crush = new CrushWrapper;
+ crush->create();
+ crush->set_type_name(2, "root");
+ crush->set_type_name(1, "host");
+ crush->set_type_name(0, "osd");
+
+ int rootno;
+ crush->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1, 2, 0, NULL,
+ NULL, &rootno);
+ crush->set_item_name(rootno, "default");
+
+ map < string, string > loc;
+ loc["root"] = "default";
+
+ int num_host = 2;
+ int num_osd = 5;
+ int osd = 0;
+ for (int h = 0; h < num_host; ++h) {
+ loc["host"] = string("host-") + stringify(h);
+ for (int o = 0; o < num_osd; ++o, ++osd) {
+ crush->insert_item(g_ceph_context, osd, 1.0,
+ string("osd.") + stringify(osd), loc);
+ }
+ }
+
+ //init
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ shec->init(*profile, &cerr);
+
+ //create_rule
+ stringstream ss;
+
+ int r = shec->create_rule("myrule", *crush, &ss);
+ EXPECT_EQ(0, r);
+ EXPECT_STREQ("myrule", crush->rule_name_map[0].c_str());
+
+ delete shec;
+ delete profile;
+ delete crush;
+}
+
+struct CreateRuleset2_3_Param_d {
+ ErasureCodeShec *shec;
+ CrushWrapper *crush;
+};
+
+TEST(ErasureCodeShec, create_rule2_3)
+{
+ //create rule
+ CrushWrapper *crush = new CrushWrapper;
+ crush->create();
+ crush->set_type_name(2, "root");
+ crush->set_type_name(1, "host");
+ crush->set_type_name(0, "osd");
+
+ int rootno;
+ crush->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1, 2, 0, NULL,
+ NULL, &rootno);
+ crush->set_item_name(rootno, "default");
+
+ map < string, string > loc;
+ loc["root"] = "default";
+
+ int num_host = 2;
+ int num_osd = 5;
+ int osd = 0;
+ for (int h = 0; h < num_host; ++h) {
+ loc["host"] = string("host-") + stringify(h);
+ for (int o = 0; o < num_osd; ++o, ++osd) {
+ crush->insert_item(g_ceph_context, osd, 1.0,
+ string("osd.") + stringify(osd), loc);
+ }
+ }
+
+ //init
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ shec->init(*profile, &cerr);
+
+ //create_rule
+ stringstream ss;
+
+ pthread_t tid;
+ g_flag = 0;
+ pthread_create(&tid, NULL, thread3, shec);
+ while (g_flag == 0) {
+ usleep(1);
+ }
+ sleep(1);
+ printf("*** test start ***\n");
+ int r = (shec->create_rule("myrule", *crush, &ss));
+ EXPECT_TRUE(r >= 0);
+ printf("*** test end ***\n");
+ g_flag = 0;
+ pthread_join(tid, NULL);
+
+ delete shec;
+ delete profile;
+ delete crush;
+}
+
+TEST(ErasureCodeShec, get_chunk_count_1)
+{
+ //init
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ shec->init(*profile, &cerr);
+
+ //get_chunk_count
+ EXPECT_EQ(7u, shec->get_chunk_count());
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, get_data_chunk_count_1)
+{
+ //init
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ shec->init(*profile, &cerr);
+
+ //get_data_chunk_count
+ EXPECT_EQ(4u, shec->get_data_chunk_count());
+
+ delete shec;
+ delete profile;
+}
+
+TEST(ErasureCodeShec, get_chunk_size_1_2)
+{
+ //init
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = "4";
+ (*profile)["m"] = "3";
+ (*profile)["c"] = "2";
+ (*profile)["w"] = "8";
+ shec->init(*profile, &cerr);
+
+ //when there is no padding(128=k*w*4)
+ EXPECT_EQ(32u, shec->get_chunk_size(128));
+ //when there is padding(126=k*w*4-2)
+ EXPECT_EQ(32u, shec->get_chunk_size(126));
+
+ delete shec;
+ delete profile;
+}
+
+void* thread1(void* pParam)
+{
+ ErasureCodeShec* shec = (ErasureCodeShec*) pParam;
+ set<int> want_to_decode;
+ set<int> available_chunks;
+ set<int> minimum_chunks;
+
+ want_to_decode.insert(0);
+ want_to_decode.insert(1);
+ available_chunks.insert(0);
+ available_chunks.insert(1);
+ available_chunks.insert(2);
+
+ printf("*** thread loop start ***\n");
+ g_flag = 1;
+ while (g_flag == 1) {
+ shec->_minimum_to_decode(want_to_decode, available_chunks, &minimum_chunks);
+ }
+ printf("*** thread loop end ***\n");
+
+ return NULL;
+}
+
+void* thread2(void* pParam)
+{
+ ErasureCodeShec* shec = (ErasureCodeShec*) pParam;
+ set<int> want_to_decode;
+ map<int, int> available_chunks;
+ set<int> minimum_chunks;
+
+ want_to_decode.insert(0);
+ want_to_decode.insert(1);
+ available_chunks[0] = 0;
+ available_chunks[1] = 1;
+ available_chunks[2] = 2;
+
+ printf("*** thread loop start ***\n");
+ g_flag = 1;
+ while (g_flag == 1) {
+ shec->minimum_to_decode_with_cost(want_to_decode, available_chunks,
+ &minimum_chunks);
+ minimum_chunks.clear();
+ }
+ printf("*** thread loop end ***\n");
+
+ return NULL;
+}
+
+void* thread3(void* pParam)
+{
+ ErasureCodeShec* shec = (ErasureCodeShec*) pParam;
+
+ std::unique_ptr<CrushWrapper> crush = std::make_unique<CrushWrapper>();
+ crush->create();
+ crush->set_type_name(2, "root");
+ crush->set_type_name(1, "host");
+ crush->set_type_name(0, "osd");
+
+ int rootno;
+ crush->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1, 2, 0, NULL,
+ NULL, &rootno);
+ crush->set_item_name(rootno, "default");
+
+ map < string, string > loc;
+ loc["root"] = "default";
+
+ int num_host = 2;
+ int num_osd = 5;
+ int osd = 0;
+ for (int h = 0; h < num_host; ++h) {
+ loc["host"] = string("host-") + stringify(h);
+ for (int o = 0; o < num_osd; ++o, ++osd) {
+ crush->insert_item(g_ceph_context, osd, 1.0,
+ string("osd.") + stringify(osd), loc);
+ }
+ }
+
+ stringstream ss;
+ int i = 0;
+ char name[30];
+
+ printf("*** thread loop start ***\n");
+ g_flag = 1;
+ while (g_flag == 1) {
+ sprintf(name, "myrule%d", i);
+ shec->create_rule(name, *crush, &ss);
+ ++i;
+ }
+ printf("*** thread loop end ***\n");
+
+ return NULL;
+}
+
+void* thread4(void* pParam)
+{
+ ErasureCodeShec* shec = (ErasureCodeShec*) pParam;
+
+ bufferlist in;
+ in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//186
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//248
+ );
+ set<int> want_to_encode;
+ for (unsigned int i = 0; i < shec->get_chunk_count(); ++i) {
+ want_to_encode.insert(i);
+ }
+
+ map<int, bufferlist> encoded;
+
+ printf("*** thread loop start ***\n");
+ g_flag = 1;
+ while (g_flag == 1) {
+ shec->encode(want_to_encode, in, &encoded);
+ encoded.clear();
+ }
+ printf("*** thread loop end ***\n");
+
+ return NULL;
+}
+
+void* thread5(void* pParam)
+{
+ ErasureCodeShec* shec = (ErasureCodeShec*) pParam;
+
+ bufferlist in;
+ in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//186
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//248
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//310
+ );
+ set<int> want_to_encode;
+ for (unsigned int i = 0; i < shec->get_chunk_count(); ++i) {
+ want_to_encode.insert(i);
+ }
+ map<int, bufferlist> encoded;
+ shec->encode(want_to_encode, in, &encoded);
+
+ int want_to_decode[] = { 0, 1, 2, 3, 4, 5 };
+ map<int, bufferlist> decoded;
+
+ printf("*** thread loop start ***\n");
+ g_flag = 1;
+ while (g_flag == 1) {
+ shec->_decode(set<int>(want_to_decode, want_to_decode + 2), encoded,
+ &decoded);
+ decoded.clear();
+ }
+ printf("*** thread loop end ***\n");
+
+ return NULL;
+}
diff --git a/src/test/erasure-code/TestErasureCodeShec_all.cc b/src/test/erasure-code/TestErasureCodeShec_all.cc
new file mode 100644
index 000000000..401b8affc
--- /dev/null
+++ b/src/test/erasure-code/TestErasureCodeShec_all.cc
@@ -0,0 +1,332 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2014,2015 FUJITSU LIMITED
+ *
+ * Author: Shotaro Kawaguchi <kawaguchi.s@jp.fujitsu.com>
+ * Author: Takanori Nakao <nakao.takanori@jp.fujitsu.com>
+ * Author: Takeshi Miyamae <miyamae.takeshi@jp.fujitsu.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+// SUMMARY: TestErasureCodeShec combination of k,m,c by 301 patterns
+
+#include <errno.h>
+#include <stdlib.h>
+
+#include "crush/CrushWrapper.h"
+#include "osd/osd_types.h"
+#include "include/stringify.h"
+#include "global/global_init.h"
+#include "erasure-code/shec/ErasureCodeShec.h"
+#include "erasure-code/ErasureCodePlugin.h"
+#include "common/ceph_argparse.h"
+#include "global/global_context.h"
+#include "gtest/gtest.h"
+
+using namespace std;
+
+struct Param_d {
+ char* k;
+ char* m;
+ char* c;
+ int ch_size;
+ char sk[16];
+ char sm[16];
+ char sc[16];
+};
+struct Param_d param[301];
+
+unsigned int g_recover = 0;
+unsigned int g_cannot_recover = 0;
+struct Recover_d {
+ int k;
+ int m;
+ int c;
+ set<int> want;
+ set<int> avail;
+};
+struct std::vector<Recover_d> cannot_recover;
+
+class ParameterTest : public ::testing::TestWithParam<struct Param_d> {
+
+};
+
+TEST_P(ParameterTest, parameter_all)
+{
+ int result;
+ //get parameters
+ char* k = GetParam().k;
+ char* m = GetParam().m;
+ char* c = GetParam().c;
+ unsigned c_size = GetParam().ch_size;
+ int i_k = atoi(k);
+ int i_m = atoi(m);
+ int i_c = atoi(c);
+
+ //init
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = k;
+ (*profile)["m"] = m;
+ (*profile)["c"] = c;
+
+ result = shec->init(*profile, &cerr);
+
+ //check profile
+ EXPECT_EQ(i_k, shec->k);
+ EXPECT_EQ(i_m, shec->m);
+ EXPECT_EQ(i_c, shec->c);
+ EXPECT_EQ(8, shec->w);
+ EXPECT_EQ(ErasureCodeShec::MULTIPLE, shec->technique);
+ EXPECT_STREQ("default", shec->rule_root.c_str());
+ EXPECT_STREQ("osd", shec->rule_failure_domain.c_str());
+ EXPECT_TRUE(shec->matrix != NULL);
+ EXPECT_EQ(0, result);
+
+ //minimum_to_decode
+ //want_to_decode will be a combination that chooses 1~c from k+m
+ set<int> want_to_decode, available_chunks, minimum_chunks;
+ int array_want_to_decode[shec->get_chunk_count()];
+ struct Recover_d comb;
+
+ for (int w = 1; w <= i_c; w++) {
+ const unsigned int r = w; // combination(k+m,r)
+
+ for (unsigned int i = 0; i < r; ++i) {
+ array_want_to_decode[i] = 1;
+ }
+ for (unsigned int i = r; i < shec->get_chunk_count(); ++i) {
+ array_want_to_decode[i] = 0;
+ }
+
+ do {
+ for (unsigned int i = 0; i < shec->get_chunk_count(); i++) {
+ available_chunks.insert(i);
+ }
+ for (unsigned int i = 0; i < shec->get_chunk_count(); i++) {
+ if (array_want_to_decode[i]) {
+ want_to_decode.insert(i);
+ available_chunks.erase(i);
+ }
+ }
+
+ result = shec->_minimum_to_decode(want_to_decode, available_chunks,
+ &minimum_chunks);
+
+ if (result == 0){
+ EXPECT_EQ(0, result);
+ EXPECT_TRUE(minimum_chunks.size());
+ g_recover++;
+ } else {
+ EXPECT_EQ(-EIO, result);
+ EXPECT_EQ(0u, minimum_chunks.size());
+ g_cannot_recover++;
+ comb.k = shec->k;
+ comb.m = shec->m;
+ comb.c = shec->c;
+ comb.want = want_to_decode;
+ comb.avail = available_chunks;
+ cannot_recover.push_back(comb);
+ }
+
+ want_to_decode.clear();
+ available_chunks.clear();
+ minimum_chunks.clear();
+ } while (std::prev_permutation(
+ array_want_to_decode,
+ array_want_to_decode + shec->get_chunk_count()));
+ }
+
+ //minimum_to_decode_with_cost
+ set<int> want_to_decode_with_cost, minimum_chunks_with_cost;
+ map<int, int> available_chunks_with_cost;
+
+ for (unsigned int i = 0; i < 1; i++) {
+ want_to_decode_with_cost.insert(i);
+ }
+ for (unsigned int i = 0; i < shec->get_chunk_count(); i++) {
+ available_chunks_with_cost[i] = i;
+ }
+
+ result = shec->minimum_to_decode_with_cost(
+ want_to_decode_with_cost,
+ available_chunks_with_cost,
+ &minimum_chunks_with_cost);
+ EXPECT_EQ(0, result);
+ EXPECT_TRUE(minimum_chunks_with_cost.size());
+
+ //encode
+ bufferlist in;
+ set<int> want_to_encode;
+ map<int, bufferlist> encoded;
+
+ in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//186
+ "012345"//192
+ );
+ for (unsigned int i = 0; i < shec->get_chunk_count(); i++) {
+ want_to_encode.insert(i);
+ }
+
+ result = shec->encode(want_to_encode, in, &encoded);
+ EXPECT_EQ(0, result);
+ EXPECT_EQ(i_k+i_m, (int)encoded.size());
+ EXPECT_EQ(c_size, encoded[0].length());
+
+ //decode
+ int want_to_decode2[i_k + i_m];
+ map<int, bufferlist> decoded;
+
+ for (unsigned int i = 0; i < shec->get_chunk_count(); i++) {
+ want_to_decode2[i] = i;
+ }
+
+ result = shec->_decode(set<int>(want_to_decode2, want_to_decode2 + 2),
+ encoded, &decoded);
+ EXPECT_EQ(0, result);
+ EXPECT_EQ(2u, decoded.size());
+ EXPECT_EQ(c_size, decoded[0].length());
+
+ //check encoded,decoded
+ bufferlist out1, out2, usable;
+
+ //out1 is "encoded"
+ for (unsigned int i = 0; i < encoded.size(); i++) {
+ out1.append(encoded[i]);
+ }
+
+ //out2 is "decoded"
+ shec->decode_concat(encoded, &out2);
+ usable.substr_of(out2, 0, in.length());
+
+ EXPECT_FALSE(out1 == in);
+ EXPECT_TRUE(usable == in);
+
+ //create_rule
+ stringstream ss;
+ CrushWrapper *crush = new CrushWrapper;
+ crush->create();
+ crush->set_type_name(2, "root");
+ crush->set_type_name(1, "host");
+ crush->set_type_name(0, "osd");
+
+ int rootno;
+ crush->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1, 2, 0, NULL,
+ NULL, &rootno);
+ crush->set_item_name(rootno, "default");
+
+ map < string, string > loc;
+ loc["root"] = "default";
+
+ int num_host = 2;
+ int num_osd = 5;
+ int osd = 0;
+ for (int h = 0; h < num_host; ++h) {
+ loc["host"] = string("host-") + stringify(h);
+ for (int o = 0; o < num_osd; ++o, ++osd) {
+ crush->insert_item(g_ceph_context, osd, 1.0,
+ string("osd.") + stringify(osd), loc);
+ }
+ }
+
+ result = shec->create_rule("myrule", *crush, &ss);
+ EXPECT_EQ(0, result);
+ EXPECT_STREQ("myrule", crush->rule_name_map[0].c_str());
+
+ //get_chunk_count
+ EXPECT_EQ(i_k+i_m, (int)shec->get_chunk_count());
+
+ //get_data_chunk_count
+ EXPECT_EQ(i_k, (int)shec->get_data_chunk_count());
+
+ //get_chunk_size
+ EXPECT_EQ(c_size, shec->get_chunk_size(192));
+
+ delete shec;
+ delete profile;
+ delete crush;
+}
+
+INSTANTIATE_TEST_SUITE_P(Test, ParameterTest, ::testing::ValuesIn(param));
+
+int main(int argc, char **argv)
+{
+ int i = 0;
+ int r;
+ const int kObjectSize = 192;
+ unsigned alignment, tail, padded_length;
+ float recovery_percentage;
+
+ //make_kmc
+ for (unsigned int k = 1; k <= 12; k++) {
+ for (unsigned int m = 1; (m <= k) && (k + m <= 20); m++) {
+ for (unsigned int c = 1; c <= m; c++) {
+ sprintf(param[i].sk, "%u", k);
+ sprintf(param[i].sm, "%u", m);
+ sprintf(param[i].sc, "%u", c);
+
+ param[i].k = param[i].sk;
+ param[i].m = param[i].sm;
+ param[i].c = param[i].sc;
+
+ alignment = k * 8 * sizeof(int);
+ tail = kObjectSize % alignment;
+ padded_length = kObjectSize + (tail ? (alignment - tail) : 0);
+ param[i].ch_size = padded_length / k;
+ i++;
+ }
+ }
+ }
+
+ auto args = argv_to_vec(argc, argv);
+
+ auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT,
+ CODE_ENVIRONMENT_UTILITY,
+ CINIT_FLAG_NO_MON_CONFIG);
+ common_init_finish(g_ceph_context);
+
+ ::testing::InitGoogleTest(&argc, argv);
+
+ r = RUN_ALL_TESTS();
+
+ std::cout << "minimum_to_decode:recover_num = " << g_recover << std::endl;
+ std::cout << "minimum_to_decode:cannot_recover_num = " << g_cannot_recover
+ << std::endl;
+ recovery_percentage = 100.0
+ - (float) (100.0 * g_cannot_recover / (g_recover + g_cannot_recover));
+ printf("recovery_percentage:%f\n",recovery_percentage);
+ if (recovery_percentage > 99.0) {
+ std::cout << "[ OK ] Recovery percentage is more than 99.0%"
+ << std::endl;
+ } else {
+ std::cout << "[ NG ] Recovery percentage is less than 99.0%"
+ << std::endl;
+ }
+ std::cout << "cannot recovery patterns:" << std::endl;
+ for (std::vector<Recover_d>::const_iterator i = cannot_recover.begin();
+ i != cannot_recover.end(); ++i) {
+ std::cout << "---" << std::endl;
+ std::cout << "k = " << i->k << ", m = " << i->m << ", c = " << i->c
+ << std::endl;
+ std::cout << "want_to_decode :" << i->want << std::endl;
+ std::cout << "available_chunks:" << i->avail << std::endl;
+ }
+ std::cout << "---" << std::endl;
+
+ return r;
+}
diff --git a/src/test/erasure-code/TestErasureCodeShec_arguments.cc b/src/test/erasure-code/TestErasureCodeShec_arguments.cc
new file mode 100644
index 000000000..075c6383e
--- /dev/null
+++ b/src/test/erasure-code/TestErasureCodeShec_arguments.cc
@@ -0,0 +1,370 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2015 FUJITSU LIMITED
+ *
+ * Author: Shotaro Kawaguchi <kawaguchi.s@jp.fujitsu.com>
+ * Author: Takanori Nakao <nakao.takanori@jp.fujitsu.com>
+ * Author: Takeshi Miyamae <miyamae.takeshi@jp.fujitsu.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+// SUMMARY: shec's gtest for each argument of minimum_to_decode()/decode()
+
+#include <algorithm>
+#include <bit>
+#include <cerrno>
+#include <cstdlib>
+
+#include "crush/CrushWrapper.h"
+#include "osd/osd_types.h"
+#include "include/stringify.h"
+#include "global/global_init.h"
+#include "erasure-code/shec/ErasureCodeShec.h"
+#include "erasure-code/ErasureCodePlugin.h"
+#include "common/ceph_argparse.h"
+#include "global/global_context.h"
+#include "gtest/gtest.h"
+
+using namespace std;
+
+unsigned int count_num = 0;
+unsigned int unexpected_count = 0;
+unsigned int value_count = 0;
+
+map<set<int>,set<set<int> > > shec_table;
+
+constexpr int getint(std::initializer_list<int> is) {
+ int a = 0;
+ for (const auto i : is) {
+ a |= 1 << i;
+ }
+ return a;
+}
+
+void create_table_shec432() {
+ set<int> table_key,vec_avails;
+ set<set<int> > table_value;
+
+ for (int want_count = 0; want_count < 7; ++want_count) {
+ for (unsigned want = 1; want < (1<<7); ++want) {
+ table_key.clear();
+ table_value.clear();
+ if (std::popcount(want) != want_count) {
+ continue;
+ }
+ {
+ for (int i = 0; i < 7; ++i) {
+ if (want & (1 << i)) {
+ table_key.insert(i);
+ }
+ }
+ }
+ vector<int> vec;
+ for (unsigned avails = 0; avails < (1<<7); ++avails) {
+ if (want & avails) {
+ continue;
+ }
+ if (std::popcount(avails) == 2 &&
+ std::popcount(want) == 1) {
+ if (std::cmp_equal(want | avails, getint({0,1,5})) ||
+ std::cmp_equal(want | avails, getint({2,3,6}))) {
+ vec.push_back(avails);
+ }
+ }
+ }
+
+ for (unsigned avails = 0; avails < (1<<7); ++avails) {
+ if (want & avails) {
+ continue;
+ }
+ if (std::popcount(avails) == 4) {
+ auto a = to_array<std::initializer_list<int>>({
+ {0,1,2,3}, {0,1,2,4}, {0,1,2,6}, {0,1,3,4}, {0,1,3,6}, {0,1,4,6},
+ {0,2,3,4}, {0,2,3,5}, {0,2,4,5}, {0,2,4,6}, {0,2,5,6}, {0,3,4,5},
+ {0,3,4,6}, {0,3,5,6}, {0,4,5,6}, {1,2,3,4}, {1,2,3,5}, {1,2,4,5},
+ {1,2,4,6}, {1,2,5,6}, {1,3,4,5}, {1,3,4,6}, {1,3,5,6}, {1,4,5,6},
+ {2,3,4,5}, {2,4,5,6}, {3,4,5,6}});
+ if (ranges::any_of(a, std::bind_front(cmp_equal<uint, int>, avails),
+ getint)) {
+ vec.push_back(avails);
+ }
+ }
+ }
+ for (int i = 0; i < (int)vec.size(); ++i) {
+ for (int j = i + 1; j < (int)vec.size(); ++j) {
+ if ((vec[i] & vec[j]) == vec[i]) {
+ vec.erase(vec.begin() + j);
+ --j;
+ }
+ }
+ }
+ for (int i = 0; i < (int)vec.size(); ++i) {
+ vec_avails.clear();
+ for (int j = 0; j < 7; ++j) {
+ if (vec[i] & (1 << j)) {
+ vec_avails.insert(j);
+ }
+ }
+ table_value.insert(vec_avails);
+ }
+ shec_table.insert(std::make_pair(table_key,table_value));
+ }
+ }
+}
+
+bool search_table_shec432(set<int> want_to_read, set<int> available_chunks) {
+ set<set<int> > tmp;
+ set<int> settmp;
+ bool found;
+
+ tmp = shec_table.find(want_to_read)->second;
+ for (set<set<int> >::iterator itr = tmp.begin();itr != tmp.end(); ++itr) {
+ found = true;
+ value_count = 0;
+ settmp = *itr;
+ for (set<int>::iterator setitr = settmp.begin();setitr != settmp.end(); ++setitr) {
+ if (!available_chunks.count(*setitr)) {
+ found = false;
+ }
+ ++value_count;
+ }
+ if (found) {
+ return true;
+ }
+ }
+ return false;
+}
+
+TEST(ParameterTest, combination_all)
+{
+ const unsigned int kObjectSize = 128;
+
+ //get profile
+ char* k = (char*)"4";
+ char* m = (char*)"3";
+ char* c = (char*)"2";
+ int i_k = atoi(k);
+ int i_m = atoi(m);
+ int i_c = atoi(c);
+ const unsigned alignment = i_k * 8 * sizeof(int);
+ const unsigned tail = kObjectSize % alignment;
+ const unsigned padded_length = kObjectSize + (tail ? (alignment - tail) : 0);
+ const unsigned c_size = padded_length / i_k;
+
+ //init
+ ErasureCodeShecTableCache tcache;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ map < std::string, std::string > *profile = new map<std::string,
+ std::string>();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = k;
+ (*profile)["m"] = m;
+ (*profile)["c"] = c;
+
+ int result = shec->init(*profile, &cerr);
+
+ //check profile
+ EXPECT_EQ(i_k, shec->k);
+ EXPECT_EQ(i_m, shec->m);
+ EXPECT_EQ(i_c, shec->c);
+ EXPECT_EQ(8, shec->w);
+ EXPECT_EQ(ErasureCodeShec::MULTIPLE, shec->technique);
+ EXPECT_STREQ("default", shec->rule_root.c_str());
+ EXPECT_STREQ("osd", shec->rule_failure_domain.c_str());
+ EXPECT_TRUE(shec->matrix != NULL);
+ EXPECT_EQ(0, result);
+
+ //encode
+ bufferlist in;
+ in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124
+ "0123"//128
+ );
+ set<int> want_to_encode;
+ for (unsigned int i = 0; i < shec->get_chunk_count(); ++i) {
+ want_to_encode.insert(i);
+ }
+
+ map<int, bufferlist> encoded;
+ result = shec->encode(want_to_encode, in, &encoded);
+ EXPECT_EQ(0, result);
+ EXPECT_EQ(i_k+i_m, (int)encoded.size());
+ EXPECT_EQ(c_size, encoded[0].length());
+ bufferlist out1;
+ //out1 is "encoded"
+ for (unsigned int i = 0; i < encoded.size(); ++i) {
+ out1.append(encoded[i]);
+ }
+ EXPECT_FALSE(out1 == in);
+
+ for (unsigned int w1 = 0; w1 <= shec->get_chunk_count(); ++w1) {
+ // combination(k+m,w1)
+ int array_want_to_read[shec->get_chunk_count()];
+ for (unsigned int i = 0; i < shec->get_chunk_count(); ++i) {
+ array_want_to_read[i] = i < w1 ? 1 : 0;
+ }
+
+ for (unsigned w2 = 0; w2 <= shec->get_chunk_count(); ++w2) {
+ // combination(k+m,w2)
+ int array_available_chunks[shec->get_chunk_count()];
+ for (unsigned int i = 0; i < shec->get_chunk_count(); ++i ) {
+ array_available_chunks[i] = i < w2 ? 1 : 0;
+ }
+
+ do {
+ do {
+ set<int> want_to_read, available_chunks;
+ map<int, bufferlist> inchunks;
+ for (unsigned int i = 0; i < shec->get_chunk_count(); ++i) {
+ if (array_want_to_read[i]) {
+ want_to_read.insert(i);
+ }
+ if (array_available_chunks[i]) {
+ available_chunks.insert(i);
+ inchunks.insert(make_pair(i,encoded[i]));
+ }
+ }
+
+ map<int, vector<pair<int,int>>> minimum_chunks;
+ map<int, bufferlist> decoded;
+ result = shec->minimum_to_decode(want_to_read, available_chunks,
+ &minimum_chunks);
+ int dresult = shec->decode(want_to_read, inchunks, &decoded,
+ shec->get_chunk_size(kObjectSize));
+ ++count_num;
+ unsigned int minimum_count = 0;
+
+ if (want_to_read.size() == 0) {
+ EXPECT_EQ(0, result);
+ EXPECT_EQ(0u, minimum_chunks.size());
+ EXPECT_EQ(0, dresult);
+ EXPECT_EQ(0u, decoded.size());
+ EXPECT_EQ(0u, decoded[0].length());
+ if (result != 0 || dresult != 0) {
+ ++unexpected_count;
+ }
+ } else {
+ // want - avail
+ set<int> want_to_read_without_avails;
+ for (auto chunk : want_to_read) {
+ if (!available_chunks.count(chunk)) {
+ want_to_read_without_avails.insert(chunk);
+ } else {
+ ++minimum_count;
+ }
+ }
+
+ if (want_to_read_without_avails.size() == 0) {
+ EXPECT_EQ(0, result);
+ EXPECT_LT(0u, minimum_chunks.size());
+ EXPECT_GE(minimum_count, minimum_chunks.size());
+ EXPECT_EQ(0, dresult);
+ EXPECT_NE(0u, decoded.size());
+ for (unsigned int i = 0; i < shec->get_data_chunk_count(); ++i) {
+ if (array_want_to_read[i]) {
+ bufferlist usable;
+ usable.substr_of(in, c_size * i, c_size);
+ int cmp = memcmp(decoded[i].c_str(), usable.c_str(), c_size);
+ EXPECT_EQ(c_size, decoded[i].length());
+ EXPECT_EQ(0, cmp);
+ if (cmp != 0) {
+ ++unexpected_count;
+ }
+ }
+ }
+ if (result != 0 || dresult != 0) {
+ ++unexpected_count;
+ }
+ } else if (want_to_read_without_avails.size() > 3) {
+ EXPECT_EQ(-EIO, result);
+ EXPECT_EQ(0u, minimum_chunks.size());
+ EXPECT_EQ(-1, dresult);
+ if (result != -EIO || dresult != -1) {
+ ++unexpected_count;
+ }
+ } else {
+ // search
+ if (search_table_shec432(want_to_read_without_avails,available_chunks)) {
+ EXPECT_EQ(0, result);
+ EXPECT_LT(0u, minimum_chunks.size());
+ EXPECT_GE(value_count + minimum_count, minimum_chunks.size());
+ EXPECT_EQ(0, dresult);
+ EXPECT_NE(0u, decoded.size());
+ for (unsigned int i = 0; i < shec->get_data_chunk_count(); ++i) {
+ if (array_want_to_read[i]) {
+ bufferlist usable;
+ usable.substr_of(in, c_size * i, c_size);
+ int cmp = memcmp(decoded[i].c_str(), usable.c_str(), c_size);
+ EXPECT_EQ(c_size, decoded[i].length());
+ EXPECT_EQ(0, cmp);
+ if (cmp != 0) {
+ ++unexpected_count;
+ std::cout << "decoded[" << i << "] = " << decoded[i].c_str() << std::endl;
+ std::cout << "usable = " << usable.c_str() << std::endl;
+ std::cout << "want_to_read :" << want_to_read << std::endl;
+ std::cout << "available_chunks:" << available_chunks << std::endl;
+ std::cout << "minimum_chunks :" << minimum_chunks << std::endl;
+ }
+ }
+ }
+ if (result != 0 || dresult != 0) {
+ ++unexpected_count;
+ }
+ } else {
+ EXPECT_EQ(-EIO, result);
+ EXPECT_EQ(0u, minimum_chunks.size());
+ EXPECT_EQ(-1, dresult);
+ if (result != -EIO || dresult != -1) {
+ ++unexpected_count;
+ }
+ }
+ }
+ }
+ } while (std::prev_permutation(
+ array_want_to_read,
+ array_want_to_read + shec->get_chunk_count()));
+
+ } while (std::prev_permutation(
+ array_available_chunks,
+ array_available_chunks + shec->get_chunk_count()));
+ }
+ }
+
+ delete shec;
+ delete profile;
+}
+
+int main(int argc, char **argv)
+{
+ auto args = argv_to_vec(argc, argv);
+
+ auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT,
+ CODE_ENVIRONMENT_UTILITY,
+ CINIT_FLAG_NO_MON_CONFIG);
+ common_init_finish(g_ceph_context);
+
+ ::testing::InitGoogleTest(&argc, argv);
+
+ create_table_shec432();
+
+ int r = RUN_ALL_TESTS();
+
+ std::cout << "minimum_to_decode:total_num = " << count_num
+ << std::endl;
+ std::cout << "minimum_to_decode:unexpected_num = " << unexpected_count
+ << std::endl;
+
+ return r;
+}
diff --git a/src/test/erasure-code/TestErasureCodeShec_thread.cc b/src/test/erasure-code/TestErasureCodeShec_thread.cc
new file mode 100644
index 000000000..c8d7bbb1e
--- /dev/null
+++ b/src/test/erasure-code/TestErasureCodeShec_thread.cc
@@ -0,0 +1,219 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2014,2015 FUJITSU LIMITED
+ *
+ * Author: Shotaro Kawaguchi <kawaguchi.s@jp.fujitsu.com>
+ * Author: Takanori Nakao <nakao.takanori@jp.fujitsu.com>
+ * Author: Takeshi Miyamae <miyamae.takeshi@jp.fujitsu.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+// SUMMARY: TestErasureCodeShec executes some threads at the same time
+
+#include <errno.h>
+#include <pthread.h>
+#include <stdlib.h>
+
+#include "crush/CrushWrapper.h"
+#include "osd/osd_types.h"
+#include "include/stringify.h"
+#include "erasure-code/shec/ErasureCodeShec.h"
+#include "erasure-code/ErasureCodePlugin.h"
+#include "global/global_context.h"
+#include "gtest/gtest.h"
+
+using namespace std;
+
+void* thread1(void* pParam);
+
+class TestParam {
+public:
+ string k, m, c, w;
+};
+
+TEST(ErasureCodeShec, thread)
+{
+ TestParam param1, param2, param3, param4, param5;
+ param1.k = "6";
+ param1.m = "4";
+ param1.c = "3";
+ param1.w = "8";
+
+ param2.k = "4";
+ param2.m = "3";
+ param2.c = "2";
+ param2.w = "16";
+
+ param3.k = "10";
+ param3.m = "8";
+ param3.c = "4";
+ param3.w = "32";
+
+ param4.k = "5";
+ param4.m = "5";
+ param4.c = "5";
+ param4.w = "8";
+
+ param5.k = "9";
+ param5.m = "9";
+ param5.c = "6";
+ param5.w = "16";
+
+ pthread_t tid1, tid2, tid3, tid4, tid5;
+ pthread_create(&tid1, NULL, thread1, (void*) &param1);
+ std::cout << "thread1 start " << std::endl;
+ pthread_create(&tid2, NULL, thread1, (void*) &param2);
+ std::cout << "thread2 start " << std::endl;
+ pthread_create(&tid3, NULL, thread1, (void*) &param3);
+ std::cout << "thread3 start " << std::endl;
+ pthread_create(&tid4, NULL, thread1, (void*) &param4);
+ std::cout << "thread4 start " << std::endl;
+ pthread_create(&tid5, NULL, thread1, (void*) &param5);
+ std::cout << "thread5 start " << std::endl;
+
+ pthread_join(tid1, NULL);
+ pthread_join(tid2, NULL);
+ pthread_join(tid3, NULL);
+ pthread_join(tid4, NULL);
+ pthread_join(tid5, NULL);
+}
+
+void* thread1(void* pParam)
+{
+ TestParam* param = static_cast<TestParam*>(pParam);
+
+ time_t start, end;
+
+ ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
+
+ instance.disable_dlclose = true;
+ {
+ std::lock_guard l{instance.lock};
+ __erasure_code_init((char*) "shec", (char*) "");
+ }
+ std::cout << "__erasure_code_init finish " << std::endl;
+
+ //encode
+ bufferlist in;
+ set<int> want_to_encode;
+ map<int, bufferlist> encoded;
+
+ in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" //length = 62
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//186
+ "012345"//192
+ );
+
+ //decode
+ int want_to_decode[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
+ map<int, bufferlist> decoded;
+ bufferlist out1, out2, usable;
+
+ time(&start);
+ time(&end);
+ const int kTestSec = 60;
+ ErasureCodeShecTableCache tcache;
+
+ while (kTestSec >= (end - start)) {
+ //init
+ int r;
+ ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
+ tcache,
+ ErasureCodeShec::MULTIPLE);
+ ErasureCodeProfile *profile = new ErasureCodeProfile();
+ (*profile)["plugin"] = "shec";
+ (*profile)["technique"] = "multiple";
+ (*profile)["crush-failure-domain"] = "osd";
+ (*profile)["k"] = param->k;
+ (*profile)["m"] = param->m;
+ (*profile)["c"] = param->c;
+ (*profile)["w"] = param->w;
+ r = shec->init(*profile, &cerr);
+
+ int i_k = std::atoi(param->k.c_str());
+ int i_m = std::atoi(param->m.c_str());
+ int i_c = std::atoi(param->c.c_str());
+ int i_w = std::atoi(param->w.c_str());
+
+ EXPECT_EQ(0, r);
+ EXPECT_EQ(i_k, shec->k);
+ EXPECT_EQ(i_m, shec->m);
+ EXPECT_EQ(i_c, shec->c);
+ EXPECT_EQ(i_w, shec->w);
+ EXPECT_EQ(ErasureCodeShec::MULTIPLE, shec->technique);
+ EXPECT_STREQ("default", shec->rule_root.c_str());
+ EXPECT_STREQ("osd", shec->rule_failure_domain.c_str());
+ EXPECT_TRUE(shec->matrix != NULL);
+ if ((shec->matrix == NULL)) {
+ std::cout << "matrix is null" << std::endl;
+ // error
+ break;
+ }
+
+ //encode
+ for (unsigned int i = 0; i < shec->get_chunk_count(); i++) {
+ want_to_encode.insert(i);
+ }
+ r = shec->encode(want_to_encode, in, &encoded);
+
+ EXPECT_EQ(0, r);
+ EXPECT_EQ(shec->get_chunk_count(), encoded.size());
+ EXPECT_EQ(shec->get_chunk_size(in.length()), encoded[0].length());
+
+ if (r != 0) {
+ std::cout << "error in encode" << std::endl;
+ //error
+ break;
+ }
+
+ //decode
+ r = shec->_decode(set<int>(want_to_decode, want_to_decode + 2),
+ encoded,
+ &decoded);
+
+ EXPECT_EQ(0, r);
+ EXPECT_EQ(2u, decoded.size());
+ EXPECT_EQ(shec->get_chunk_size(in.length()), decoded[0].length());
+
+ if (r != 0) {
+ std::cout << "error in decode" << std::endl;
+ //error
+ break;
+ }
+
+ //out1 is "encoded"
+ for (unsigned int i = 0; i < encoded.size(); i++) {
+ out1.append(encoded[i]);
+ }
+ //out2 is "decoded"
+ shec->decode_concat(encoded, &out2);
+ usable.substr_of(out2, 0, in.length());
+ EXPECT_FALSE(out1 == in);
+ EXPECT_TRUE(usable == in);
+ if (out1 == in || !(usable == in)) {
+ std::cout << "encode(decode) result is not correct" << std::endl;
+ break;
+ }
+
+ delete shec;
+ delete profile;
+ want_to_encode.clear();
+ encoded.clear();
+ decoded.clear();
+ out1.clear();
+ out2.clear();
+ usable.clear();
+
+ time(&end);
+ }
+
+ return NULL;
+}
diff --git a/src/test/erasure-code/ceph_erasure_code_benchmark.cc b/src/test/erasure-code/ceph_erasure_code_benchmark.cc
new file mode 100644
index 000000000..c86e58697
--- /dev/null
+++ b/src/test/erasure-code/ceph_erasure_code_benchmark.cc
@@ -0,0 +1,354 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph distributed storage system
+ *
+ * Copyright (C) 2013,2014 Cloudwatt <libre.licensing@cloudwatt.com>
+ * Copyright (C) 2014 Red Hat <contact@redhat.com>
+ *
+ * Author: Loic Dachary <loic@dachary.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#include <boost/scoped_ptr.hpp>
+#include <boost/lexical_cast.hpp>
+#include <boost/program_options/option.hpp>
+#include <boost/program_options/options_description.hpp>
+#include <boost/program_options/variables_map.hpp>
+#include <boost/program_options/cmdline.hpp>
+#include <boost/program_options/parsers.hpp>
+#include <boost/algorithm/string.hpp>
+
+#include "global/global_context.h"
+#include "global/global_init.h"
+#include "common/ceph_argparse.h"
+#include "common/ceph_context.h"
+#include "common/config.h"
+#include "common/Clock.h"
+#include "include/utime.h"
+#include "erasure-code/ErasureCodePlugin.h"
+#include "erasure-code/ErasureCode.h"
+#include "ceph_erasure_code_benchmark.h"
+
+using std::endl;
+using std::cerr;
+using std::cout;
+using std::map;
+using std::set;
+using std::string;
+using std::stringstream;
+using std::vector;
+
+namespace po = boost::program_options;
+
+int ErasureCodeBench::setup(int argc, char** argv) {
+
+ po::options_description desc("Allowed options");
+ desc.add_options()
+ ("help,h", "produce help message")
+ ("verbose,v", "explain what happens")
+ ("size,s", po::value<int>()->default_value(1024 * 1024),
+ "size of the buffer to be encoded")
+ ("iterations,i", po::value<int>()->default_value(1),
+ "number of encode/decode runs")
+ ("plugin,p", po::value<string>()->default_value("jerasure"),
+ "erasure code plugin name")
+ ("workload,w", po::value<string>()->default_value("encode"),
+ "run either encode or decode")
+ ("erasures,e", po::value<int>()->default_value(1),
+ "number of erasures when decoding")
+ ("erased", po::value<vector<int> >(),
+ "erased chunk (repeat if more than one chunk is erased)")
+ ("erasures-generation,E", po::value<string>()->default_value("random"),
+ "If set to 'random', pick the number of chunks to recover (as specified by "
+ " --erasures) at random. If set to 'exhaustive' try all combinations of erasures "
+ " (i.e. k=4,m=3 with one erasure will try to recover from the erasure of "
+ " the first chunk, then the second etc.)")
+ ("parameter,P", po::value<vector<string> >(),
+ "add a parameter to the erasure code profile")
+ ;
+
+ po::variables_map vm;
+ po::parsed_options parsed =
+ po::command_line_parser(argc, argv).options(desc).allow_unregistered().run();
+ po::store(
+ parsed,
+ vm);
+ po::notify(vm);
+
+ vector<const char *> ceph_options;
+ vector<string> ceph_option_strings = po::collect_unrecognized(
+ parsed.options, po::include_positional);
+ ceph_options.reserve(ceph_option_strings.size());
+ for (vector<string>::iterator i = ceph_option_strings.begin();
+ i != ceph_option_strings.end();
+ ++i) {
+ ceph_options.push_back(i->c_str());
+ }
+
+ cct = global_init(
+ NULL, ceph_options, CEPH_ENTITY_TYPE_CLIENT,
+ CODE_ENVIRONMENT_UTILITY,
+ CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+ common_init_finish(g_ceph_context);
+ g_ceph_context->_conf.apply_changes(nullptr);
+
+ if (vm.count("help")) {
+ cout << desc << std::endl;
+ return 1;
+ }
+
+ if (vm.count("parameter")) {
+ const vector<string> &p = vm["parameter"].as< vector<string> >();
+ for (vector<string>::const_iterator i = p.begin();
+ i != p.end();
+ ++i) {
+ std::vector<std::string> strs;
+ boost::split(strs, *i, boost::is_any_of("="));
+ if (strs.size() != 2) {
+ cerr << "--parameter " << *i << " ignored because it does not contain exactly one =" << endl;
+ } else {
+ profile[strs[0]] = strs[1];
+ }
+ }
+ }
+
+ in_size = vm["size"].as<int>();
+ max_iterations = vm["iterations"].as<int>();
+ plugin = vm["plugin"].as<string>();
+ workload = vm["workload"].as<string>();
+ erasures = vm["erasures"].as<int>();
+ if (vm.count("erasures-generation") > 0 &&
+ vm["erasures-generation"].as<string>() == "exhaustive")
+ exhaustive_erasures = true;
+ else
+ exhaustive_erasures = false;
+ if (vm.count("erased") > 0)
+ erased = vm["erased"].as<vector<int> >();
+
+ try {
+ k = stoi(profile["k"]);
+ m = stoi(profile["m"]);
+ } catch (const std::logic_error& e) {
+ cout << "Invalid k and/or m: k=" << profile["k"] << ", m=" << profile["m"]
+ << " (" << e.what() << ")" << endl;
+ return -EINVAL;
+ }
+ if (k <= 0) {
+ cout << "parameter k is " << k << ". But k needs to be > 0." << endl;
+ return -EINVAL;
+ } else if ( m < 0 ) {
+ cout << "parameter m is " << m << ". But m needs to be >= 0." << endl;
+ return -EINVAL;
+ }
+
+ verbose = vm.count("verbose") > 0 ? true : false;
+
+ return 0;
+}
+
+int ErasureCodeBench::run() {
+ ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
+ instance.disable_dlclose = true;
+
+ if (workload == "encode")
+ return encode();
+ else
+ return decode();
+}
+
+int ErasureCodeBench::encode()
+{
+ ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
+ ErasureCodeInterfaceRef erasure_code;
+ stringstream messages;
+ int code = instance.factory(plugin,
+ g_conf().get_val<std::string>("erasure_code_dir"),
+ profile, &erasure_code, &messages);
+ if (code) {
+ cerr << messages.str() << endl;
+ return code;
+ }
+
+ bufferlist in;
+ in.append(string(in_size, 'X'));
+ in.rebuild_aligned(ErasureCode::SIMD_ALIGN);
+ set<int> want_to_encode;
+ for (int i = 0; i < k + m; i++) {
+ want_to_encode.insert(i);
+ }
+ utime_t begin_time = ceph_clock_now();
+ for (int i = 0; i < max_iterations; i++) {
+ std::map<int,bufferlist> encoded;
+ code = erasure_code->encode(want_to_encode, in, &encoded);
+ if (code)
+ return code;
+ }
+ utime_t end_time = ceph_clock_now();
+ cout << (end_time - begin_time) << "\t" << (max_iterations * (in_size / 1024)) << endl;
+ return 0;
+}
+
+static void display_chunks(const map<int,bufferlist> &chunks,
+ unsigned int chunk_count) {
+ cout << "chunks ";
+ for (unsigned int chunk = 0; chunk < chunk_count; chunk++) {
+ if (chunks.count(chunk) == 0) {
+ cout << "(" << chunk << ")";
+ } else {
+ cout << " " << chunk << " ";
+ }
+ cout << " ";
+ }
+ cout << "(X) is an erased chunk" << endl;
+}
+
+int ErasureCodeBench::decode_erasures(const map<int,bufferlist> &all_chunks,
+ const map<int,bufferlist> &chunks,
+ unsigned i,
+ unsigned want_erasures,
+ ErasureCodeInterfaceRef erasure_code)
+{
+ int code = 0;
+
+ if (want_erasures == 0) {
+ if (verbose)
+ display_chunks(chunks, erasure_code->get_chunk_count());
+ set<int> want_to_read;
+ for (unsigned int chunk = 0; chunk < erasure_code->get_chunk_count(); chunk++)
+ if (chunks.count(chunk) == 0)
+ want_to_read.insert(chunk);
+
+ map<int,bufferlist> decoded;
+ code = erasure_code->decode(want_to_read, chunks, &decoded, 0);
+ if (code)
+ return code;
+ for (set<int>::iterator chunk = want_to_read.begin();
+ chunk != want_to_read.end();
+ ++chunk) {
+ if (all_chunks.find(*chunk)->second.length() != decoded[*chunk].length()) {
+ cerr << "chunk " << *chunk << " length=" << all_chunks.find(*chunk)->second.length()
+ << " decoded with length=" << decoded[*chunk].length() << endl;
+ return -1;
+ }
+ bufferlist tmp = all_chunks.find(*chunk)->second;
+ if (!tmp.contents_equal(decoded[*chunk])) {
+ cerr << "chunk " << *chunk
+ << " content and recovered content are different" << endl;
+ return -1;
+ }
+ }
+ return 0;
+ }
+
+ for (; i < erasure_code->get_chunk_count(); i++) {
+ map<int,bufferlist> one_less = chunks;
+ one_less.erase(i);
+ code = decode_erasures(all_chunks, one_less, i + 1, want_erasures - 1, erasure_code);
+ if (code)
+ return code;
+ }
+
+ return 0;
+}
+
+int ErasureCodeBench::decode()
+{
+ ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
+ ErasureCodeInterfaceRef erasure_code;
+ stringstream messages;
+ int code = instance.factory(plugin,
+ g_conf().get_val<std::string>("erasure_code_dir"),
+ profile, &erasure_code, &messages);
+ if (code) {
+ cerr << messages.str() << endl;
+ return code;
+ }
+
+ bufferlist in;
+ in.append(string(in_size, 'X'));
+ in.rebuild_aligned(ErasureCode::SIMD_ALIGN);
+
+ set<int> want_to_encode;
+ for (int i = 0; i < k + m; i++) {
+ want_to_encode.insert(i);
+ }
+
+ map<int,bufferlist> encoded;
+ code = erasure_code->encode(want_to_encode, in, &encoded);
+ if (code)
+ return code;
+
+ set<int> want_to_read = want_to_encode;
+
+ if (erased.size() > 0) {
+ for (vector<int>::const_iterator i = erased.begin();
+ i != erased.end();
+ ++i)
+ encoded.erase(*i);
+ display_chunks(encoded, erasure_code->get_chunk_count());
+ }
+
+ utime_t begin_time = ceph_clock_now();
+ for (int i = 0; i < max_iterations; i++) {
+ if (exhaustive_erasures) {
+ code = decode_erasures(encoded, encoded, 0, erasures, erasure_code);
+ if (code)
+ return code;
+ } else if (erased.size() > 0) {
+ map<int,bufferlist> decoded;
+ code = erasure_code->decode(want_to_read, encoded, &decoded, 0);
+ if (code)
+ return code;
+ } else {
+ map<int,bufferlist> chunks = encoded;
+ for (int j = 0; j < erasures; j++) {
+ int erasure;
+ do {
+ erasure = rand() % ( k + m );
+ } while(chunks.count(erasure) == 0);
+ chunks.erase(erasure);
+ }
+ map<int,bufferlist> decoded;
+ code = erasure_code->decode(want_to_read, chunks, &decoded, 0);
+ if (code)
+ return code;
+ }
+ }
+ utime_t end_time = ceph_clock_now();
+ cout << (end_time - begin_time) << "\t" << (max_iterations * (in_size / 1024)) << endl;
+ return 0;
+}
+
+int main(int argc, char** argv) {
+ ErasureCodeBench ecbench;
+ try {
+ int err = ecbench.setup(argc, argv);
+ if (err)
+ return err;
+ return ecbench.run();
+ } catch(po::error &e) {
+ cerr << e.what() << endl;
+ return 1;
+ }
+}
+
+/*
+ * Local Variables:
+ * compile-command: "cd ../../../build ; make -j4 ceph_erasure_code_benchmark &&
+ * valgrind --tool=memcheck --leak-check=full \
+ * ./bin/ceph_erasure_code_benchmark \
+ * --plugin jerasure \
+ * --parameter directory=lib \
+ * --parameter technique=reed_sol_van \
+ * --parameter k=2 \
+ * --parameter m=2 \
+ * --iterations 1
+ * "
+ * End:
+ */
diff --git a/src/test/erasure-code/ceph_erasure_code_benchmark.h b/src/test/erasure-code/ceph_erasure_code_benchmark.h
new file mode 100644
index 000000000..59149a74c
--- /dev/null
+++ b/src/test/erasure-code/ceph_erasure_code_benchmark.h
@@ -0,0 +1,62 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph distributed storage system
+ *
+ * Copyright (C) 2013,2014 Cloudwatt <libre.licensing@cloudwatt.com>
+ * Copyright (C) 2014 Red Hat <contact@redhat.com>
+ *
+ * Author: Loic Dachary <loic@dachary.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#ifndef CEPH_ERASURE_CODE_BENCHMARK_H
+#define CEPH_ERASURE_CODE_BENCHMARK_H
+
+#include <string>
+#include <map>
+#include <vector>
+
+#include <boost/intrusive_ptr.hpp>
+
+#include "include/buffer.h"
+
+#include "common/ceph_context.h"
+
+#include "erasure-code/ErasureCodeInterface.h"
+
+class ErasureCodeBench {
+ int in_size;
+ int max_iterations;
+ int erasures;
+ int k;
+ int m;
+
+ std::string plugin;
+
+ bool exhaustive_erasures;
+ std::vector<int> erased;
+ std::string workload;
+
+ ceph::ErasureCodeProfile profile;
+
+ bool verbose;
+ boost::intrusive_ptr<CephContext> cct;
+public:
+ int setup(int argc, char** argv);
+ int run();
+ int decode_erasures(const std::map<int, ceph::buffer::list> &all_chunks,
+ const std::map<int, ceph::buffer::list> &chunks,
+ unsigned i,
+ unsigned want_erasures,
+ ErasureCodeInterfaceRef erasure_code);
+ int decode();
+ int encode();
+};
+
+#endif
diff --git a/src/test/erasure-code/ceph_erasure_code_non_regression.cc b/src/test/erasure-code/ceph_erasure_code_non_regression.cc
new file mode 100644
index 000000000..3ce31b243
--- /dev/null
+++ b/src/test/erasure-code/ceph_erasure_code_non_regression.cc
@@ -0,0 +1,327 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph distributed storage system
+ *
+ * Red Hat (C) 2014, 2015 Red Hat <contact@redhat.com>
+ *
+ * Author: Loic Dachary <loic@dachary.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#include <errno.h>
+#include <stdlib.h>
+#include <boost/scoped_ptr.hpp>
+#include <boost/lexical_cast.hpp>
+#include <boost/program_options/option.hpp>
+#include <boost/program_options/options_description.hpp>
+#include <boost/program_options/variables_map.hpp>
+#include <boost/program_options/cmdline.hpp>
+#include <boost/program_options/parsers.hpp>
+#include <boost/algorithm/string.hpp>
+
+#include "global/global_context.h"
+#include "global/global_init.h"
+#include "common/errno.h"
+#include "common/ceph_context.h"
+#include "common/ceph_argparse.h"
+#include "common/config.h"
+#include "erasure-code/ErasureCodePlugin.h"
+
+namespace po = boost::program_options;
+using namespace std;
+
+class ErasureCodeNonRegression {
+ unsigned stripe_width;
+ string plugin;
+ bool create;
+ bool check;
+ string base;
+ string directory;
+ ErasureCodeProfile profile;
+ boost::intrusive_ptr<CephContext> cct;
+public:
+ int setup(int argc, char** argv);
+ int run();
+ int run_create();
+ int run_check();
+ int decode_erasures(ErasureCodeInterfaceRef erasure_code,
+ set<int> erasures,
+ map<int,bufferlist> chunks);
+ string content_path();
+ string chunk_path(unsigned int chunk);
+};
+
+int ErasureCodeNonRegression::setup(int argc, char** argv) {
+
+ po::options_description desc("Allowed options");
+ desc.add_options()
+ ("help,h", "produce help message")
+ ("stripe-width,s", po::value<int>()->default_value(4 * 1024),
+ "stripe_width, i.e. the size of the buffer to be encoded")
+ ("plugin,p", po::value<string>()->default_value("jerasure"),
+ "erasure code plugin name")
+ ("base", po::value<string>()->default_value("."),
+ "prefix all paths with base")
+ ("parameter,P", po::value<vector<string> >(),
+ "add a parameter to the erasure code profile")
+ ("create", "create the erasure coded content in the directory")
+ ("check", "check the content in the directory matches the chunks and vice versa")
+ ;
+
+ po::variables_map vm;
+ po::parsed_options parsed =
+ po::command_line_parser(argc, argv).options(desc).allow_unregistered().run();
+ po::store(
+ parsed,
+ vm);
+ po::notify(vm);
+
+ vector<const char *> ceph_options;
+ vector<string> ceph_option_strings = po::collect_unrecognized(
+ parsed.options, po::include_positional);
+ ceph_options.reserve(ceph_option_strings.size());
+ for (vector<string>::iterator i = ceph_option_strings.begin();
+ i != ceph_option_strings.end();
+ ++i) {
+ ceph_options.push_back(i->c_str());
+ }
+
+ cct = global_init(NULL, ceph_options, CEPH_ENTITY_TYPE_CLIENT,
+ CODE_ENVIRONMENT_UTILITY,
+ CINIT_FLAG_NO_MON_CONFIG);
+ common_init_finish(g_ceph_context);
+ g_ceph_context->_conf.apply_changes(nullptr);
+
+ if (vm.count("help")) {
+ cout << desc << std::endl;
+ return 1;
+ }
+
+ stripe_width = vm["stripe-width"].as<int>();
+ plugin = vm["plugin"].as<string>();
+ base = vm["base"].as<string>();
+ check = vm.count("check") > 0;
+ create = vm.count("create") > 0;
+
+ if (!check && !create) {
+ cerr << "must specifify either --check, or --create" << endl;
+ return 1;
+ }
+
+ {
+ stringstream path;
+ path << base << "/" << "plugin=" << plugin << " stripe-width=" << stripe_width;
+ directory = path.str();
+ }
+
+ if (vm.count("parameter")) {
+ const vector<string> &p = vm["parameter"].as< vector<string> >();
+ for (vector<string>::const_iterator i = p.begin();
+ i != p.end();
+ ++i) {
+ std::vector<std::string> strs;
+ boost::split(strs, *i, boost::is_any_of("="));
+ if (strs.size() != 2) {
+ cerr << "--parameter " << *i << " ignored because it does not contain exactly one =" << endl;
+ } else {
+ profile[strs[0]] = strs[1];
+ }
+ directory += " " + *i;
+ }
+ }
+
+ return 0;
+}
+
+int ErasureCodeNonRegression::run()
+ {
+ int ret = 0;
+ if(create && (ret = run_create()))
+ return ret;
+ if(check && (ret = run_check()))
+ return ret;
+ return ret;
+}
+
+int ErasureCodeNonRegression::run_create()
+{
+ ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
+ ErasureCodeInterfaceRef erasure_code;
+ stringstream messages;
+ int code = instance.factory(plugin,
+ g_conf().get_val<std::string>("erasure_code_dir"),
+ profile, &erasure_code, &messages);
+ if (code) {
+ cerr << messages.str() << endl;
+ return code;
+ }
+
+ if (::mkdir(directory.c_str(), 0755)) {
+ cerr << "mkdir(" << directory << "): " << cpp_strerror(errno) << endl;
+ return 1;
+ }
+ unsigned payload_chunk_size = 37;
+ string payload;
+ for (unsigned j = 0; j < payload_chunk_size; ++j)
+ payload.push_back('a' + (rand() % 26));
+ bufferlist in;
+ for (unsigned j = 0; j < stripe_width; j += payload_chunk_size)
+ in.append(payload);
+ if (stripe_width < in.length())
+ in.splice(stripe_width, in.length() - stripe_width);
+ if (in.write_file(content_path().c_str()))
+ return 1;
+ set<int> want_to_encode;
+ for (unsigned int i = 0; i < erasure_code->get_chunk_count(); i++) {
+ want_to_encode.insert(i);
+ }
+ map<int,bufferlist> encoded;
+ code = erasure_code->encode(want_to_encode, in, &encoded);
+ if (code)
+ return code;
+ for (map<int,bufferlist>::iterator chunk = encoded.begin();
+ chunk != encoded.end();
+ ++chunk) {
+ if (chunk->second.write_file(chunk_path(chunk->first).c_str()))
+ return 1;
+ }
+ return 0;
+}
+
+int ErasureCodeNonRegression::decode_erasures(ErasureCodeInterfaceRef erasure_code,
+ set<int> erasures,
+ map<int,bufferlist> chunks)
+{
+ map<int,bufferlist> available;
+ for (map<int,bufferlist>::iterator chunk = chunks.begin();
+ chunk != chunks.end();
+ ++chunk) {
+ if (erasures.count(chunk->first) == 0)
+ available[chunk->first] = chunk->second;
+
+ }
+ map<int,bufferlist> decoded;
+ int code = erasure_code->decode(erasures, available, &decoded, available.begin()->second.length());
+ if (code)
+ return code;
+ for (set<int>::iterator erasure = erasures.begin();
+ erasure != erasures.end();
+ ++erasure) {
+ if (!chunks[*erasure].contents_equal(decoded[*erasure])) {
+ cerr << "chunk " << *erasure << " incorrectly recovered" << endl;
+ return 1;
+ }
+ }
+ return 0;
+}
+
+int ErasureCodeNonRegression::run_check()
+{
+ ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
+ ErasureCodeInterfaceRef erasure_code;
+ stringstream messages;
+ int code = instance.factory(plugin,
+ g_conf().get_val<std::string>("erasure_code_dir"),
+ profile, &erasure_code, &messages);
+ if (code) {
+ cerr << messages.str() << endl;
+ return code;
+ }
+ string errors;
+ bufferlist in;
+ if (in.read_file(content_path().c_str(), &errors)) {
+ cerr << errors << endl;
+ return 1;
+ }
+ set<int> want_to_encode;
+ for (unsigned int i = 0; i < erasure_code->get_chunk_count(); i++) {
+ want_to_encode.insert(i);
+ }
+
+ map<int,bufferlist> encoded;
+ code = erasure_code->encode(want_to_encode, in, &encoded);
+ if (code)
+ return code;
+
+ for (map<int,bufferlist>::iterator chunk = encoded.begin();
+ chunk != encoded.end();
+ ++chunk) {
+ bufferlist existing;
+ if (existing.read_file(chunk_path(chunk->first).c_str(), &errors)) {
+ cerr << errors << endl;
+ return 1;
+ }
+ bufferlist &old = chunk->second;
+ if (existing.length() != old.length() ||
+ memcmp(existing.c_str(), old.c_str(), old.length())) {
+ cerr << "chunk " << chunk->first << " encodes differently" << endl;
+ return 1;
+ }
+ }
+
+ // erasing a single chunk is likely to use a specific code path in every plugin
+ set<int> erasures;
+ erasures.clear();
+ erasures.insert(0);
+ code = decode_erasures(erasure_code, erasures, encoded);
+ if (code)
+ return code;
+
+ if (erasure_code->get_chunk_count() - erasure_code->get_data_chunk_count() > 1) {
+ // erasing two chunks is likely to be the general case
+ erasures.clear();
+ erasures.insert(0);
+ erasures.insert(erasure_code->get_chunk_count() - 1);
+ code = decode_erasures(erasure_code, erasures, encoded);
+ if (code)
+ return code;
+ }
+
+ return 0;
+}
+
+string ErasureCodeNonRegression::content_path()
+{
+ stringstream path;
+ path << directory << "/content";
+ return path.str();
+}
+
+string ErasureCodeNonRegression::chunk_path(unsigned int chunk)
+{
+ stringstream path;
+ path << directory << "/" << chunk;
+ return path.str();
+}
+
+int main(int argc, char** argv) {
+ ErasureCodeNonRegression non_regression;
+ int err = non_regression.setup(argc, argv);
+ if (err)
+ return err;
+ return non_regression.run();
+}
+
+/*
+ * Local Variables:
+ * compile-command: "cd ../.. ; make -j4 &&
+ * make ceph_erasure_code_non_regression &&
+ * libtool --mode=execute valgrind --tool=memcheck --leak-check=full \
+ * ./ceph_erasure_code_non_regression \
+ * --plugin jerasure \
+ * --parameter technique=reed_sol_van \
+ * --parameter k=2 \
+ * --parameter m=2 \
+ * --directory /tmp/ceph_erasure_code_non_regression \
+ * --stripe-width 3181 \
+ * --create \
+ * --check
+ * "
+ * End:
+ */