diff options
Diffstat (limited to 'src/rgw/rgw_kmip_client_impl.cc')
-rw-r--r-- | src/rgw/rgw_kmip_client_impl.cc | 728 |
1 files changed, 728 insertions, 0 deletions
diff --git a/src/rgw/rgw_kmip_client_impl.cc b/src/rgw/rgw_kmip_client_impl.cc new file mode 100644 index 000000000..0824273e6 --- /dev/null +++ b/src/rgw/rgw_kmip_client_impl.cc @@ -0,0 +1,728 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#include <boost/intrusive/list.hpp> +#include <atomic> +#include <mutex> +#include <string.h> + +#include "include/compat.h" +#include "common/errno.h" +#include "rgw_common.h" +#include "rgw_kmip_client.h" +#include "rgw_kmip_client_impl.h" + +#include <openssl/err.h> +#include <openssl/ssl.h> +extern "C" { +#include "kmip.h" +#include "kmip_bio.h" +#include "kmip_memset.h" +}; + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rgw + +static enum kmip_version protocol_version = KMIP_1_0; + +struct RGWKmipHandle { + int uses; + mono_time lastuse; + SSL_CTX *ctx; + SSL *ssl; + BIO *bio; + KMIP kmip_ctx[1]; + TextString textstrings[2]; + UsernamePasswordCredential upc[1]; + Credential credential[1]; + int need_to_free_kmip; + size_t buffer_blocks, buffer_block_size, buffer_total_size; + uint8 *encoding; + + explicit RGWKmipHandle() : + uses(0), ctx(0), ssl(0), bio(0), + need_to_free_kmip(0), + encoding(0) { + memset(kmip_ctx, 0, sizeof kmip_ctx); + memset(textstrings, 0, sizeof textstrings); + memset(upc, 0, sizeof upc); + memset(credential, 0, sizeof credential); + }; +}; + +struct RGWKmipWorker: public Thread { + RGWKMIPManagerImpl &m; + RGWKmipWorker(RGWKMIPManagerImpl& m) : m(m) {} + void *entry() override; + void signal() { + std::lock_guard l{m.lock}; + m.cond.notify_all(); + } +}; + +static void +kmip_free_handle_stuff(RGWKmipHandle *kmip) +{ + if (kmip->encoding) { + kmip_free_buffer(kmip->kmip_ctx, + kmip->encoding, + kmip->buffer_total_size); + kmip_set_buffer(kmip->kmip_ctx, NULL, 0); + } + if (kmip->need_to_free_kmip) + kmip_destroy(kmip->kmip_ctx); + if (kmip->bio) + BIO_free_all(kmip->bio); + if (kmip->ctx) + SSL_CTX_free(kmip->ctx); +} + +class RGWKmipHandleBuilder { +private: + CephContext *cct; + const char *clientcert = 0; + const char *clientkey = 0; + const char *capath = 0; + const char *host = 0; + const char *portstring = 0; + const char *username = 0; + const char *password = 0; +public: + RGWKmipHandleBuilder(CephContext *cct) : cct(cct) {}; + RGWKmipHandleBuilder& set_clientcert(const std::string &v) { + const char *s = v.c_str(); + if (*s) { + clientcert = s; + } + return *this; + } + RGWKmipHandleBuilder& set_clientkey(const std::string &v) { + const char *s = v.c_str(); + if (*s) { + clientkey = s; + } + return *this; + } + RGWKmipHandleBuilder& set_capath(const std::string &v) { + const char *s = v.c_str(); + if (*s) { + capath = s; + } + return *this; + } + RGWKmipHandleBuilder& set_host(const char *v) { + host = v; + return *this; + } + RGWKmipHandleBuilder& set_portstring(const char *v) { + portstring = v; + return *this; + } + RGWKmipHandleBuilder& set_username(const std::string &v) { + const char *s = v.c_str(); + if (*s) { + username = s; + } + return *this; + } + RGWKmipHandleBuilder& set_password(const std::string& v) { + const char *s = v.c_str(); + if (*s) { + password = s; + } + return *this; + } + RGWKmipHandle *build() const; +}; + +static int +kmip_write_an_error_helper(const char *s, size_t l, void *u) { + CephContext *cct = (CephContext *)u; + std::string_view es(s, l); + lderr(cct) << es << dendl; + return l; +} + +void +ERR_print_errors_ceph(CephContext *cct) +{ + ERR_print_errors_cb(kmip_write_an_error_helper, cct); +} + +RGWKmipHandle * +RGWKmipHandleBuilder::build() const +{ + int failed = 1; + RGWKmipHandle *r = new RGWKmipHandle(); + TextString *up = 0; + size_t ns; + + r->ctx = SSL_CTX_new(TLS_client_method()); + + if (!clientcert) + ; + else if (SSL_CTX_use_certificate_file(r->ctx, clientcert, SSL_FILETYPE_PEM) != 1) { + lderr(cct) << "ERROR: can't load client cert from " + << clientcert << dendl; + ERR_print_errors_ceph(cct); + goto Done; + } + + if (!clientkey) + ; + else if (SSL_CTX_use_PrivateKey_file(r->ctx, clientkey, + SSL_FILETYPE_PEM) != 1) { + lderr(cct) << "ERROR: can't load client key from " + << clientkey << dendl; + ERR_print_errors_ceph(cct); + goto Done; + } + + if (!capath) + ; + else if (SSL_CTX_load_verify_locations(r->ctx, capath, NULL) != 1) { + lderr(cct) << "ERROR: can't load cacert from " + << capath << dendl; + ERR_print_errors_ceph(cct); + goto Done; + } + r->bio = BIO_new_ssl_connect(r->ctx); + if (!r->bio) { + lderr(cct) << "BIO_new_ssl_connect failed" << dendl; + goto Done; + } + BIO_get_ssl(r->bio, &r->ssl); + SSL_set_mode(r->ssl, SSL_MODE_AUTO_RETRY); + + BIO_set_conn_hostname(r->bio, host); + BIO_set_conn_port(r->bio, portstring); + if (BIO_do_connect(r->bio) != 1) { + lderr(cct) << "BIO_do_connect failed to " << host + << ":" << portstring << dendl; + ERR_print_errors_ceph(cct); + goto Done; + } + + // setup kmip + + kmip_init(r->kmip_ctx, NULL, 0, protocol_version); + r->need_to_free_kmip = 1; + r->buffer_blocks = 1; + r->buffer_block_size = 1024; + r->encoding = static_cast<uint8*>(r->kmip_ctx->calloc_func( + r->kmip_ctx->state, r->buffer_blocks, r->buffer_block_size)); + if (!r->encoding) { + lderr(cct) << "kmip buffer alloc failed: " + << r->buffer_blocks << + " * " << r->buffer_block_size << dendl; + goto Done; + } + ns = r->buffer_blocks * r->buffer_block_size; + kmip_set_buffer(r->kmip_ctx, r->encoding, ns); + r->buffer_total_size = ns; + + up = r->textstrings; + if (username) { + memset(r->upc, 0, sizeof *r->upc); + up->value = (char *) username; + up->size = strlen(username); + r->upc->username = up++; + if (password) { + up->value = (char *) password; + up->size = strlen(password); + r->upc->password = up++; + } + r->credential->credential_type = KMIP_CRED_USERNAME_AND_PASSWORD; + r->credential->credential_value = r->upc; + int i = kmip_add_credential(r->kmip_ctx, r->credential); + if (i != KMIP_OK) { + fprintf(stderr,"failed to add credential to kmip\n"); + goto Done; + } + } + + failed = 0; +Done: + if (!failed) + ; + else if (!r) + ; + else { + kmip_free_handle_stuff(r); + delete r; + r = 0; + } + return r; +} + +struct RGWKmipHandles : public Thread { + CephContext *cct; + ceph::mutex cleaner_lock = ceph::make_mutex("RGWKmipHandles::cleaner_lock"); + std::vector<RGWKmipHandle*> saved_kmip; + int cleaner_shutdown; + bool cleaner_active = false; + ceph::condition_variable cleaner_cond; + RGWKmipHandles(CephContext *cct) : + cct(cct), cleaner_shutdown{0} { + } + RGWKmipHandle* get_kmip_handle(); + void release_kmip_handle_now(RGWKmipHandle* kmip); + void release_kmip_handle(RGWKmipHandle* kmip); + void flush_kmip_handles(); + int do_one_entry(RGWKMIPTransceiver &element); + void* entry(); + void start(); + void stop(); +}; + +RGWKmipHandle* +RGWKmipHandles::get_kmip_handle() +{ + RGWKmipHandle* kmip = 0; + const char *hostaddr = cct->_conf->rgw_crypt_kmip_addr.c_str(); + { + std::lock_guard lock{cleaner_lock}; + if (!saved_kmip.empty()) { + kmip = *saved_kmip.begin(); + saved_kmip.erase(saved_kmip.begin()); + } + } + if (!kmip && hostaddr) { + char *hosttemp = strdup(hostaddr); + char *port = strchr(hosttemp, ':'); + if (port) + *port++ = 0; + kmip = RGWKmipHandleBuilder{cct} + .set_clientcert(cct->_conf->rgw_crypt_kmip_client_cert) + .set_clientkey(cct->_conf->rgw_crypt_kmip_client_key) + .set_capath(cct->_conf->rgw_crypt_kmip_ca_path) + .set_host(hosttemp) + .set_portstring(port ? port : "5696") + .set_username(cct->_conf->rgw_crypt_kmip_username) + .set_password(cct->_conf->rgw_crypt_kmip_password) + .build(); + free(hosttemp); + } + return kmip; +} + +void +RGWKmipHandles::release_kmip_handle_now(RGWKmipHandle* kmip) +{ + kmip_free_handle_stuff(kmip); + delete kmip; +} + +#define MAXIDLE 5 +void +RGWKmipHandles::release_kmip_handle(RGWKmipHandle* kmip) +{ + if (cleaner_shutdown) { + release_kmip_handle_now(kmip); + } else { + std::lock_guard lock{cleaner_lock}; + kmip->lastuse = mono_clock::now(); + saved_kmip.insert(saved_kmip.begin(), 1, kmip); + } +} + +void* +RGWKmipHandles::entry() +{ + RGWKmipHandle* kmip; + std::unique_lock lock{cleaner_lock}; + + for (;;) { + if (cleaner_shutdown) { + if (saved_kmip.empty()) + break; + } else { + cleaner_cond.wait_for(lock, std::chrono::seconds(MAXIDLE)); + } + mono_time now = mono_clock::now(); + while (!saved_kmip.empty()) { + auto cend = saved_kmip.end(); + --cend; + kmip = *cend; + if (!cleaner_shutdown && now - kmip->lastuse + < std::chrono::seconds(MAXIDLE)) + break; + saved_kmip.erase(cend); + release_kmip_handle_now(kmip); + } + } + return nullptr; +} + +void +RGWKmipHandles::start() +{ + std::lock_guard lock{cleaner_lock}; + if (!cleaner_active) { + cleaner_active = true; + this->create("KMIPcleaner"); // len<16!!! + } +} + +void +RGWKmipHandles::stop() +{ + std::unique_lock lock{cleaner_lock}; + cleaner_shutdown = 1; + cleaner_cond.notify_all(); + if (cleaner_active) { + lock.unlock(); + this->join(); + cleaner_active = false; + } +} + +void +RGWKmipHandles::flush_kmip_handles() +{ + stop(); + join(); + if (!saved_kmip.empty()) { + ldout(cct, 0) << "ERROR: " << __func__ << " failed final cleanup" << dendl; + } + saved_kmip.shrink_to_fit(); +} + +int +RGWKMIPManagerImpl::start() +{ + if (worker) { + lderr(cct) << "kmip worker already started" << dendl; + return -1; + } + worker = new RGWKmipWorker(*this); + worker->create("kmip worker"); + return 0; +} + +void +RGWKMIPManagerImpl::stop() +{ + going_down = true; + if (worker) { + worker->signal(); + worker->join(); + delete worker; + worker = 0; + } +} + +int +RGWKMIPManagerImpl::add_request(RGWKMIPTransceiver *req) +{ + std::unique_lock l{lock}; + if (going_down) + return -ECANCELED; + requests.push_back(*new Request{*req}); + l.unlock(); + if (worker) + worker->signal(); + return 0; +} + +int +RGWKmipHandles::do_one_entry(RGWKMIPTransceiver &element) +{ + auto h = get_kmip_handle(); + std::unique_lock l{element.lock}; + Attribute a[8], *ap; + TextString nvalue[1], uvalue[1]; + Name nattr[1]; + enum cryptographic_algorithm alg = KMIP_CRYPTOALG_AES; + int32 length = 256; + int32 mask = KMIP_CRYPTOMASK_ENCRYPT | KMIP_CRYPTOMASK_DECRYPT; + size_t ns; + ProtocolVersion pv[1]; + RequestHeader rh[1]; + RequestMessage rm[1]; + Authentication auth[1]; + ResponseMessage resp_m[1]; + int i; + union { + CreateRequestPayload create_req[1]; + LocateRequestPayload locate_req[1]; + GetRequestPayload get_req[1]; + GetAttributeListRequestPayload lsattrs_req[1]; + GetAttributesRequestPayload getattrs_req[1]; + } u[1]; + RequestBatchItem rbi[1]; + TemplateAttribute ta[1]; + const char *what = "?"; + int need_to_free_response = 0; + char *response = NULL; + int response_size = 0; + enum result_status rs; + ResponseBatchItem *req; + + if (!h) { + element.ret = -ERR_SERVICE_UNAVAILABLE; + return element.ret; + } + memset(a, 0, sizeof *a); + for (i = 0; i < (int)(sizeof a/sizeof *a); ++i) + kmip_init_attribute(a+i); + ap = a; + switch(element.operation) { + case RGWKMIPTransceiver::CREATE: + ap->type = KMIP_ATTR_CRYPTOGRAPHIC_ALGORITHM; + ap->value = &alg; + ++ap; + ap->type = KMIP_ATTR_CRYPTOGRAPHIC_LENGTH; + ap->value = &length; + ++ap; + ap->type = KMIP_ATTR_CRYPTOGRAPHIC_USAGE_MASK; + ap->value = &mask; + ++ap; + break; + default: + break; + } + if (element.name) { + memset(nvalue, 0, sizeof *nvalue); + nvalue->value = element.name; + nvalue->size = strlen(element.name); + memset(nattr, 0, sizeof *nattr); + nattr->value = nvalue; + nattr->type = KMIP_NAME_UNINTERPRETED_TEXT_STRING; + ap->type = KMIP_ATTR_NAME; + ap->value = nattr; + ++ap; + } + if (element.unique_id) { + memset(uvalue, 0, sizeof *uvalue); + uvalue->value = element.unique_id; + uvalue->size = strlen(element.unique_id); + } + memset(pv, 0, sizeof *pv); + memset(rh, 0, sizeof *rh); + memset(rm, 0, sizeof *rm); + memset(auth, 0, sizeof *auth); + memset(resp_m, 0, sizeof *resp_m); + kmip_init_protocol_version(pv, h->kmip_ctx->version); + kmip_init_request_header(rh); + rh->protocol_version = pv; + rh->maximum_response_size = h->kmip_ctx->max_message_size; + rh->time_stamp = time(NULL); + rh->batch_count = 1; + memset(rbi, 0, sizeof *rbi); + kmip_init_request_batch_item(rbi); + memset(u, 0, sizeof *u); + rbi->request_payload = u; + switch(element.operation) { + case RGWKMIPTransceiver::CREATE: + memset(ta, 0, sizeof *ta); + ta->attributes = a; + ta->attribute_count = ap-a; + u->create_req->object_type = KMIP_OBJTYPE_SYMMETRIC_KEY; + u->create_req->template_attribute = ta; + rbi->operation = KMIP_OP_CREATE; + what = "create"; + break; + case RGWKMIPTransceiver::GET: + if (element.unique_id) + u->get_req->unique_identifier = uvalue; + rbi->operation = KMIP_OP_GET; + what = "get"; + break; + case RGWKMIPTransceiver::LOCATE: + if (ap > a) { + u->locate_req->attributes = a; + u->locate_req->attribute_count = ap - a; + } + rbi->operation = KMIP_OP_LOCATE; + what = "locate"; + break; + case RGWKMIPTransceiver::GET_ATTRIBUTES: + case RGWKMIPTransceiver::GET_ATTRIBUTE_LIST: + case RGWKMIPTransceiver::DESTROY: + default: + lderr(cct) << "Missing operation logic op=" << element.operation << dendl; + element.ret = -EINVAL; + goto Done; + } + rm->request_header = rh; + rm->batch_items = rbi; + rm->batch_count = 1; + if (h->kmip_ctx->credential_list) { + LinkedListItem *item = h->kmip_ctx->credential_list->head; + if (item) { + auth->credential = (Credential *)item->data; + rh->authentication = auth; + } + } + for (;;) { + i = kmip_encode_request_message(h->kmip_ctx, rm); + if (i != KMIP_ERROR_BUFFER_FULL) break; + h->kmip_ctx->free_func(h->kmip_ctx->state, h->encoding); + h->encoding = 0; + ++h->buffer_blocks; + h->encoding = static_cast<uint8*>(h->kmip_ctx->calloc_func(h->kmip_ctx->state, h->buffer_blocks, h->buffer_block_size)); + if (!h->encoding) { + lderr(cct) << "kmip buffer alloc failed: " + << h->buffer_blocks + << " * " << h->buffer_block_size << dendl; + element.ret = -ENOMEM; + goto Done; + } + ns = h->buffer_blocks * h->buffer_block_size; + kmip_set_buffer(h->kmip_ctx, h->encoding, ns); + h->buffer_total_size = ns; + } + if (i != KMIP_OK) { + lderr(cct) << " Failed to encode " << what + << " request; err=" << i + << " ctx error message " << h->kmip_ctx->error_message + << dendl; + element.ret = -EINVAL; + goto Done; + } + i = kmip_bio_send_request_encoding(h->kmip_ctx, h->bio, + (char*)h->encoding, + h->kmip_ctx->index - h->kmip_ctx->buffer, + &response, &response_size); + if (i < 0) { + lderr(cct) << "Problem sending request to " << what << " " << i << " context error message " << h->kmip_ctx->error_message << dendl; + element.ret = -EINVAL; + goto Done; + } + kmip_free_buffer(h->kmip_ctx, h->encoding, + h->buffer_total_size); + h->encoding = 0; + kmip_set_buffer(h->kmip_ctx, response, response_size); + need_to_free_response = 1; + i = kmip_decode_response_message(h->kmip_ctx, resp_m); + if (i != KMIP_OK) { + lderr(cct) << "Failed to decode " << what << " " << i << " context error message " << h->kmip_ctx->error_message << dendl; + element.ret = -EINVAL; + goto Done; + } + if (resp_m->batch_count != 1) { + lderr(cct) << "Failed; weird response count doing " << what << " " << resp_m->batch_count << dendl; + element.ret = -EINVAL; + goto Done; + } + req = resp_m->batch_items; + rs = req->result_status; + if (rs != KMIP_STATUS_SUCCESS) { + lderr(cct) << "Failed; result status not success " << rs << dendl; + element.ret = -EINVAL; + goto Done; + } + if (req->operation != rbi->operation) { + lderr(cct) << "Failed; response operation mismatch, got " << req->operation << " expected " << rbi->operation << dendl; + element.ret = -EINVAL; + goto Done; + } + switch(req->operation) + { + case KMIP_OP_CREATE: { + CreateResponsePayload *pld = (CreateResponsePayload *)req->response_payload; + element.out = static_cast<char *>(malloc(pld->unique_identifier->size+1)); + memcpy(element.out, pld->unique_identifier->value, pld->unique_identifier->size); + element.out[pld->unique_identifier->size] = 0; + } break; + case KMIP_OP_LOCATE: { + LocateResponsePayload *pld = (LocateResponsePayload *)req->response_payload; + char **list = static_cast<char **>(malloc(sizeof (char*) * (1 + pld->unique_identifiers_count))); + for (i = 0; i < pld->unique_identifiers_count; ++i) { + list[i] = static_cast<char *>(malloc(pld->unique_identifiers[i].size+1)); + memcpy(list[i], pld->unique_identifiers[i].value, pld->unique_identifiers[i].size); + list[i][pld->unique_identifiers[i].size] = 0; + } + list[i] = 0; + element.outlist->strings = list; + element.outlist->string_count = pld->unique_identifiers_count; + } break; + case KMIP_OP_GET: { + GetResponsePayload *pld = (GetResponsePayload *)req->response_payload; + element.out = static_cast<char *>(malloc(pld->unique_identifier->size+1)); + memcpy(element.out, pld->unique_identifier->value, pld->unique_identifier->size); + element.out[pld->unique_identifier->size] = 0; + if (pld->object_type != KMIP_OBJTYPE_SYMMETRIC_KEY) { + lderr(cct) << "get: expected symmetric key got " << pld->object_type << dendl; + element.ret = -EINVAL; + goto Done; + } + KeyBlock *kp = static_cast<SymmetricKey *>(pld->object)->key_block; + ByteString *bp; + if (kp->key_format_type != KMIP_KEYFORMAT_RAW) { + lderr(cct) << "get: expected raw key fromat got " << kp->key_format_type << dendl; + element.ret = -EINVAL; + goto Done; + } + KeyValue *kv = static_cast<KeyValue *>(kp->key_value); + bp = static_cast<ByteString*>(kv->key_material); + element.outkey->data = static_cast<unsigned char *>(malloc(bp->size)); + element.outkey->keylen = bp->size; + memcpy(element.outkey->data, bp->value, bp->size); + } break; + case KMIP_OP_GET_ATTRIBUTES: { + GetAttributesResponsePayload *pld = (GetAttributesResponsePayload *)req->response_payload; + element.out = static_cast<char *>(malloc(pld->unique_identifier->size+1)); + memcpy(element.out, pld->unique_identifier->value, pld->unique_identifier->size); + element.out[pld->unique_identifier->size] = 0; + } break; + case KMIP_OP_GET_ATTRIBUTE_LIST: { + GetAttributeListResponsePayload *pld = (GetAttributeListResponsePayload *)req->response_payload; + element.out = static_cast<char *>(malloc(pld->unique_identifier->size+1)); + memcpy(element.out, pld->unique_identifier->value, pld->unique_identifier->size); + element.out[pld->unique_identifier->size] = 0; + } break; + case KMIP_OP_DESTROY: { + DestroyResponsePayload *pld = (DestroyResponsePayload *)req->response_payload; + element.out = static_cast<char *>(malloc(pld->unique_identifier->size+1)); + memcpy(element.out, pld->unique_identifier->value, pld->unique_identifier->size); + element.out[pld->unique_identifier->size] = 0; + } break; + default: + lderr(cct) << "Missing response logic op=" << element.operation << dendl; + element.ret = -EINVAL; + goto Done; + } + element.ret = 0; +Done: + if (need_to_free_response) + kmip_free_response_message(h->kmip_ctx, resp_m); + element.done = true; + element.cond.notify_all(); + release_kmip_handle(h); + return element.ret; +} + +void * +RGWKmipWorker::entry() +{ + std::unique_lock entry_lock{m.lock}; + ldout(m.cct, 10) << __func__ << " start" << dendl; + RGWKmipHandles handles{m.cct}; + handles.start(); + while (!m.going_down) { + if (m.requests.empty()) { + m.cond.wait_for(entry_lock, std::chrono::seconds(MAXIDLE)); + continue; + } + auto iter = m.requests.begin(); + auto element = *iter; + m.requests.erase(iter); + entry_lock.unlock(); + (void) handles.do_one_entry(element.details); + entry_lock.lock(); + } + for (;;) { + if (m.requests.empty()) break; + auto iter = m.requests.begin(); + auto element = std::move(*iter); + m.requests.erase(iter); + element.details.ret = -666; + element.details.done = true; + element.details.cond.notify_all(); + } + handles.stop(); + ldout(m.cct, 10) << __func__ << " finish" << dendl; + return nullptr; +} |