From e6918187568dbd01842d8d1d2c808ce16a894239 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 21 Apr 2024 13:54:28 +0200 Subject: Adding upstream version 18.2.2. Signed-off-by: Daniel Baumann --- src/crypto/qat/CMakeLists.txt | 20 ++ src/crypto/qat/qat_crypto_accel.cc | 42 ++++ src/crypto/qat/qat_crypto_accel.h | 35 +++ src/crypto/qat/qat_crypto_plugin.cc | 35 +++ src/crypto/qat/qat_crypto_plugin.h | 42 ++++ src/crypto/qat/qcccrypto.cc | 471 ++++++++++++++++++++++++++++++++++++ src/crypto/qat/qcccrypto.h | 176 ++++++++++++++ 7 files changed, 821 insertions(+) create mode 100644 src/crypto/qat/CMakeLists.txt create mode 100644 src/crypto/qat/qat_crypto_accel.cc create mode 100644 src/crypto/qat/qat_crypto_accel.h create mode 100644 src/crypto/qat/qat_crypto_plugin.cc create mode 100644 src/crypto/qat/qat_crypto_plugin.h create mode 100644 src/crypto/qat/qcccrypto.cc create mode 100644 src/crypto/qat/qcccrypto.h (limited to 'src/crypto/qat') diff --git a/src/crypto/qat/CMakeLists.txt b/src/crypto/qat/CMakeLists.txt new file mode 100644 index 000000000..fb751967a --- /dev/null +++ b/src/crypto/qat/CMakeLists.txt @@ -0,0 +1,20 @@ +## +# QAT wrapper for Ceph +## + +set(qat_crypto_plugin_srcs + qat_crypto_accel.cc + qat_crypto_plugin.cc + qcccrypto.cc) + +add_library(ceph_crypto_qat SHARED ${qat_crypto_plugin_srcs}) + +add_dependencies(crypto_plugins ceph_crypto_qat) + +target_link_libraries(ceph_crypto_qat PRIVATE + QatDrv::qat_s + QatDrv::usdm_drv_s) + +add_dependencies(crypto_plugins ceph_crypto_qat) +set_target_properties(ceph_crypto_qat PROPERTIES VERSION 1.0.0 SOVERSION 1) +install(TARGETS ceph_crypto_qat DESTINATION ${crypto_plugin_dir}) diff --git a/src/crypto/qat/qat_crypto_accel.cc b/src/crypto/qat/qat_crypto_accel.cc new file mode 100644 index 000000000..23f86edfa --- /dev/null +++ b/src/crypto/qat/qat_crypto_accel.cc @@ -0,0 +1,42 @@ +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2017 Intel Corporation + * + * Author: Qiaowei Ren + * Author: Ganesh Mahalingam + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +#include "crypto/qat/qat_crypto_accel.h" + +bool QccCryptoAccel::cbc_encrypt(unsigned char* out, const unsigned char* in, size_t size, + const unsigned char (&iv)[AES_256_IVSIZE], + const unsigned char (&key)[AES_256_KEYSIZE]) +{ + if ((size % AES_256_IVSIZE) != 0) { + return false; + } + + return qcccrypto.perform_op(out, in, size, + const_cast(&iv[0]), + const_cast(&key[0]), CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT); +} + +bool QccCryptoAccel::cbc_decrypt(unsigned char* out, const unsigned char* in, size_t size, + const unsigned char (&iv)[AES_256_IVSIZE], + const unsigned char (&key)[AES_256_KEYSIZE]) +{ + if ((size % AES_256_IVSIZE) != 0) { + return false; + } + + return qcccrypto.perform_op(out, in, size, + const_cast(&iv[0]), + const_cast(&key[0]), CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT); +} diff --git a/src/crypto/qat/qat_crypto_accel.h b/src/crypto/qat/qat_crypto_accel.h new file mode 100644 index 000000000..5badefc28 --- /dev/null +++ b/src/crypto/qat/qat_crypto_accel.h @@ -0,0 +1,35 @@ +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2017 Intel Corporation + * + * Author: Qiaowei Ren + * Author: Ganesh Mahalingam + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +#ifndef QAT_CRYPTO_ACCEL_H +#define QAT_CRYPTO_ACCEL_H + +#include "crypto/crypto_accel.h" +#include "crypto/qat/qcccrypto.h" + +class QccCryptoAccel : public CryptoAccel { + public: + QccCrypto qcccrypto; + QccCryptoAccel() { qcccrypto.init(); }; + ~QccCryptoAccel() { qcccrypto.destroy(); }; + + bool cbc_encrypt(unsigned char* out, const unsigned char* in, size_t size, + const unsigned char (&iv)[AES_256_IVSIZE], + const unsigned char (&key)[AES_256_KEYSIZE]) override; + bool cbc_decrypt(unsigned char* out, const unsigned char* in, size_t size, + const unsigned char (&iv)[AES_256_IVSIZE], + const unsigned char (&key)[AES_256_KEYSIZE]) override; +}; +#endif diff --git a/src/crypto/qat/qat_crypto_plugin.cc b/src/crypto/qat/qat_crypto_plugin.cc new file mode 100644 index 000000000..4bf3d61bb --- /dev/null +++ b/src/crypto/qat/qat_crypto_plugin.cc @@ -0,0 +1,35 @@ +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2017 Intel Corporation + * + * Author: Qiaowei Ren + * Author: Ganesh Mahalingam + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + + +#include "crypto/qat/qat_crypto_plugin.h" + +#include "ceph_ver.h" + +std::mutex QccCryptoPlugin::qat_init; + +const char *__ceph_plugin_version() +{ + return CEPH_GIT_NICE_VER; +} + +int __ceph_plugin_init(CephContext *cct, + const std::string& type, + const std::string& name) +{ + PluginRegistry *instance = cct->get_plugin_registry(); + + return instance->add(type, name, new QccCryptoPlugin(cct)); +} diff --git a/src/crypto/qat/qat_crypto_plugin.h b/src/crypto/qat/qat_crypto_plugin.h new file mode 100644 index 000000000..a8d4df7cb --- /dev/null +++ b/src/crypto/qat/qat_crypto_plugin.h @@ -0,0 +1,42 @@ +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2017 Intel Corporation + * + * Author: Qiaowei Ren + * Author: Ganesh Mahalingam + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +#ifndef QAT_CRYPTO_PLUGIN_H +#define QAT_CRYPTO_PLUGIN_H + +#include "crypto/crypto_plugin.h" +#include "crypto/qat/qat_crypto_accel.h" + + +class QccCryptoPlugin : public CryptoPlugin { + static std::mutex qat_init; + +public: + + explicit QccCryptoPlugin(CephContext* cct) : CryptoPlugin(cct) + {} + ~QccCryptoPlugin() + {} + virtual int factory(CryptoAccelRef *cs, std::ostream *ss) + { + std::lock_guard l(qat_init); + if (cryptoaccel == nullptr) + cryptoaccel = CryptoAccelRef(new QccCryptoAccel); + + *cs = cryptoaccel; + return 0; + } +}; +#endif diff --git a/src/crypto/qat/qcccrypto.cc b/src/crypto/qat/qcccrypto.cc new file mode 100644 index 000000000..a3f253726 --- /dev/null +++ b/src/crypto/qat/qcccrypto.cc @@ -0,0 +1,471 @@ +#include "qcccrypto.h" +#include +#include "string.h" +#include +#include "common/debug.h" +#include "include/scope_guard.h" +#include "common/dout.h" +#include "common/errno.h" + +// ----------------------------------------------------------------------------- +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rgw +#undef dout_prefix +#define dout_prefix _prefix(_dout) + +static std::ostream& _prefix(std::ostream* _dout) +{ + return *_dout << "QccCrypto: "; +} +// ----------------------------------------------------------------------------- + +/* + * Poller thread & functions +*/ +static std::mutex qcc_alloc_mutex; +static std::mutex qcc_eng_mutex; +static std::atomic init_called = { false }; + +void* QccCrypto::crypt_thread(void *args) { + struct qcc_thread_args *thread_args = (struct qcc_thread_args *)args; + thread_args->qccinstance->do_crypt(thread_args); + return thread_args; +} + +void QccCrypto::QccFreeInstance(int entry) { + std::lock_guard freeinst(qcc_alloc_mutex); + open_instances.push(entry); +} + +int QccCrypto::QccGetFreeInstance() { + int ret = -1; + std::lock_guard getinst(qcc_alloc_mutex); + if (!open_instances.empty()) { + ret = open_instances.front(); + open_instances.pop(); + } + return ret; +} + +void QccCrypto::cleanup() { + icp_sal_userStop(); + qaeMemDestroy(); + is_init = false; + init_stat = stat; + init_called = false; + derr << "Failure during QAT init sequence. Quitting" << dendl; +} + +/* + * We initialize QAT instance and everything that is common for all ops +*/ +bool QccCrypto::init() +{ + + std::lock_guard l(qcc_eng_mutex); + + if(init_called) { + dout(10) << "Init sequence already called. Skipping duplicate call" << dendl; + return true; + } + + // First call to init + dout(15) << "First init for QAT" << dendl; + init_called = true; + + // Find if the usermode memory driver is available. We need to this to + // create contiguous memory needed by QAT. + stat = qaeMemInit(); + if(stat != CPA_STATUS_SUCCESS) { + derr << "Unable to load memory driver" << dendl; + this->cleanup(); + return false; + } + + stat = icp_sal_userStart("CEPH"); + if(stat != CPA_STATUS_SUCCESS) { + derr << "Unable to start qat device" << dendl; + this->cleanup(); + return false; + } + + qcc_os_mem_alloc((void **)&qcc_inst, sizeof(QCCINST)); + if(qcc_inst == NULL) { + derr << "Unable to alloc mem for instance struct" << dendl; + this->cleanup(); + return false; + } + + // Initialize contents of qcc_inst + qcc_inst->num_instances = 0; + qcc_inst->cy_inst_handles = NULL; + + stat = cpaCyGetNumInstances(&(qcc_inst->num_instances)); + if ((stat != CPA_STATUS_SUCCESS) || (qcc_inst->num_instances <= 0)) { + derr << "Unable to find available instances" << dendl; + this->cleanup(); + return false; + } + + qcc_os_mem_alloc((void **)&qcc_inst->cy_inst_handles, + ((int)qcc_inst->num_instances * sizeof(CpaInstanceHandle))); + if (qcc_inst->cy_inst_handles == NULL) { + derr << "Unable to allocate instances array memory" << dendl; + this->cleanup(); + return false; + } + + stat = cpaCyGetInstances(qcc_inst->num_instances, qcc_inst->cy_inst_handles); + if (stat != CPA_STATUS_SUCCESS) { + derr << "Unable to get instances" << dendl; + this->cleanup(); + return false; + } + + int iter = 0; + //Start Instances + for(iter = 0; iter < qcc_inst->num_instances; iter++) { + stat = cpaCyStartInstance(qcc_inst->cy_inst_handles[iter]); + if(stat != CPA_STATUS_SUCCESS) { + derr << "Unable to start instance" << dendl; + this->cleanup(); + return false; + } + } + + qcc_os_mem_alloc((void **)&qcc_inst->is_polled, + ((int)qcc_inst->num_instances * sizeof(CpaBoolean))); + CpaInstanceInfo2 info; + for(iter = 0; iter < qcc_inst->num_instances; iter++) { + qcc_inst->is_polled[iter] = cpaCyInstanceGetInfo2(qcc_inst->cy_inst_handles[iter], + &info) == CPA_STATUS_SUCCESS ? info.isPolled : CPA_FALSE; + } + + // Allocate memory structures for all instances + qcc_os_mem_alloc((void **)&qcc_sess, + ((int)qcc_inst->num_instances * sizeof(QCCSESS))); + if(qcc_sess == NULL) { + derr << "Unable to allocate memory for session struct" << dendl; + this->cleanup(); + return false; + } + + qcc_os_mem_alloc((void **)&qcc_op_mem, + ((int)qcc_inst->num_instances * sizeof(QCCOPMEM))); + if(qcc_sess == NULL) { + derr << "Unable to allocate memory for opmem struct" << dendl; + this->cleanup(); + return false; + } + + qcc_os_mem_alloc((void **)&cypollthreads, + ((int)qcc_inst->num_instances * sizeof(pthread_t))); + if(cypollthreads == NULL) { + derr << "Unable to allocate memory for pthreads" << dendl; + this->cleanup(); + return false; + } + + //At this point we are only doing an user-space version. + //To-Do: Maybe a kernel based one + for(iter = 0; iter < qcc_inst->num_instances; iter++) { + stat = cpaCySetAddressTranslation(qcc_inst->cy_inst_handles[iter], + qaeVirtToPhysNUMA); + if(stat == CPA_STATUS_SUCCESS) { + // Start HW Polling Thread + // To-Do: Enable epoll & interrupt based later? + // QccCyStartPoll(iter); + // Setup the session structures for crypto operation and populate + // whatever we can now. Rest will be filled in when crypto operation + // happens. + qcc_sess[iter].sess_ctx_sz = 0; + qcc_sess[iter].sess_ctx = NULL; + qcc_sess[iter].sess_stp_data.sessionPriority = CPA_CY_PRIORITY_NORMAL; + qcc_sess[iter].sess_stp_data.symOperation = CPA_CY_SYM_OP_CIPHER; + open_instances.push(iter); + qcc_op_mem[iter].is_mem_alloc = false; + qcc_op_mem[iter].op_complete = false; + qcc_op_mem[iter].op_result = CPA_STATUS_SUCCESS; + qcc_op_mem[iter].sym_op_data = NULL; + qcc_op_mem[iter].buff_meta_size = qcc_op_mem[iter].buff_size = 0; + qcc_op_mem[iter].src_buff_meta = qcc_op_mem[iter].src_buff + = qcc_op_mem[iter].iv_buff = NULL; + qcc_op_mem[iter].src_buff_list = NULL; + qcc_op_mem[iter].src_buff_flat = NULL; + qcc_op_mem[iter].num_buffers = 1; + } else { + derr << "Unable to find address translations of instance " << iter << dendl; + this->cleanup(); + return false; + } + } + is_init = true; + dout(10) << "Init complete" << dendl; + return true; +} + +bool QccCrypto::destroy() { + if((!is_init) || (!init_called)) { + dout(15) << "QAT not initialized here. Nothing to do" << dendl; + return false; + } + + unsigned int retry = 0; + while(retry <= QCC_MAX_RETRIES) { + if(open_instances.size() == qcc_inst->num_instances) { + break; + } else { + retry++; + } + dout(5) << "QAT is still busy and cannot free resources yet" << dendl; + return false; + } + + dout(10) << "Destroying QAT crypto & related memory" << dendl; + int iter = 0; + + // Free up op related memory + for (iter =0; iter < qcc_inst->num_instances; iter++) { + qcc_contig_mem_free((void **)&(qcc_op_mem[iter].src_buff)); + qcc_contig_mem_free((void **)&(qcc_op_mem[iter].iv_buff)); + qcc_os_mem_free((void **)&(qcc_op_mem[iter].src_buff_list)); + qcc_os_mem_free((void **)&(qcc_op_mem[iter].src_buff_flat)); + qcc_contig_mem_free((void **)&(qcc_op_mem[iter].sym_op_data)); + } + + // Free up Session memory + for(iter = 0; iter < qcc_inst->num_instances; iter++) { + cpaCySymRemoveSession(qcc_inst->cy_inst_handles[iter], qcc_sess[iter].sess_ctx); + qcc_contig_mem_free((void **)&(qcc_sess[iter].sess_ctx)); + } + + // Stop QAT Instances + for(iter = 0; iter < qcc_inst->num_instances; iter++) { + cpaCyStopInstance(qcc_inst->cy_inst_handles[iter]); + } + + // Free up the base structures we use + qcc_os_mem_free((void **)&qcc_op_mem); + qcc_os_mem_free((void **)&qcc_sess); + qcc_os_mem_free((void **)&(qcc_inst->cy_inst_handles)); + qcc_os_mem_free((void **)&(qcc_inst->is_polled)); + qcc_os_mem_free((void **)&cypollthreads); + qcc_os_mem_free((void **)&qcc_inst); + + //Un-init memory driver and QAT HW + icp_sal_userStop(); + qaeMemDestroy(); + init_called = false; + is_init = false; + return true; +} + +void QccCrypto::do_crypt(qcc_thread_args *thread_args) { + auto entry = thread_args->entry; + qcc_op_mem[entry].op_result = cpaCySymPerformOp(qcc_inst->cy_inst_handles[entry], + NULL, + qcc_op_mem[entry].sym_op_data, + qcc_op_mem[entry].src_buff_list, + qcc_op_mem[entry].src_buff_list, + NULL); + qcc_op_mem[entry].op_complete = true; + free(thread_args); +} + +bool QccCrypto::perform_op(unsigned char* out, const unsigned char* in, + size_t size, uint8_t *iv, uint8_t *key, CpaCySymCipherDirection op_type) +{ + if (!init_called) { + dout(10) << "QAT not intialized yet. Initializing now..." << dendl; + if(!QccCrypto::init()) { + derr << "QAT init failed" << dendl; + return false; + } + } + + if(!is_init) + { + dout(10) << "QAT not initialized in this instance or init failed with possible error " << (int)init_stat << dendl; + return is_init; + } + + int avail_inst = -1; + unsigned int retrycount = 0; + while(retrycount <= QCC_MAX_RETRIES) { + avail_inst = QccGetFreeInstance(); + if(avail_inst != -1) { + break; + } else { + retrycount++; + usleep(qcc_sleep_duration); + } + } + + if(avail_inst == -1) { + derr << "Unable to get an QAT instance. Failing request" << dendl; + return false; + } + + dout(15) << "Using inst " << avail_inst << dendl; + // Start polling threads for this instance + //QccCyStartPoll(avail_inst); + + auto sg = make_scope_guard([=] { + //free up the instance irrespective of the op status + dout(15) << "Completed task under " << avail_inst << dendl; + qcc_op_mem[avail_inst].op_complete = false; + QccCrypto::QccFreeInstance(avail_inst); + }); + + /* + * Allocate buffers for this version of the instance if not already done. + * Hold onto to most of them until destructor is called. + */ + if (qcc_op_mem[avail_inst].is_mem_alloc == false) { + + qcc_sess[avail_inst].sess_stp_data.cipherSetupData.cipherAlgorithm = + CPA_CY_SYM_CIPHER_AES_CBC; + qcc_sess[avail_inst].sess_stp_data.cipherSetupData.cipherKeyLenInBytes = + AES_256_KEY_SIZE; + + // Allocate contig memory for buffers that are independent of the + // input/output + stat = cpaCyBufferListGetMetaSize(qcc_inst->cy_inst_handles[avail_inst], + qcc_op_mem[avail_inst].num_buffers, &(qcc_op_mem[avail_inst].buff_meta_size)); + if(stat != CPA_STATUS_SUCCESS) { + derr << "Unable to get buff meta size" << dendl; + return false; + } + + // Allocate Buffer List Private metadata + stat = qcc_contig_mem_alloc((void **)&(qcc_op_mem[avail_inst].src_buff_meta), + qcc_op_mem[avail_inst].buff_meta_size, 1); + if(stat != CPA_STATUS_SUCCESS) { + derr << "Unable to allocate private metadata memory" << dendl; + return false; + } + + // Allocate Buffer List Memory + qcc_os_mem_alloc((void **)&(qcc_op_mem[avail_inst].src_buff_list), sizeof(CpaBufferList)); + qcc_os_mem_alloc((void **)&(qcc_op_mem[avail_inst].src_buff_flat), + (qcc_op_mem[avail_inst].num_buffers * sizeof(CpaFlatBuffer))); + if(qcc_op_mem[avail_inst].src_buff_list == NULL || qcc_op_mem[avail_inst].src_buff_flat == NULL) { + derr << "Unable to allocate bufferlist memory" << dendl; + return false; + } + + // Allocate IV memory + stat = qcc_contig_mem_alloc((void **)&(qcc_op_mem[avail_inst].iv_buff), AES_256_IV_LEN); + if(stat != CPA_STATUS_SUCCESS) { + derr << "Unable to allocate bufferlist memory" << dendl; + return false; + } + + //Assign src stuff for the operation + (qcc_op_mem[avail_inst].src_buff_list)->pBuffers = qcc_op_mem[avail_inst].src_buff_flat; + (qcc_op_mem[avail_inst].src_buff_list)->numBuffers = qcc_op_mem[avail_inst].num_buffers; + (qcc_op_mem[avail_inst].src_buff_list)->pPrivateMetaData = qcc_op_mem[avail_inst].src_buff_meta; + + //Setup OpData + stat = qcc_contig_mem_alloc((void **)&(qcc_op_mem[avail_inst].sym_op_data), + sizeof(CpaCySymOpData)); + if(stat != CPA_STATUS_SUCCESS) { + derr << "Unable to allocate opdata memory" << dendl; + return false; + } + + // Assuming op to be encryption for initiation. This will be reset when we + // exit this block + qcc_sess[avail_inst].sess_stp_data.cipherSetupData.cipherDirection = + CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT; + // Allocate Session memory + stat = cpaCySymSessionCtxGetSize(qcc_inst->cy_inst_handles[avail_inst], + &(qcc_sess[avail_inst].sess_stp_data), &(qcc_sess[avail_inst].sess_ctx_sz)); + if(stat != CPA_STATUS_SUCCESS) { + derr << "Unable to find session size" << dendl; + return false; + } + + stat = qcc_contig_mem_alloc((void **)&(qcc_sess[avail_inst].sess_ctx), + qcc_sess[avail_inst].sess_ctx_sz); + if(stat != CPA_STATUS_SUCCESS) { + derr << "Unable to allocate contig memory" << dendl; + return false; + } + + // Set memalloc flag so that we don't go through this exercise again. + qcc_op_mem[avail_inst].is_mem_alloc = true; + dout(15) << "Instantiation complete for " << avail_inst << dendl; + } + + // Section that runs on every call + // Identify the operation and assign to session + qcc_sess[avail_inst].sess_stp_data.cipherSetupData.cipherDirection = op_type; + qcc_sess[avail_inst].sess_stp_data.cipherSetupData.pCipherKey = (Cpa8U *)key; + + stat = cpaCySymInitSession(qcc_inst->cy_inst_handles[avail_inst], + NULL, + &(qcc_sess[avail_inst].sess_stp_data), + qcc_sess[avail_inst].sess_ctx); + if (stat != CPA_STATUS_SUCCESS) { + derr << "Unable to init session" << dendl; + return false; + } + + // Allocate actual buffers that will hold data + if (qcc_op_mem[avail_inst].buff_size != (Cpa32U)size) { + qcc_contig_mem_free((void **)&(qcc_op_mem[avail_inst].src_buff)); + qcc_op_mem[avail_inst].buff_size = (Cpa32U)size; + stat = qcc_contig_mem_alloc((void **)&(qcc_op_mem[avail_inst].src_buff), + qcc_op_mem[avail_inst].buff_size); + if(stat != CPA_STATUS_SUCCESS) { + derr << "Unable to allocate contig memory" << dendl; + return false; + } + } + + // Copy src & iv into the respective buffers + memcpy(qcc_op_mem[avail_inst].src_buff, in, size); + memcpy(qcc_op_mem[avail_inst].iv_buff, iv, AES_256_IV_LEN); + + //Assign the reminder of the stuff + qcc_op_mem[avail_inst].src_buff_flat->dataLenInBytes = qcc_op_mem[avail_inst].buff_size; + qcc_op_mem[avail_inst].src_buff_flat->pData = qcc_op_mem[avail_inst].src_buff; + + //OpData assignment + qcc_op_mem[avail_inst].sym_op_data->sessionCtx = qcc_sess[avail_inst].sess_ctx; + qcc_op_mem[avail_inst].sym_op_data->packetType = CPA_CY_SYM_PACKET_TYPE_FULL; + qcc_op_mem[avail_inst].sym_op_data->pIv = qcc_op_mem[avail_inst].iv_buff; + qcc_op_mem[avail_inst].sym_op_data->ivLenInBytes = AES_256_IV_LEN; + qcc_op_mem[avail_inst].sym_op_data->cryptoStartSrcOffsetInBytes = 0; + qcc_op_mem[avail_inst].sym_op_data->messageLenToCipherInBytes = qcc_op_mem[avail_inst].buff_size; + + // Perform cipher operation in a thread + qcc_thread_args* thread_args = new qcc_thread_args(); + thread_args->qccinstance = this; + thread_args->entry = avail_inst; + + if (pthread_create(&cypollthreads[avail_inst], NULL, crypt_thread, (void *)thread_args) != 0) { + derr << "Unable to create thread for crypt operation" << dendl; + return false; + } + if (qcc_inst->is_polled[avail_inst] == CPA_TRUE) { + while (!qcc_op_mem[avail_inst].op_complete) { + icp_sal_CyPollInstance(qcc_inst->cy_inst_handles[avail_inst], 0); + } + } + pthread_join(cypollthreads[avail_inst], NULL); + + if(qcc_op_mem[avail_inst].op_result != CPA_STATUS_SUCCESS) { + derr << "Unable to perform crypt operation" << dendl; + return false; + } + + //Copy data back to out buffer + memcpy(out, qcc_op_mem[avail_inst].src_buff, size); + //Always cleanup memory holding user-data at the end + memset(qcc_op_mem[avail_inst].iv_buff, 0, AES_256_IV_LEN); + memset(qcc_op_mem[avail_inst].src_buff, 0, qcc_op_mem[avail_inst].buff_size); + + return true; +} diff --git a/src/crypto/qat/qcccrypto.h b/src/crypto/qat/qcccrypto.h new file mode 100644 index 000000000..a36b0898b --- /dev/null +++ b/src/crypto/qat/qcccrypto.h @@ -0,0 +1,176 @@ +#ifndef QCCCRYPTO_H +#define QCCCRYPTO_H + +#include +#include +#include +#include +#include +#include +extern "C" { +#include "cpa.h" +#include "lac/cpa_cy_sym.h" +#include "lac/cpa_cy_im.h" +#include "qae_mem.h" +#include "icp_sal_user.h" +#include "icp_sal_poll.h" +#include "qae_mem_utils.h" +} + +class QccCrypto { + + public: + CpaCySymCipherDirection qcc_op_type; + + QccCrypto() {}; + ~QccCrypto() {}; + + bool init(); + bool destroy(); + bool perform_op(unsigned char* out, const unsigned char* in, size_t size, + uint8_t *iv, + uint8_t *key, + CpaCySymCipherDirection op_type); + + private: + + // Currently only supporting AES_256_CBC. + // To-Do: Needs to be expanded + static const size_t AES_256_IV_LEN = 16; + static const size_t AES_256_KEY_SIZE = 32; + static const size_t QCC_MAX_RETRIES = 5000; + + /* + * Struct to hold an instance of QAT to handle the crypto operations. These + * will be identified at the start and held until the destructor is called + * To-Do: + * The struct was creating assuming that we will use all the instances. + * Expand current implementation to allow multiple instances to operate + * independently. + */ + struct QCCINST { + CpaInstanceHandle *cy_inst_handles; + CpaBoolean *is_polled; + Cpa16U num_instances; + } *qcc_inst; + + /* + * QAT Crypto Session + * Crypto Session Context and setupdata holds + * priority, type of crypto operation (cipher/chained), + * cipher algorithm (AES, DES, etc), + * single crypto or multi-buffer crypto. + */ + struct QCCSESS { + CpaCySymSessionSetupData sess_stp_data; + Cpa32U sess_ctx_sz; + CpaCySymSessionCtx sess_ctx; + } *qcc_sess; + + /* + * Cipher Memory Allocations + * Holds bufferlist, flatbuffer, cipher opration data and buffermeta needed + * by QAT to perform the operation. Also buffers for IV, SRC, DEST. + */ + struct QCCOPMEM { + // Op common items + bool is_mem_alloc; + bool op_complete; + CpaStatus op_result; + CpaCySymOpData *sym_op_data; + Cpa32U buff_meta_size; + Cpa32U num_buffers; + Cpa32U buff_size; + + //Src data items + Cpa8U *src_buff_meta; + CpaBufferList *src_buff_list; + CpaFlatBuffer *src_buff_flat; + Cpa8U *src_buff; + Cpa8U *iv_buff; + } *qcc_op_mem; + + //QAT HW polling thread input structure + struct qcc_thread_args { + QccCrypto* qccinstance; + int entry; + }; + + + /* + * Function to handle the crypt operation. Will run while the main thread + * runs the polling function on the instance doing the op + */ + void do_crypt(qcc_thread_args *thread_args); + + /* + * Handle queue with free instances to handle op + */ + std::queue open_instances; + int QccGetFreeInstance(); + void QccFreeInstance(int entry); + + /* + * Contiguous Memory Allocator and de-allocator. We are using the usdm + * driver that comes along with QAT to get us direct memory access using + * hugepages. + * To-Do: A kernel based one. + */ + static inline void qcc_contig_mem_free(void **ptr) { + if (*ptr) { + qaeMemFreeNUMA(ptr); + *ptr = NULL; + } + } + + static inline CpaStatus qcc_contig_mem_alloc(void **ptr, Cpa32U size, Cpa32U alignment = 1) { + *ptr = qaeMemAllocNUMA(size, 0, alignment); + if (NULL == *ptr) + { + return CPA_STATUS_RESOURCE; + } + return CPA_STATUS_SUCCESS; + } + + /* + * Malloc & free calls masked to maintain consistency and future kernel + * alloc support. + */ + static inline void qcc_os_mem_free(void **ptr) { + if (*ptr) { + free(*ptr); + *ptr = NULL; + } + } + + static inline CpaStatus qcc_os_mem_alloc(void **ptr, Cpa32U size) { + *ptr = malloc(size); + if (*ptr == NULL) + { + return CPA_STATUS_RESOURCE; + } + return CPA_STATUS_SUCCESS; + } + + std::atomic is_init = { false }; + CpaStatus init_stat, stat; + + /* + * Function to cleanup memory if constructor fails + */ + void cleanup(); + + /* + * Crypto Polling Function & helpers + * This helps to retrieve data from the QAT rings and dispatching the + * associated callbacks. For synchronous operation (like this one), QAT + * library creates an internal callback for the operation. + */ + static void* crypt_thread(void* entry); + CpaStatus QccCyStartPoll(int entry); + void poll_instance(int entry); + + pthread_t *cypollthreads; + static const size_t qcc_sleep_duration = 2; +}; +#endif //QCCCRYPTO_H -- cgit v1.2.3