diff options
Diffstat (limited to '')
| -rw-r--r-- | src/crypto/qat/qcccrypto.cc | 469 |
1 files changed, 469 insertions, 0 deletions
diff --git a/src/crypto/qat/qcccrypto.cc b/src/crypto/qat/qcccrypto.cc new file mode 100644 index 000000000..410727d3b --- /dev/null +++ b/src/crypto/qat/qcccrypto.cc @@ -0,0 +1,469 @@ +#include "qcccrypto.h" +#include <iostream> +#include "string.h" +#include <pthread.h> +#include "common/debug.h" +#include "include/scope_guard.h" + +// ----------------------------------------------------------------------------- +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rgw +#undef dout_prefix +#define dout_prefix _prefix(_dout) + +static ostream& +_prefix(std::ostream* _dout) +{ + return *_dout << "QccCrypto: "; +} +// ----------------------------------------------------------------------------- + +/* + * Poller thread & functions +*/ +static std::mutex qcc_alloc_mutex; +static std::mutex qcc_eng_mutex; +static std::atomic<bool> init_called = { false }; + +void* QccCrypto::crypt_thread(void *args) { + struct qcc_thread_args *thread_args = (struct qcc_thread_args *)args; + thread_args->qccinstance->do_crypt(thread_args); +} + +void QccCrypto::QccFreeInstance(int entry) { + std::lock_guard<std::mutex> freeinst(qcc_alloc_mutex); + open_instances.push(entry); +} + +int QccCrypto::QccGetFreeInstance() { + int ret = -1; + std::lock_guard<std::mutex> getinst(qcc_alloc_mutex); + if (!open_instances.empty()) { + ret = open_instances.front(); + open_instances.pop(); + } + return ret; +} + +void QccCrypto::cleanup() { + icp_sal_userStop(); + qaeMemDestroy(); + is_init = false; + init_stat = stat; + init_called = false; + derr << "Failure during QAT init sequence. Quitting" << dendl; +} + +/* + * We initialize QAT instance and everything that is common for all ops +*/ +bool QccCrypto::init() +{ + + std::lock_guard<std::mutex> l(qcc_eng_mutex); + + if(init_called) { + dout(10) << "Init sequence already called. Skipping duplicate call" << dendl; + return true; + } + + // First call to init + dout(15) << "First init for QAT" << dendl; + init_called = true; + + // Find if the usermode memory driver is available. We need to this to + // create contiguous memory needed by QAT. + stat = qaeMemInit(); + if(stat != CPA_STATUS_SUCCESS) { + derr << "Unable to load memory driver" << dendl; + this->cleanup(); + return false; + } + + stat = icp_sal_userStart("CEPH"); + if(stat != CPA_STATUS_SUCCESS) { + derr << "Unable to start qat device" << dendl; + this->cleanup(); + return false; + } + + qcc_os_mem_alloc((void **)&qcc_inst, sizeof(QCCINST)); + if(qcc_inst == NULL) { + derr << "Unable to alloc mem for instance struct" << dendl; + this->cleanup(); + return false; + } + + // Initialize contents of qcc_inst + qcc_inst->num_instances = 0; + qcc_inst->cy_inst_handles = NULL; + + stat = cpaCyGetNumInstances(&(qcc_inst->num_instances)); + if ((stat != CPA_STATUS_SUCCESS) || (qcc_inst->num_instances <= 0)) { + derr << "Unable to find available instances" << dendl; + this->cleanup(); + return false; + } + + qcc_os_mem_alloc((void **)&qcc_inst->cy_inst_handles, + ((int)qcc_inst->num_instances * sizeof(CpaInstanceHandle))); + if (qcc_inst->cy_inst_handles == NULL) { + derr << "Unable to allocate instances array memory" << dendl; + this->cleanup(); + return false; + } + + stat = cpaCyGetInstances(qcc_inst->num_instances, qcc_inst->cy_inst_handles); + if (stat != CPA_STATUS_SUCCESS) { + derr << "Unable to get instances" << dendl; + this->cleanup(); + return false; + } + + int iter = 0; + //Start Instances + for(iter = 0; iter < qcc_inst->num_instances; iter++) { + stat = cpaCyStartInstance(qcc_inst->cy_inst_handles[iter]); + if(stat != CPA_STATUS_SUCCESS) { + derr << "Unable to start instance" << dendl; + this->cleanup(); + return false; + } + } + + qcc_os_mem_alloc((void **)&qcc_inst->is_polled, + ((int)qcc_inst->num_instances * sizeof(CpaBoolean))); + CpaInstanceInfo2 info; + for(iter = 0; iter < qcc_inst->num_instances; iter++) { + qcc_inst->is_polled[iter] = cpaCyInstanceGetInfo2(qcc_inst->cy_inst_handles[iter], + &info) == CPA_STATUS_SUCCESS ? info.isPolled : CPA_FALSE; + } + + // Allocate memory structures for all instances + qcc_os_mem_alloc((void **)&qcc_sess, + ((int)qcc_inst->num_instances * sizeof(QCCSESS))); + if(qcc_sess == NULL) { + derr << "Unable to allocate memory for session struct" << dendl; + this->cleanup(); + return false; + } + + qcc_os_mem_alloc((void **)&qcc_op_mem, + ((int)qcc_inst->num_instances * sizeof(QCCOPMEM))); + if(qcc_sess == NULL) { + derr << "Unable to allocate memory for opmem struct" << dendl; + this->cleanup(); + return false; + } + + qcc_os_mem_alloc((void **)&cypollthreads, + ((int)qcc_inst->num_instances * sizeof(pthread_t))); + if(cypollthreads == NULL) { + derr << "Unable to allocate memory for pthreads" << dendl; + this->cleanup(); + return false; + } + + //At this point we are only doing an user-space version. + //To-Do: Maybe a kernel based one + for(iter = 0; iter < qcc_inst->num_instances; iter++) { + stat = cpaCySetAddressTranslation(qcc_inst->cy_inst_handles[iter], + qaeVirtToPhysNUMA); + if(stat == CPA_STATUS_SUCCESS) { + // Start HW Polling Thread + // To-Do: Enable epoll & interrupt based later? + // QccCyStartPoll(iter); + // Setup the session structures for crypto operation and populate + // whatever we can now. Rest will be filled in when crypto operation + // happens. + qcc_sess[iter].sess_ctx_sz = 0; + qcc_sess[iter].sess_ctx = NULL; + qcc_sess[iter].sess_stp_data.sessionPriority = CPA_CY_PRIORITY_NORMAL; + qcc_sess[iter].sess_stp_data.symOperation = CPA_CY_SYM_OP_CIPHER; + open_instances.push(iter); + qcc_op_mem[iter].is_mem_alloc = false; + qcc_op_mem[iter].op_complete = false; + qcc_op_mem[iter].op_result = CPA_STATUS_SUCCESS; + qcc_op_mem[iter].sym_op_data = NULL; + qcc_op_mem[iter].buff_meta_size = qcc_op_mem[iter].buff_size = 0; + qcc_op_mem[iter].src_buff_meta = qcc_op_mem[iter].src_buff + = qcc_op_mem[iter].iv_buff = NULL; + qcc_op_mem[iter].src_buff_list = NULL; + qcc_op_mem[iter].src_buff_flat = NULL; + qcc_op_mem[iter].num_buffers = 1; + } else { + derr << "Unable to find address translations of instance " << iter << dendl; + this->cleanup(); + return false; + } + } + is_init = true; + dout(10) << "Init complete" << dendl; + return true; +} + +bool QccCrypto::destroy() { + if((!is_init) || (!init_called)) { + dout(15) << "QAT not initialized here. Nothing to do" << dendl; + return false; + } + + unsigned int retry = 0; + while(retry <= QCC_MAX_RETRIES) { + if(open_instances.size() == qcc_inst->num_instances) { + break; + } else { + retry++; + } + dout(5) << "QAT is still busy and cannot free resources yet" << dendl; + return false; + } + + dout(10) << "Destroying QAT crypto & related memory" << dendl; + int iter = 0; + + // Free up op related memory + for (iter =0; iter < qcc_inst->num_instances; iter++) { + qcc_contig_mem_free((void **)&(qcc_op_mem[iter].src_buff)); + qcc_contig_mem_free((void **)&(qcc_op_mem[iter].iv_buff)); + qcc_os_mem_free((void **)&(qcc_op_mem[iter].src_buff_list)); + qcc_os_mem_free((void **)&(qcc_op_mem[iter].src_buff_flat)); + qcc_contig_mem_free((void **)&(qcc_op_mem[iter].sym_op_data)); + } + + // Free up Session memory + for(iter = 0; iter < qcc_inst->num_instances; iter++) { + cpaCySymRemoveSession(qcc_inst->cy_inst_handles[iter], qcc_sess[iter].sess_ctx); + qcc_contig_mem_free((void **)&(qcc_sess[iter].sess_ctx)); + } + + // Stop QAT Instances + for(iter = 0; iter < qcc_inst->num_instances; iter++) { + cpaCyStopInstance(qcc_inst->cy_inst_handles[iter]); + } + + // Free up the base structures we use + qcc_os_mem_free((void **)&qcc_op_mem); + qcc_os_mem_free((void **)&qcc_sess); + qcc_os_mem_free((void **)&(qcc_inst->cy_inst_handles)); + qcc_os_mem_free((void **)&(qcc_inst->is_polled)); + qcc_os_mem_free((void **)&cypollthreads); + qcc_os_mem_free((void **)&qcc_inst); + + //Un-init memory driver and QAT HW + icp_sal_userStop(); + qaeMemDestroy(); + init_called = false; + is_init = false; + return true; +} + +void QccCrypto::do_crypt(qcc_thread_args *thread_args) { + auto entry = thread_args->entry; + qcc_op_mem[entry].op_result = cpaCySymPerformOp(qcc_inst->cy_inst_handles[entry], + NULL, + qcc_op_mem[entry].sym_op_data, + qcc_op_mem[entry].src_buff_list, + qcc_op_mem[entry].src_buff_list, + NULL); + qcc_op_mem[entry].op_complete = true; + free(thread_args); +} + +bool QccCrypto::perform_op(unsigned char* out, const unsigned char* in, + size_t size, uint8_t *iv, uint8_t *key, CpaCySymCipherDirection op_type) +{ + if (!init_called) { + dout(10) << "QAT not intialized yet. Initializing now..." << dendl; + if(!QccCrypto::init()) { + derr << "QAT init failed" << dendl; + return false; + } + } + + if(!is_init) + { + dout(10) << "QAT not initialized in this instance or init failed with possible error " << (int)init_stat << dendl; + return is_init; + } + + int avail_inst = -1; + unsigned int retrycount = 0; + while(retrycount <= QCC_MAX_RETRIES) { + avail_inst = QccGetFreeInstance(); + if(avail_inst != -1) { + break; + } else { + retrycount++; + usleep(qcc_sleep_duration); + } + } + + if(avail_inst == -1) { + derr << "Unable to get an QAT instance. Failing request" << dendl; + return false; + } + + dout(15) << "Using inst " << avail_inst << dendl; + // Start polling threads for this instance + //QccCyStartPoll(avail_inst); + + auto sg = make_scope_guard([=] { + //free up the instance irrespective of the op status + dout(15) << "Completed task under " << avail_inst << dendl; + qcc_op_mem[avail_inst].op_complete = false; + QccCrypto::QccFreeInstance(avail_inst); + }); + + /* + * Allocate buffers for this version of the instance if not already done. + * Hold onto to most of them until destructor is called. + */ + if (qcc_op_mem[avail_inst].is_mem_alloc == false) { + + qcc_sess[avail_inst].sess_stp_data.cipherSetupData.cipherAlgorithm = + CPA_CY_SYM_CIPHER_AES_CBC; + qcc_sess[avail_inst].sess_stp_data.cipherSetupData.cipherKeyLenInBytes = + AES_256_KEY_SIZE; + + // Allocate contig memory for buffers that are independent of the + // input/output + stat = cpaCyBufferListGetMetaSize(qcc_inst->cy_inst_handles[avail_inst], + qcc_op_mem[avail_inst].num_buffers, &(qcc_op_mem[avail_inst].buff_meta_size)); + if(stat != CPA_STATUS_SUCCESS) { + derr << "Unable to get buff meta size" << dendl; + return false; + } + + // Allocate Buffer List Private metadata + stat = qcc_contig_mem_alloc((void **)&(qcc_op_mem[avail_inst].src_buff_meta), + qcc_op_mem[avail_inst].buff_meta_size, 1); + if(stat != CPA_STATUS_SUCCESS) { + derr << "Unable to allocate private metadata memory" << dendl; + return false; + } + + // Allocate Buffer List Memory + qcc_os_mem_alloc((void **)&(qcc_op_mem[avail_inst].src_buff_list), sizeof(CpaBufferList)); + qcc_os_mem_alloc((void **)&(qcc_op_mem[avail_inst].src_buff_flat), + (qcc_op_mem[avail_inst].num_buffers * sizeof(CpaFlatBuffer))); + if(qcc_op_mem[avail_inst].src_buff_list == NULL || qcc_op_mem[avail_inst].src_buff_flat == NULL) { + derr << "Unable to allocate bufferlist memory" << dendl; + return false; + } + + // Allocate IV memory + stat = qcc_contig_mem_alloc((void **)&(qcc_op_mem[avail_inst].iv_buff), AES_256_IV_LEN); + if(stat != CPA_STATUS_SUCCESS) { + derr << "Unable to allocate bufferlist memory" << dendl; + return false; + } + + //Assign src stuff for the operation + (qcc_op_mem[avail_inst].src_buff_list)->pBuffers = qcc_op_mem[avail_inst].src_buff_flat; + (qcc_op_mem[avail_inst].src_buff_list)->numBuffers = qcc_op_mem[avail_inst].num_buffers; + (qcc_op_mem[avail_inst].src_buff_list)->pPrivateMetaData = qcc_op_mem[avail_inst].src_buff_meta; + + //Setup OpData + stat = qcc_contig_mem_alloc((void **)&(qcc_op_mem[avail_inst].sym_op_data), + sizeof(CpaCySymOpData)); + if(stat != CPA_STATUS_SUCCESS) { + derr << "Unable to allocate opdata memory" << dendl; + return false; + } + + // Assuming op to be encryption for initiation. This will be reset when we + // exit this block + qcc_sess[avail_inst].sess_stp_data.cipherSetupData.cipherDirection = + CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT; + // Allocate Session memory + stat = cpaCySymSessionCtxGetSize(qcc_inst->cy_inst_handles[avail_inst], + &(qcc_sess[avail_inst].sess_stp_data), &(qcc_sess[avail_inst].sess_ctx_sz)); + if(stat != CPA_STATUS_SUCCESS) { + derr << "Unable to find session size" << dendl; + return false; + } + + stat = qcc_contig_mem_alloc((void **)&(qcc_sess[avail_inst].sess_ctx), + qcc_sess[avail_inst].sess_ctx_sz); + if(stat != CPA_STATUS_SUCCESS) { + derr << "Unable to allocate contig memory" << dendl; + return false; + } + + // Set memalloc flag so that we don't go through this exercise again. + qcc_op_mem[avail_inst].is_mem_alloc = true; + dout(15) << "Instantiation complete for " << avail_inst << dendl; + } + + // Section that runs on every call + // Identify the operation and assign to session + qcc_sess[avail_inst].sess_stp_data.cipherSetupData.cipherDirection = op_type; + qcc_sess[avail_inst].sess_stp_data.cipherSetupData.pCipherKey = (Cpa8U *)key; + + stat = cpaCySymInitSession(qcc_inst->cy_inst_handles[avail_inst], + NULL, + &(qcc_sess[avail_inst].sess_stp_data), + qcc_sess[avail_inst].sess_ctx); + if (stat != CPA_STATUS_SUCCESS) { + derr << "Unable to init session" << dendl; + return false; + } + + // Allocate actual buffers that will hold data + if (qcc_op_mem[avail_inst].buff_size != (Cpa32U)size) { + qcc_contig_mem_free((void **)&(qcc_op_mem[avail_inst].src_buff)); + qcc_op_mem[avail_inst].buff_size = (Cpa32U)size; + stat = qcc_contig_mem_alloc((void **)&(qcc_op_mem[avail_inst].src_buff), + qcc_op_mem[avail_inst].buff_size); + if(stat != CPA_STATUS_SUCCESS) { + derr << "Unable to allocate contig memory" << dendl; + return false; + } + } + + // Copy src & iv into the respective buffers + memcpy(qcc_op_mem[avail_inst].src_buff, in, size); + memcpy(qcc_op_mem[avail_inst].iv_buff, iv, AES_256_IV_LEN); + + //Assign the reminder of the stuff + qcc_op_mem[avail_inst].src_buff_flat->dataLenInBytes = qcc_op_mem[avail_inst].buff_size; + qcc_op_mem[avail_inst].src_buff_flat->pData = qcc_op_mem[avail_inst].src_buff; + + //OpData assignment + qcc_op_mem[avail_inst].sym_op_data->sessionCtx = qcc_sess[avail_inst].sess_ctx; + qcc_op_mem[avail_inst].sym_op_data->packetType = CPA_CY_SYM_PACKET_TYPE_FULL; + qcc_op_mem[avail_inst].sym_op_data->pIv = qcc_op_mem[avail_inst].iv_buff; + qcc_op_mem[avail_inst].sym_op_data->ivLenInBytes = AES_256_IV_LEN; + qcc_op_mem[avail_inst].sym_op_data->cryptoStartSrcOffsetInBytes = 0; + qcc_op_mem[avail_inst].sym_op_data->messageLenToCipherInBytes = qcc_op_mem[avail_inst].buff_size; + + // Perform cipher operation in a thread + qcc_thread_args* thread_args = new qcc_thread_args(); + thread_args->qccinstance = this; + thread_args->entry = avail_inst; + + if (pthread_create(&cypollthreads[avail_inst], NULL, crypt_thread, (void *)thread_args) != 0) { + derr << "Unable to create thread for crypt operation" << dendl; + return false; + } + if (qcc_inst->is_polled[avail_inst] == CPA_TRUE) { + while (!qcc_op_mem[avail_inst].op_complete) { + icp_sal_CyPollInstance(qcc_inst->cy_inst_handles[avail_inst], 0); + } + } + pthread_join(cypollthreads[avail_inst], NULL); + + if(qcc_op_mem[avail_inst].op_result != CPA_STATUS_SUCCESS) { + derr << "Unable to perform crypt operation" << dendl; + return false; + } + + //Copy data back to out buffer + memcpy(out, qcc_op_mem[avail_inst].src_buff, size); + //Always cleanup memory holding user-data at the end + memset(qcc_op_mem[avail_inst].iv_buff, 0, AES_256_IV_LEN); + memset(qcc_op_mem[avail_inst].src_buff, 0, qcc_op_mem[avail_inst].buff_size); + + return true; +} |