Adding upstream version 14.2.21.upstream/14.2.21upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 469 insertions, 0 deletions

diff --git a/src/crypto/qat/qcccrypto.cc b/src/crypto/qat/qcccrypto.cc
new file mode 100644
index 00000000..410727d3
--- /dev/null
+++ b/src/crypto/qat/qcccrypto.cc
@@ -0,0 +1,469 @@
+#include "qcccrypto.h"
+#include  <iostream>
+#include "string.h"
+#include <pthread.h>
+#include "common/debug.h"
+#include "include/scope_guard.h"
+
+// -----------------------------------------------------------------------------
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_rgw
+#undef dout_prefix
+#define dout_prefix _prefix(_dout)
+
+static ostream&
+_prefix(std::ostream* _dout)
+{
+  return *_dout << "QccCrypto: ";
+}
+// -----------------------------------------------------------------------------
+
+/*
+ * Poller thread & functions
+*/
+static std::mutex qcc_alloc_mutex;
+static std::mutex qcc_eng_mutex;
+static std::atomic<bool> init_called = { false };
+
+void* QccCrypto::crypt_thread(void *args) {
+  struct qcc_thread_args *thread_args = (struct qcc_thread_args *)args;
+  thread_args->qccinstance->do_crypt(thread_args);
+}
+
+void QccCrypto::QccFreeInstance(int entry) {
+  std::lock_guard<std::mutex> freeinst(qcc_alloc_mutex);
+  open_instances.push(entry);
+}
+
+int QccCrypto::QccGetFreeInstance() {
+  int ret = -1;
+  std::lock_guard<std::mutex> getinst(qcc_alloc_mutex);
+  if (!open_instances.empty()) {
+    ret = open_instances.front();
+    open_instances.pop();
+  }
+  return ret;
+}
+
+void QccCrypto::cleanup() {
+  icp_sal_userStop();
+  qaeMemDestroy();
+  is_init = false;
+  init_stat = stat;
+  init_called = false;
+  derr << "Failure during QAT init sequence. Quitting" << dendl;
+}
+
+/*
+ * We initialize QAT instance and everything that is common for all ops
+*/
+bool QccCrypto::init()
+{
+
+  std::lock_guard<std::mutex> l(qcc_eng_mutex);
+
+  if(init_called) {
+    dout(10) << "Init sequence already called. Skipping duplicate call" << dendl;
+    return true;
+  }
+
+  // First call to init
+  dout(15) << "First init for QAT" << dendl;
+  init_called = true;
+
+  // Find if the usermode memory driver is available. We need to this to
+  // create contiguous memory needed by QAT.
+  stat = qaeMemInit();
+  if(stat != CPA_STATUS_SUCCESS) {
+    derr << "Unable to load memory driver" << dendl;
+    this->cleanup();
+    return false;
+  }
+
+  stat = icp_sal_userStart("CEPH");
+  if(stat != CPA_STATUS_SUCCESS) {
+    derr << "Unable to start qat device" << dendl;
+    this->cleanup();
+    return false;
+  }
+
+  qcc_os_mem_alloc((void **)&qcc_inst, sizeof(QCCINST));
+  if(qcc_inst == NULL) {
+    derr << "Unable to alloc mem for instance struct" << dendl;
+    this->cleanup();
+    return false;
+  }
+
+  // Initialize contents of qcc_inst
+  qcc_inst->num_instances = 0;
+  qcc_inst->cy_inst_handles = NULL;
+
+  stat = cpaCyGetNumInstances(&(qcc_inst->num_instances));
+  if ((stat != CPA_STATUS_SUCCESS) || (qcc_inst->num_instances <= 0)) {
+    derr << "Unable to find available instances" << dendl;
+    this->cleanup();
+    return false;
+  }
+
+  qcc_os_mem_alloc((void **)&qcc_inst->cy_inst_handles,
+      ((int)qcc_inst->num_instances * sizeof(CpaInstanceHandle)));
+  if (qcc_inst->cy_inst_handles == NULL) {
+    derr << "Unable to allocate instances array memory" << dendl;
+    this->cleanup();
+    return false;
+  }
+
+  stat = cpaCyGetInstances(qcc_inst->num_instances, qcc_inst->cy_inst_handles);
+  if (stat != CPA_STATUS_SUCCESS) {
+    derr << "Unable to get instances" << dendl;
+    this->cleanup();
+    return false;
+  }
+
+  int iter = 0;
+  //Start Instances
+  for(iter = 0; iter < qcc_inst->num_instances; iter++) {
+    stat = cpaCyStartInstance(qcc_inst->cy_inst_handles[iter]);
+    if(stat != CPA_STATUS_SUCCESS) {
+      derr << "Unable to start instance" << dendl;
+      this->cleanup();
+      return false;
+    }
+  }
+
+  qcc_os_mem_alloc((void **)&qcc_inst->is_polled,
+      ((int)qcc_inst->num_instances * sizeof(CpaBoolean)));
+  CpaInstanceInfo2 info;
+  for(iter = 0; iter < qcc_inst->num_instances; iter++) {
+    qcc_inst->is_polled[iter] = cpaCyInstanceGetInfo2(qcc_inst->cy_inst_handles[iter],
+        &info) == CPA_STATUS_SUCCESS ? info.isPolled : CPA_FALSE;
+  }
+
+  // Allocate memory structures for all instances
+  qcc_os_mem_alloc((void **)&qcc_sess,
+      ((int)qcc_inst->num_instances * sizeof(QCCSESS)));
+  if(qcc_sess == NULL) {
+    derr << "Unable to allocate memory for session struct" << dendl;
+    this->cleanup();
+    return false;
+  }
+
+  qcc_os_mem_alloc((void **)&qcc_op_mem,
+      ((int)qcc_inst->num_instances * sizeof(QCCOPMEM)));
+  if(qcc_sess == NULL) {
+    derr << "Unable to allocate memory for opmem struct" << dendl;
+    this->cleanup();
+    return false;
+  }
+
+  qcc_os_mem_alloc((void **)&cypollthreads,
+      ((int)qcc_inst->num_instances * sizeof(pthread_t)));
+  if(cypollthreads == NULL) {
+    derr << "Unable to allocate memory for pthreads" << dendl;
+    this->cleanup();
+    return false;
+  }
+
+  //At this point we are only doing an user-space version.
+  //To-Do: Maybe a kernel based one
+  for(iter = 0; iter < qcc_inst->num_instances; iter++) {
+    stat = cpaCySetAddressTranslation(qcc_inst->cy_inst_handles[iter],
+                                       qaeVirtToPhysNUMA);
+    if(stat == CPA_STATUS_SUCCESS) {
+      // Start HW Polling Thread
+      // To-Do: Enable epoll & interrupt based later?
+      // QccCyStartPoll(iter);
+      // Setup the session structures for crypto operation and populate
+      // whatever we can now. Rest will be filled in when crypto operation
+      // happens.
+      qcc_sess[iter].sess_ctx_sz = 0;
+      qcc_sess[iter].sess_ctx = NULL;
+      qcc_sess[iter].sess_stp_data.sessionPriority = CPA_CY_PRIORITY_NORMAL;
+      qcc_sess[iter].sess_stp_data.symOperation = CPA_CY_SYM_OP_CIPHER;
+      open_instances.push(iter);
+      qcc_op_mem[iter].is_mem_alloc = false;
+      qcc_op_mem[iter].op_complete = false;
+      qcc_op_mem[iter].op_result = CPA_STATUS_SUCCESS;
+      qcc_op_mem[iter].sym_op_data = NULL;
+      qcc_op_mem[iter].buff_meta_size = qcc_op_mem[iter].buff_size = 0;
+      qcc_op_mem[iter].src_buff_meta = qcc_op_mem[iter].src_buff
+        = qcc_op_mem[iter].iv_buff = NULL;
+      qcc_op_mem[iter].src_buff_list = NULL;
+      qcc_op_mem[iter].src_buff_flat = NULL;
+      qcc_op_mem[iter].num_buffers = 1;
+    } else {
+      derr << "Unable to find address translations of instance " << iter << dendl;
+      this->cleanup();
+      return false;
+    }
+  }
+  is_init = true;
+  dout(10) << "Init complete" << dendl;
+  return true;
+}
+
+bool QccCrypto::destroy() {
+  if((!is_init) || (!init_called)) {
+    dout(15) << "QAT not initialized here. Nothing to do" << dendl;
+    return false;
+  }
+
+  unsigned int retry = 0;
+  while(retry <= QCC_MAX_RETRIES) {
+    if(open_instances.size() == qcc_inst->num_instances) {
+      break;
+    } else {
+      retry++;
+    }
+    dout(5) << "QAT is still busy and cannot free resources yet" << dendl;
+    return false;
+  }
+
+  dout(10) << "Destroying QAT crypto & related memory" << dendl;
+  int iter = 0;
+
+  // Free up op related memory
+  for (iter =0; iter < qcc_inst->num_instances; iter++) {
+    qcc_contig_mem_free((void **)&(qcc_op_mem[iter].src_buff));
+    qcc_contig_mem_free((void **)&(qcc_op_mem[iter].iv_buff));
+    qcc_os_mem_free((void **)&(qcc_op_mem[iter].src_buff_list));
+    qcc_os_mem_free((void **)&(qcc_op_mem[iter].src_buff_flat));
+    qcc_contig_mem_free((void **)&(qcc_op_mem[iter].sym_op_data));
+  }
+
+  // Free up Session memory
+  for(iter = 0; iter < qcc_inst->num_instances; iter++) {
+    cpaCySymRemoveSession(qcc_inst->cy_inst_handles[iter], qcc_sess[iter].sess_ctx);
+    qcc_contig_mem_free((void **)&(qcc_sess[iter].sess_ctx));
+  }
+
+  // Stop QAT Instances
+  for(iter = 0; iter < qcc_inst->num_instances; iter++) {
+    cpaCyStopInstance(qcc_inst->cy_inst_handles[iter]);
+  }
+
+  // Free up the base structures we use
+  qcc_os_mem_free((void **)&qcc_op_mem);
+  qcc_os_mem_free((void **)&qcc_sess);
+  qcc_os_mem_free((void **)&(qcc_inst->cy_inst_handles));
+  qcc_os_mem_free((void **)&(qcc_inst->is_polled));
+  qcc_os_mem_free((void **)&cypollthreads);
+  qcc_os_mem_free((void **)&qcc_inst);
+
+  //Un-init memory driver and QAT HW
+  icp_sal_userStop();
+  qaeMemDestroy();
+  init_called = false;
+  is_init = false;
+  return true;
+}
+
+void QccCrypto::do_crypt(qcc_thread_args *thread_args) {
+  auto entry = thread_args->entry;
+  qcc_op_mem[entry].op_result = cpaCySymPerformOp(qcc_inst->cy_inst_handles[entry],
+      NULL,
+      qcc_op_mem[entry].sym_op_data,
+      qcc_op_mem[entry].src_buff_list,
+      qcc_op_mem[entry].src_buff_list,
+      NULL);
+  qcc_op_mem[entry].op_complete = true;
+  free(thread_args);
+}
+
+bool QccCrypto::perform_op(unsigned char* out, const unsigned char* in,
+    size_t size, uint8_t *iv, uint8_t *key, CpaCySymCipherDirection op_type)
+{
+  if (!init_called) {
+    dout(10) << "QAT not intialized yet. Initializing now..." << dendl;
+    if(!QccCrypto::init()) {
+      derr << "QAT init failed" << dendl;
+      return false;
+    }
+  }
+
+  if(!is_init)
+  {
+    dout(10) << "QAT not initialized in this instance or init failed with possible error " << (int)init_stat << dendl;
+    return is_init;
+  }
+
+  int avail_inst = -1;
+  unsigned int retrycount = 0;
+  while(retrycount <= QCC_MAX_RETRIES) {
+    avail_inst = QccGetFreeInstance();
+    if(avail_inst != -1) {
+      break;
+    } else {
+      retrycount++;
+      usleep(qcc_sleep_duration);
+    }
+  }
+
+  if(avail_inst == -1) {
+    derr << "Unable to get an QAT instance. Failing request" << dendl;
+    return false;
+  }
+
+  dout(15) << "Using inst " << avail_inst << dendl;
+  // Start polling threads for this instance
+  //QccCyStartPoll(avail_inst);
+
+  auto sg = make_scope_guard([=] {
+      //free up the instance irrespective of the op status
+      dout(15) << "Completed task under " << avail_inst << dendl;
+      qcc_op_mem[avail_inst].op_complete = false;
+      QccCrypto::QccFreeInstance(avail_inst);
+      });
+
+  /*
+   * Allocate buffers for this version of the instance if not already done.
+   * Hold onto to most of them until destructor is called.
+  */
+  if (qcc_op_mem[avail_inst].is_mem_alloc == false) {
+
+    qcc_sess[avail_inst].sess_stp_data.cipherSetupData.cipherAlgorithm =
+                                                     CPA_CY_SYM_CIPHER_AES_CBC;
+    qcc_sess[avail_inst].sess_stp_data.cipherSetupData.cipherKeyLenInBytes =
+                                                              AES_256_KEY_SIZE;
+
+    // Allocate contig memory for buffers that are independent of the
+    // input/output
+    stat = cpaCyBufferListGetMetaSize(qcc_inst->cy_inst_handles[avail_inst],
+        qcc_op_mem[avail_inst].num_buffers, &(qcc_op_mem[avail_inst].buff_meta_size));
+    if(stat != CPA_STATUS_SUCCESS) {
+      derr << "Unable to get buff meta size" << dendl;
+      return false;
+    }
+
+    // Allocate Buffer List Private metadata
+    stat = qcc_contig_mem_alloc((void **)&(qcc_op_mem[avail_inst].src_buff_meta),
+        qcc_op_mem[avail_inst].buff_meta_size, 1);
+    if(stat != CPA_STATUS_SUCCESS) {
+      derr << "Unable to allocate private metadata memory" << dendl;
+      return false;
+    }
+
+    // Allocate Buffer List Memory
+    qcc_os_mem_alloc((void **)&(qcc_op_mem[avail_inst].src_buff_list), sizeof(CpaBufferList));
+    qcc_os_mem_alloc((void **)&(qcc_op_mem[avail_inst].src_buff_flat),
+              (qcc_op_mem[avail_inst].num_buffers * sizeof(CpaFlatBuffer)));
+    if(qcc_op_mem[avail_inst].src_buff_list == NULL || qcc_op_mem[avail_inst].src_buff_flat == NULL) {
+      derr << "Unable to allocate bufferlist memory" << dendl;
+      return false;
+    }
+
+    // Allocate IV memory
+    stat = qcc_contig_mem_alloc((void **)&(qcc_op_mem[avail_inst].iv_buff), AES_256_IV_LEN);
+    if(stat != CPA_STATUS_SUCCESS) {
+      derr << "Unable to allocate bufferlist memory" << dendl;
+      return false;
+    }
+
+    //Assign src stuff for the operation
+    (qcc_op_mem[avail_inst].src_buff_list)->pBuffers = qcc_op_mem[avail_inst].src_buff_flat;
+    (qcc_op_mem[avail_inst].src_buff_list)->numBuffers = qcc_op_mem[avail_inst].num_buffers;
+    (qcc_op_mem[avail_inst].src_buff_list)->pPrivateMetaData = qcc_op_mem[avail_inst].src_buff_meta;
+
+    //Setup OpData
+    stat = qcc_contig_mem_alloc((void **)&(qcc_op_mem[avail_inst].sym_op_data),
+        sizeof(CpaCySymOpData));
+    if(stat != CPA_STATUS_SUCCESS) {
+      derr << "Unable to allocate opdata memory" << dendl;
+      return false;
+    }
+
+    // Assuming op to be encryption for initiation. This will be reset when we
+    // exit this block
+    qcc_sess[avail_inst].sess_stp_data.cipherSetupData.cipherDirection =
+                                            CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT;
+    // Allocate Session memory
+    stat = cpaCySymSessionCtxGetSize(qcc_inst->cy_inst_handles[avail_inst],
+        &(qcc_sess[avail_inst].sess_stp_data), &(qcc_sess[avail_inst].sess_ctx_sz));
+    if(stat != CPA_STATUS_SUCCESS) {
+      derr << "Unable to find session size" << dendl;
+      return false;
+    }
+
+    stat = qcc_contig_mem_alloc((void **)&(qcc_sess[avail_inst].sess_ctx),
+        qcc_sess[avail_inst].sess_ctx_sz);
+    if(stat != CPA_STATUS_SUCCESS) {
+      derr << "Unable to allocate contig memory" << dendl;
+      return false;
+    }
+
+    // Set memalloc flag so that we don't go through this exercise again.
+    qcc_op_mem[avail_inst].is_mem_alloc = true;
+    dout(15) << "Instantiation complete for " << avail_inst << dendl;
+  }
+
+  // Section that runs on every call
+  // Identify the operation and assign to session
+  qcc_sess[avail_inst].sess_stp_data.cipherSetupData.cipherDirection = op_type;
+  qcc_sess[avail_inst].sess_stp_data.cipherSetupData.pCipherKey = (Cpa8U *)key;
+
+  stat = cpaCySymInitSession(qcc_inst->cy_inst_handles[avail_inst],
+      NULL,
+      &(qcc_sess[avail_inst].sess_stp_data),
+      qcc_sess[avail_inst].sess_ctx);
+  if (stat != CPA_STATUS_SUCCESS) {
+    derr << "Unable to init session" << dendl;
+    return false;
+  }
+
+  // Allocate actual buffers that will hold data
+  if (qcc_op_mem[avail_inst].buff_size != (Cpa32U)size) {
+    qcc_contig_mem_free((void **)&(qcc_op_mem[avail_inst].src_buff));
+    qcc_op_mem[avail_inst].buff_size = (Cpa32U)size;
+    stat = qcc_contig_mem_alloc((void **)&(qcc_op_mem[avail_inst].src_buff),
+              qcc_op_mem[avail_inst].buff_size);
+    if(stat != CPA_STATUS_SUCCESS) {
+      derr << "Unable to allocate contig memory" << dendl;
+      return false;
+    }
+  }
+
+  // Copy src & iv into the respective buffers
+  memcpy(qcc_op_mem[avail_inst].src_buff, in, size);
+  memcpy(qcc_op_mem[avail_inst].iv_buff, iv, AES_256_IV_LEN);
+
+  //Assign the reminder of the stuff
+  qcc_op_mem[avail_inst].src_buff_flat->dataLenInBytes = qcc_op_mem[avail_inst].buff_size;
+  qcc_op_mem[avail_inst].src_buff_flat->pData = qcc_op_mem[avail_inst].src_buff;
+
+  //OpData assignment
+  qcc_op_mem[avail_inst].sym_op_data->sessionCtx = qcc_sess[avail_inst].sess_ctx;
+  qcc_op_mem[avail_inst].sym_op_data->packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
+  qcc_op_mem[avail_inst].sym_op_data->pIv = qcc_op_mem[avail_inst].iv_buff;
+  qcc_op_mem[avail_inst].sym_op_data->ivLenInBytes = AES_256_IV_LEN;
+  qcc_op_mem[avail_inst].sym_op_data->cryptoStartSrcOffsetInBytes = 0;
+  qcc_op_mem[avail_inst].sym_op_data->messageLenToCipherInBytes = qcc_op_mem[avail_inst].buff_size;
+
+  // Perform cipher operation in a thread
+  qcc_thread_args* thread_args = new qcc_thread_args();
+  thread_args->qccinstance = this;
+  thread_args->entry = avail_inst;
+
+  if (pthread_create(&cypollthreads[avail_inst], NULL, crypt_thread, (void *)thread_args) != 0) {
+    derr << "Unable to create thread for crypt operation" << dendl;
+    return false;
+  }
+  if (qcc_inst->is_polled[avail_inst] == CPA_TRUE) {
+    while (!qcc_op_mem[avail_inst].op_complete) {
+      icp_sal_CyPollInstance(qcc_inst->cy_inst_handles[avail_inst], 0);
+    }
+  }
+  pthread_join(cypollthreads[avail_inst], NULL);
+
+  if(qcc_op_mem[avail_inst].op_result != CPA_STATUS_SUCCESS) {
+    derr << "Unable to perform crypt operation" << dendl;
+    return false;
+  }
+
+  //Copy data back to out buffer
+  memcpy(out, qcc_op_mem[avail_inst].src_buff, size);
+  //Always cleanup memory holding user-data at the end
+  memset(qcc_op_mem[avail_inst].iv_buff, 0, AES_256_IV_LEN);
+  memset(qcc_op_mem[avail_inst].src_buff, 0, qcc_op_mem[avail_inst].buff_size);
+
+  return true;
+}