summaryrefslogtreecommitdiffstats
path: root/src/spdk/intel-ipsec-mb/mb_mgr_code.h
diff options
context:
space:
mode:
Diffstat (limited to 'src/spdk/intel-ipsec-mb/mb_mgr_code.h')
-rw-r--r--src/spdk/intel-ipsec-mb/mb_mgr_code.h1770
1 files changed, 1770 insertions, 0 deletions
diff --git a/src/spdk/intel-ipsec-mb/mb_mgr_code.h b/src/spdk/intel-ipsec-mb/mb_mgr_code.h
new file mode 100644
index 000000000..1b8857826
--- /dev/null
+++ b/src/spdk/intel-ipsec-mb/mb_mgr_code.h
@@ -0,0 +1,1770 @@
+/*******************************************************************************
+ Copyright (c) 2012-2018, Intel Corporation
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ * Neither the name of Intel Corporation nor the names of its contributors
+ may be used to endorse or promote products derived from this software
+ without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*******************************************************************************/
+
+/*
+ * This contains the bulk of the mb_mgr code, with #define's to build
+ * an SSE, AVX, AVX2 or AVX512 version (see mb_mgr_sse.c, mb_mgr_avx.c, etc.)
+ *
+ * get_next_job() returns a job object. This must be filled in and returned
+ * via submit_job() before get_next_job() is called again.
+ *
+ * submit_job() and flush_job() returns a job object. This job object ceases
+ * to be usable at the next call to get_next_job()
+ */
+
+#include <string.h> /* memcpy(), memset() */
+
+#include "include/clear_regs_mem.h"
+
+/*
+ * JOBS() and ADV_JOBS() moved into mb_mgr_code.h
+ * get_next_job() and get_completed_job() API's are no longer inlines.
+ * For binary compatibility they have been made proper symbols.
+ */
+__forceinline
+JOB_AES_HMAC *JOBS(MB_MGR *state, const int offset)
+{
+ char *cp = (char *)state->jobs;
+
+ return (JOB_AES_HMAC *)(cp + offset);
+}
+
+__forceinline
+void ADV_JOBS(int *ptr)
+{
+ *ptr += sizeof(JOB_AES_HMAC);
+ if (*ptr >= (int) (MAX_JOBS * sizeof(JOB_AES_HMAC)))
+ *ptr = 0;
+}
+
+/* ========================================================================= */
+/* Lower level "out of order" schedulers */
+/* ========================================================================= */
+
+__forceinline
+JOB_AES_HMAC *
+SUBMIT_JOB_AES128_DEC(JOB_AES_HMAC *job)
+{
+ AES_CBC_DEC_128(job->src + job->cipher_start_src_offset_in_bytes,
+ job->iv,
+ job->aes_dec_key_expanded,
+ job->dst,
+ job->msg_len_to_cipher_in_bytes & (~15));
+ job->status |= STS_COMPLETED_AES;
+ return job;
+}
+
+__forceinline
+JOB_AES_HMAC *
+SUBMIT_JOB_AES192_DEC(JOB_AES_HMAC *job)
+{
+ AES_CBC_DEC_192(job->src + job->cipher_start_src_offset_in_bytes,
+ job->iv,
+ job->aes_dec_key_expanded,
+ job->dst,
+ job->msg_len_to_cipher_in_bytes);
+ job->status |= STS_COMPLETED_AES;
+ return job;
+}
+
+__forceinline
+JOB_AES_HMAC *
+SUBMIT_JOB_AES256_DEC(JOB_AES_HMAC *job)
+{
+ AES_CBC_DEC_256(job->src + job->cipher_start_src_offset_in_bytes,
+ job->iv,
+ job->aes_dec_key_expanded,
+ job->dst,
+ job->msg_len_to_cipher_in_bytes);
+ job->status |= STS_COMPLETED_AES;
+ return job;
+}
+
+__forceinline
+JOB_AES_HMAC *
+SUBMIT_JOB_AES_ECB_128_ENC(JOB_AES_HMAC *job)
+{
+ AES_ECB_ENC_128(job->src + job->cipher_start_src_offset_in_bytes,
+ job->aes_enc_key_expanded,
+ job->dst,
+ job->msg_len_to_cipher_in_bytes & (~15));
+ job->status |= STS_COMPLETED_AES;
+ return job;
+}
+
+__forceinline
+JOB_AES_HMAC *
+SUBMIT_JOB_AES_ECB_192_ENC(JOB_AES_HMAC *job)
+{
+ AES_ECB_ENC_192(job->src + job->cipher_start_src_offset_in_bytes,
+ job->aes_enc_key_expanded,
+ job->dst,
+ job->msg_len_to_cipher_in_bytes & (~15));
+ job->status |= STS_COMPLETED_AES;
+ return job;
+}
+
+__forceinline
+JOB_AES_HMAC *
+SUBMIT_JOB_AES_ECB_256_ENC(JOB_AES_HMAC *job)
+{
+ AES_ECB_ENC_256(job->src + job->cipher_start_src_offset_in_bytes,
+ job->aes_enc_key_expanded,
+ job->dst,
+ job->msg_len_to_cipher_in_bytes & (~15));
+ job->status |= STS_COMPLETED_AES;
+ return job;
+}
+
+__forceinline
+JOB_AES_HMAC *
+SUBMIT_JOB_AES_ECB_128_DEC(JOB_AES_HMAC *job)
+{
+ AES_ECB_DEC_128(job->src + job->cipher_start_src_offset_in_bytes,
+ job->aes_dec_key_expanded,
+ job->dst,
+ job->msg_len_to_cipher_in_bytes & (~15));
+ job->status |= STS_COMPLETED_AES;
+ return job;
+}
+
+__forceinline
+JOB_AES_HMAC *
+SUBMIT_JOB_AES_ECB_192_DEC(JOB_AES_HMAC *job)
+{
+ AES_ECB_DEC_192(job->src + job->cipher_start_src_offset_in_bytes,
+ job->aes_dec_key_expanded,
+ job->dst,
+ job->msg_len_to_cipher_in_bytes & (~15));
+ job->status |= STS_COMPLETED_AES;
+ return job;
+}
+
+__forceinline
+JOB_AES_HMAC *
+SUBMIT_JOB_AES_ECB_256_DEC(JOB_AES_HMAC *job)
+{
+ AES_ECB_DEC_256(job->src + job->cipher_start_src_offset_in_bytes,
+ job->aes_dec_key_expanded,
+ job->dst,
+ job->msg_len_to_cipher_in_bytes & (~15));
+ job->status |= STS_COMPLETED_AES;
+ return job;
+}
+
+/* ========================================================================= */
+/* Custom hash / cipher */
+/* ========================================================================= */
+
+__forceinline
+JOB_AES_HMAC *
+JOB_CUSTOM_CIPHER(JOB_AES_HMAC *job)
+{
+ if (!(job->status & STS_COMPLETED_AES)) {
+ if (job->cipher_func(job))
+ job->status = STS_INTERNAL_ERROR;
+ else
+ job->status |= STS_COMPLETED_AES;
+ }
+ return job;
+}
+
+__forceinline
+JOB_AES_HMAC *
+SUBMIT_JOB_CUSTOM_CIPHER(JOB_AES_HMAC *job)
+{
+ return JOB_CUSTOM_CIPHER(job);
+}
+
+__forceinline
+JOB_AES_HMAC *
+FLUSH_JOB_CUSTOM_CIPHER(JOB_AES_HMAC *job)
+{
+ return JOB_CUSTOM_CIPHER(job);
+}
+
+__forceinline
+JOB_AES_HMAC *
+JOB_CUSTOM_HASH(JOB_AES_HMAC *job)
+{
+ if (!(job->status & STS_COMPLETED_HMAC)) {
+ if (job->hash_func(job))
+ job->status = STS_INTERNAL_ERROR;
+ else
+ job->status |= STS_COMPLETED_HMAC;
+ }
+ return job;
+}
+
+__forceinline
+JOB_AES_HMAC *
+SUBMIT_JOB_CUSTOM_HASH(JOB_AES_HMAC *job)
+{
+ return JOB_CUSTOM_HASH(job);
+}
+
+__forceinline
+JOB_AES_HMAC *
+FLUSH_JOB_CUSTOM_HASH(JOB_AES_HMAC *job)
+{
+ return JOB_CUSTOM_HASH(job);
+}
+
+/* ========================================================================= */
+/* DOCSIS AES (AES128 CBC + AES128 CFB) */
+/* ========================================================================= */
+
+#define AES_BLOCK_SIZE 16
+
+/**
+ * @brief Encrypts/decrypts the last partial block for DOCSIS SEC v3.1 BPI
+ *
+ * The last partial block is encrypted/decrypted using AES CFB128.
+ * IV is always the next last ciphered block.
+ *
+ * @note It is assumed that length is bigger than one AES 128 block.
+ *
+ * @param job desriptor of performed crypto operation
+ * @return It always returns value passed in \a job
+ */
+__forceinline
+JOB_AES_HMAC *
+DOCSIS_LAST_BLOCK(JOB_AES_HMAC *job)
+{
+ const void *iv = NULL;
+ uint64_t offset = 0;
+ uint64_t partial_bytes = 0;
+
+ if (job == NULL)
+ return job;
+
+ IMB_ASSERT((job->cipher_direction == DECRYPT) ||
+ (job->status & STS_COMPLETED_AES));
+
+ partial_bytes = job->msg_len_to_cipher_in_bytes & (AES_BLOCK_SIZE - 1);
+ offset = job->msg_len_to_cipher_in_bytes & (~(AES_BLOCK_SIZE - 1));
+
+ if (!partial_bytes)
+ return job;
+
+ /* in either case IV has to be next last ciphered block */
+ if (job->cipher_direction == ENCRYPT)
+ iv = job->dst + offset - AES_BLOCK_SIZE;
+ else
+ iv = job->src + job->cipher_start_src_offset_in_bytes +
+ offset - AES_BLOCK_SIZE;
+
+ IMB_ASSERT(partial_bytes <= AES_BLOCK_SIZE);
+ AES_CFB_128_ONE(job->dst + offset,
+ job->src + job->cipher_start_src_offset_in_bytes +
+ offset,
+ iv, job->aes_enc_key_expanded, partial_bytes);
+
+ return job;
+}
+
+/**
+ * @brief Encrypts/decrypts the first and only partial block for
+ * DOCSIS SEC v3.1 BPI
+ *
+ * The first partial block is encrypted/decrypted using AES CFB128.
+ *
+ * @param job desriptor of performed crypto operation
+ * @return It always returns value passed in \a job
+ */
+__forceinline
+JOB_AES_HMAC *
+DOCSIS_FIRST_BLOCK(JOB_AES_HMAC *job)
+{
+ IMB_ASSERT(!(job->status & STS_COMPLETED_AES));
+ IMB_ASSERT(job->msg_len_to_cipher_in_bytes <= AES_BLOCK_SIZE);
+ AES_CFB_128_ONE(job->dst,
+ job->src + job->cipher_start_src_offset_in_bytes,
+ job->iv, job->aes_enc_key_expanded,
+ job->msg_len_to_cipher_in_bytes);
+ job->status |= STS_COMPLETED_AES;
+ return job;
+}
+
+/* ========================================================================= */
+/* DES, 3DES and DOCSIS DES (DES CBC + DES CFB) */
+/* ========================================================================= */
+
+/**
+ * @brief DOCSIS DES cipher encryption
+ *
+ * @param job desriptor of performed crypto operation
+ * @return It always returns value passed in \a job
+ */
+__forceinline
+JOB_AES_HMAC *
+DOCSIS_DES_ENC(JOB_AES_HMAC *job)
+{
+ IMB_ASSERT(!(job->status & STS_COMPLETED_AES));
+ docsis_des_enc_basic(job->src + job->cipher_start_src_offset_in_bytes,
+ job->dst,
+ (int) job->msg_len_to_cipher_in_bytes,
+ job->aes_enc_key_expanded,
+ (const uint64_t *)job->iv);
+ job->status |= STS_COMPLETED_AES;
+ return job;
+}
+
+/**
+ * @brief DOCSIS DES cipher decryption
+ *
+ * @param job desriptor of performed crypto operation
+ * @return It always returns value passed in \a job
+ */
+__forceinline
+JOB_AES_HMAC *
+DOCSIS_DES_DEC(JOB_AES_HMAC *job)
+{
+ IMB_ASSERT(!(job->status & STS_COMPLETED_AES));
+ docsis_des_dec_basic(job->src + job->cipher_start_src_offset_in_bytes,
+ job->dst,
+ (int) job->msg_len_to_cipher_in_bytes,
+ job->aes_dec_key_expanded,
+ (const uint64_t *)job->iv);
+ job->status |= STS_COMPLETED_AES;
+ return job;
+}
+
+/**
+ * @brief DES cipher encryption
+ *
+ * @param job desriptor of performed crypto operation
+ * @return It always returns value passed in \a job
+ */
+__forceinline
+JOB_AES_HMAC *
+DES_CBC_ENC(JOB_AES_HMAC *job)
+{
+ IMB_ASSERT(!(job->status & STS_COMPLETED_AES));
+ des_enc_cbc_basic(job->src + job->cipher_start_src_offset_in_bytes,
+ job->dst,
+ job->msg_len_to_cipher_in_bytes &
+ (~(DES_BLOCK_SIZE - 1)),
+ job->aes_enc_key_expanded, (const uint64_t *)job->iv);
+ job->status |= STS_COMPLETED_AES;
+ return job;
+}
+
+/**
+ * @brief DES cipher decryption
+ *
+ * @param job desriptor of performed crypto operation
+ * @return It always returns value passed in \a job
+ */
+__forceinline
+JOB_AES_HMAC *
+DES_CBC_DEC(JOB_AES_HMAC *job)
+{
+ IMB_ASSERT(!(job->status & STS_COMPLETED_AES));
+ des_dec_cbc_basic(job->src + job->cipher_start_src_offset_in_bytes,
+ job->dst,
+ job->msg_len_to_cipher_in_bytes &
+ (~(DES_BLOCK_SIZE - 1)),
+ job->aes_dec_key_expanded, (const uint64_t *)job->iv);
+ job->status |= STS_COMPLETED_AES;
+ return job;
+}
+
+/**
+ * @brief 3DES cipher encryption
+ *
+ * @param job desriptor of performed crypto operation
+ * @return It always returns value passed in \a job
+ */
+__forceinline
+JOB_AES_HMAC *
+DES3_CBC_ENC(JOB_AES_HMAC *job)
+{
+ const void * const *ks_ptr =
+ (const void * const *)job->aes_enc_key_expanded;
+
+ IMB_ASSERT(!(job->status & STS_COMPLETED_AES));
+ des3_enc_cbc_basic(job->src + job->cipher_start_src_offset_in_bytes,
+ job->dst,
+ job->msg_len_to_cipher_in_bytes &
+ (~(DES_BLOCK_SIZE - 1)),
+ ks_ptr[0], ks_ptr[1], ks_ptr[2],
+ (const uint64_t *)job->iv);
+ job->status |= STS_COMPLETED_AES;
+ return job;
+}
+
+/**
+ * @brief 3DES cipher decryption
+ *
+ * @param job desriptor of performed crypto operation
+ * @return It always returns value passed in \a job
+ */
+__forceinline
+JOB_AES_HMAC *
+DES3_CBC_DEC(JOB_AES_HMAC *job)
+{
+ const void * const *ks_ptr =
+ (const void * const *)job->aes_dec_key_expanded;
+
+ IMB_ASSERT(!(job->status & STS_COMPLETED_AES));
+ des3_dec_cbc_basic(job->src + job->cipher_start_src_offset_in_bytes,
+ job->dst,
+ job->msg_len_to_cipher_in_bytes &
+ (~(DES_BLOCK_SIZE - 1)),
+ ks_ptr[0], ks_ptr[1], ks_ptr[2],
+ (const uint64_t *)job->iv);
+ job->status |= STS_COMPLETED_AES;
+ return job;
+}
+
+/* ========================================================================= */
+/* Cipher submit & flush functions */
+/* ========================================================================= */
+__forceinline
+JOB_AES_HMAC *
+SUBMIT_JOB_AES_ENC(MB_MGR *state, JOB_AES_HMAC *job)
+{
+ if (CBC == job->cipher_mode) {
+ if (16 == job->aes_key_len_in_bytes) {
+ return SUBMIT_JOB_AES128_ENC(&state->aes128_ooo, job);
+ } else if (24 == job->aes_key_len_in_bytes) {
+ return SUBMIT_JOB_AES192_ENC(&state->aes192_ooo, job);
+ } else { /* assume 32 */
+ return SUBMIT_JOB_AES256_ENC(&state->aes256_ooo, job);
+ }
+ } else if (CNTR == job->cipher_mode) {
+ return SUBMIT_JOB_AES_CNTR(job);
+ } else if (CNTR_BITLEN == job->cipher_mode) {
+ return SUBMIT_JOB_AES_CNTR_BIT(job);
+ } else if (ECB == job->cipher_mode) {
+ if (16 == job->aes_key_len_in_bytes) {
+ return SUBMIT_JOB_AES_ECB_128_ENC(job);
+ } else if (24 == job->aes_key_len_in_bytes) {
+ return SUBMIT_JOB_AES_ECB_192_ENC(job);
+ } else { /* assume 32 */
+ return SUBMIT_JOB_AES_ECB_256_ENC(job);
+ }
+ } else if (DOCSIS_SEC_BPI == job->cipher_mode) {
+ if (job->msg_len_to_cipher_in_bytes >= AES_BLOCK_SIZE) {
+ JOB_AES_HMAC *tmp;
+
+ tmp = SUBMIT_JOB_AES128_ENC(&state->docsis_sec_ooo,
+ job);
+ return DOCSIS_LAST_BLOCK(tmp);
+ } else
+ return DOCSIS_FIRST_BLOCK(job);
+ } else if (PON_AES_CNTR == job->cipher_mode) {
+ if (job->msg_len_to_cipher_in_bytes == 0)
+ return SUBMIT_JOB_PON_ENC_NO_CTR(job);
+ else
+ return SUBMIT_JOB_PON_ENC(job);
+#ifndef NO_GCM
+ } else if (GCM == job->cipher_mode) {
+ return SUBMIT_JOB_AES_GCM_ENC(state, job);
+#endif /* NO_GCM */
+ } else if (CUSTOM_CIPHER == job->cipher_mode) {
+ return SUBMIT_JOB_CUSTOM_CIPHER(job);
+ } else if (DES == job->cipher_mode) {
+#ifdef SUBMIT_JOB_DES_CBC_ENC
+ return SUBMIT_JOB_DES_CBC_ENC(&state->des_enc_ooo, job);
+#else
+ return DES_CBC_ENC(job);
+#endif /* SUBMIT_JOB_DES_CBC_ENC */
+ } else if (DOCSIS_DES == job->cipher_mode) {
+#ifdef SUBMIT_JOB_DOCSIS_DES_ENC
+ return SUBMIT_JOB_DOCSIS_DES_ENC(&state->docsis_des_enc_ooo,
+ job);
+#else
+ return DOCSIS_DES_ENC(job);
+#endif /* SUBMIT_JOB_DOCSIS_DES_ENC */
+ } else if (DES3 == job->cipher_mode) {
+#ifdef SUBMIT_JOB_3DES_CBC_ENC
+ return SUBMIT_JOB_3DES_CBC_ENC(&state->des3_enc_ooo, job);
+#else
+ return DES3_CBC_ENC(job);
+#endif
+ } else if (CCM == job->cipher_mode) {
+ return AES_CNTR_CCM_128(job);
+ } else { /* assume NULL_CIPHER */
+ job->status |= STS_COMPLETED_AES;
+ return job;
+ }
+}
+
+__forceinline
+JOB_AES_HMAC *
+FLUSH_JOB_AES_ENC(MB_MGR *state, JOB_AES_HMAC *job)
+{
+ if (CBC == job->cipher_mode) {
+ if (16 == job->aes_key_len_in_bytes) {
+ return FLUSH_JOB_AES128_ENC(&state->aes128_ooo);
+ } else if (24 == job->aes_key_len_in_bytes) {
+ return FLUSH_JOB_AES192_ENC(&state->aes192_ooo);
+ } else { /* assume 32 */
+ return FLUSH_JOB_AES256_ENC(&state->aes256_ooo);
+ }
+#ifndef NO_GCM
+ } else if (GCM == job->cipher_mode) {
+ return FLUSH_JOB_AES_GCM_ENC(state, job);
+#endif /* NO_GCM */
+ } else if (DOCSIS_SEC_BPI == job->cipher_mode) {
+ JOB_AES_HMAC *tmp;
+
+ tmp = FLUSH_JOB_AES128_ENC(&state->docsis_sec_ooo);
+ return DOCSIS_LAST_BLOCK(tmp);
+#ifdef FLUSH_JOB_DES_CBC_ENC
+ } else if (DES == job->cipher_mode) {
+ return FLUSH_JOB_DES_CBC_ENC(&state->des_enc_ooo);
+#endif /* FLUSH_JOB_DES_CBC_ENC */
+#ifdef FLUSH_JOB_3DES_CBC_ENC
+ } else if (DES3 == job->cipher_mode) {
+ return FLUSH_JOB_3DES_CBC_ENC(&state->des3_enc_ooo);
+#endif /* FLUSH_JOB_3DES_CBC_ENC */
+#ifdef FLUSH_JOB_DOCSIS_DES_ENC
+ } else if (DOCSIS_DES == job->cipher_mode) {
+ return FLUSH_JOB_DOCSIS_DES_ENC(&state->docsis_des_enc_ooo);
+#endif /* FLUSH_JOB_DOCSIS_DES_ENC */
+ } else if (CUSTOM_CIPHER == job->cipher_mode) {
+ return FLUSH_JOB_CUSTOM_CIPHER(job);
+ } else { /* assume CNTR/CNTR_BITLEN, ECB, CCM or NULL_CIPHER */
+ return NULL;
+ }
+}
+
+__forceinline
+JOB_AES_HMAC *
+SUBMIT_JOB_AES_DEC(MB_MGR *state, JOB_AES_HMAC *job)
+{
+ if (CBC == job->cipher_mode) {
+ if (16 == job->aes_key_len_in_bytes) {
+ return SUBMIT_JOB_AES128_DEC(job);
+ } else if (24 == job->aes_key_len_in_bytes) {
+ return SUBMIT_JOB_AES192_DEC(job);
+ } else { /* assume 32 */
+ return SUBMIT_JOB_AES256_DEC(job);
+ }
+ } else if (CNTR == job->cipher_mode) {
+ return SUBMIT_JOB_AES_CNTR(job);
+ } else if (CNTR_BITLEN == job->cipher_mode) {
+ return SUBMIT_JOB_AES_CNTR_BIT(job);
+ } else if (ECB == job->cipher_mode) {
+ if (16 == job->aes_key_len_in_bytes) {
+ return SUBMIT_JOB_AES_ECB_128_DEC(job);
+ } else if (24 == job->aes_key_len_in_bytes) {
+ return SUBMIT_JOB_AES_ECB_192_DEC(job);
+ } else { /* assume 32 */
+ return SUBMIT_JOB_AES_ECB_256_DEC(job);
+ }
+ } else if (DOCSIS_SEC_BPI == job->cipher_mode) {
+ if (job->msg_len_to_cipher_in_bytes >= AES_BLOCK_SIZE) {
+ DOCSIS_LAST_BLOCK(job);
+ return SUBMIT_JOB_AES128_DEC(job);
+ } else {
+ return DOCSIS_FIRST_BLOCK(job);
+ }
+ } else if (PON_AES_CNTR == job->cipher_mode) {
+ if (job->msg_len_to_cipher_in_bytes == 0)
+ return SUBMIT_JOB_PON_DEC_NO_CTR(job);
+ else
+ return SUBMIT_JOB_PON_DEC(job);
+#ifndef NO_GCM
+ } else if (GCM == job->cipher_mode) {
+ return SUBMIT_JOB_AES_GCM_DEC(state, job);
+#endif /* NO_GCM */
+ } else if (DES == job->cipher_mode) {
+#ifdef SUBMIT_JOB_DES_CBC_DEC
+ return SUBMIT_JOB_DES_CBC_DEC(&state->des_dec_ooo, job);
+#else
+ (void) state;
+ return DES_CBC_DEC(job);
+#endif /* SUBMIT_JOB_DES_CBC_DEC */
+ } else if (DOCSIS_DES == job->cipher_mode) {
+#ifdef SUBMIT_JOB_DOCSIS_DES_DEC
+ return SUBMIT_JOB_DOCSIS_DES_DEC(&state->docsis_des_dec_ooo,
+ job);
+#else
+ return DOCSIS_DES_DEC(job);
+#endif /* SUBMIT_JOB_DOCSIS_DES_DEC */
+ } else if (DES3 == job->cipher_mode) {
+#ifdef SUBMIT_JOB_3DES_CBC_DEC
+ return SUBMIT_JOB_3DES_CBC_DEC(&state->des3_dec_ooo, job);
+#else
+ return DES3_CBC_DEC(job);
+#endif
+ } else if (CUSTOM_CIPHER == job->cipher_mode) {
+ return SUBMIT_JOB_CUSTOM_CIPHER(job);
+ } else if (CCM == job->cipher_mode) {
+ return AES_CNTR_CCM_128(job);
+ } else {
+ /* assume NULL_CIPHER */
+ job->status |= STS_COMPLETED_AES;
+ return job;
+ }
+}
+
+__forceinline
+JOB_AES_HMAC *
+FLUSH_JOB_AES_DEC(MB_MGR *state, JOB_AES_HMAC *job)
+{
+#ifndef NO_GCM
+ if (GCM == job->cipher_mode)
+ return FLUSH_JOB_AES_GCM_DEC(state, job);
+#endif /* NO_GCM */
+#ifdef FLUSH_JOB_DES_CBC_DEC
+ if (DES == job->cipher_mode)
+ return FLUSH_JOB_DES_CBC_DEC(&state->des_dec_ooo);
+#endif /* FLUSH_JOB_DES_CBC_DEC */
+#ifdef FLUSH_JOB_3DES_CBC_DEC
+ if (DES3 == job->cipher_mode)
+ return FLUSH_JOB_3DES_CBC_DEC(&state->des3_dec_ooo);
+#endif /* FLUSH_JOB_3DES_CBC_DEC */
+#ifdef FLUSH_JOB_DOCSIS_DES_DEC
+ if (DOCSIS_DES == job->cipher_mode)
+ return FLUSH_JOB_DOCSIS_DES_DEC(&state->docsis_des_dec_ooo);
+#endif /* FLUSH_JOB_DOCSIS_DES_DEC */
+ (void) state;
+ return NULL;
+}
+
+/* ========================================================================= */
+/* Hash submit & flush functions */
+/* ========================================================================= */
+
+__forceinline
+JOB_AES_HMAC *
+SUBMIT_JOB_HASH(MB_MGR *state, JOB_AES_HMAC *job)
+{
+#ifdef VERBOSE
+ printf("--------Enter SUBMIT_JOB_HASH --------------\n");
+#endif
+ switch (job->hash_alg) {
+ case SHA1:
+#ifdef HASH_USE_SHAEXT
+ if (state->features & IMB_FEATURE_SHANI)
+ return SUBMIT_JOB_HMAC_NI(&state->hmac_sha_1_ooo, job);
+#endif
+ return SUBMIT_JOB_HMAC(&state->hmac_sha_1_ooo, job);
+ case SHA_224:
+#ifdef HASH_USE_SHAEXT
+ if (state->features & IMB_FEATURE_SHANI)
+ return SUBMIT_JOB_HMAC_SHA_224_NI
+ (&state->hmac_sha_224_ooo, job);
+#endif
+ return SUBMIT_JOB_HMAC_SHA_224(&state->hmac_sha_224_ooo, job);
+ case SHA_256:
+#ifdef HASH_USE_SHAEXT
+ if (state->features & IMB_FEATURE_SHANI)
+ return SUBMIT_JOB_HMAC_SHA_256_NI
+ (&state->hmac_sha_256_ooo, job);
+#endif
+ return SUBMIT_JOB_HMAC_SHA_256(&state->hmac_sha_256_ooo, job);
+ case SHA_384:
+ return SUBMIT_JOB_HMAC_SHA_384(&state->hmac_sha_384_ooo, job);
+ case SHA_512:
+ return SUBMIT_JOB_HMAC_SHA_512(&state->hmac_sha_512_ooo, job);
+ case AES_XCBC:
+ return SUBMIT_JOB_AES_XCBC(&state->aes_xcbc_ooo, job);
+ case MD5:
+ return SUBMIT_JOB_HMAC_MD5(&state->hmac_md5_ooo, job);
+ case CUSTOM_HASH:
+ return SUBMIT_JOB_CUSTOM_HASH(job);
+ case AES_CCM:
+ return SUBMIT_JOB_AES_CCM_AUTH(&state->aes_ccm_ooo, job);
+ case AES_CMAC:
+ /*
+ * CMAC OOO MGR assumes job len in bits
+ * (for CMAC length is provided in bytes)
+ */
+ job->msg_len_to_hash_in_bits =
+ job->msg_len_to_hash_in_bytes * 8;
+ return SUBMIT_JOB_AES_CMAC_AUTH(&state->aes_cmac_ooo, job);
+ case AES_CMAC_BITLEN:
+ return SUBMIT_JOB_AES_CMAC_AUTH(&state->aes_cmac_ooo, job);
+ case PLAIN_SHA1:
+ IMB_SHA1(state,
+ job->src + job->hash_start_src_offset_in_bytes,
+ job->msg_len_to_hash_in_bytes, job->auth_tag_output);
+ job->status |= STS_COMPLETED_HMAC;
+ return job;
+ case PLAIN_SHA_224:
+ IMB_SHA224(state,
+ job->src + job->hash_start_src_offset_in_bytes,
+ job->msg_len_to_hash_in_bytes, job->auth_tag_output);
+ job->status |= STS_COMPLETED_HMAC;
+ return job;
+ case PLAIN_SHA_256:
+ IMB_SHA256(state,
+ job->src + job->hash_start_src_offset_in_bytes,
+ job->msg_len_to_hash_in_bytes, job->auth_tag_output);
+ job->status |= STS_COMPLETED_HMAC;
+ return job;
+ case PLAIN_SHA_384:
+ IMB_SHA384(state,
+ job->src + job->hash_start_src_offset_in_bytes,
+ job->msg_len_to_hash_in_bytes, job->auth_tag_output);
+ job->status |= STS_COMPLETED_HMAC;
+ return job;
+ case PLAIN_SHA_512:
+ IMB_SHA512(state,
+ job->src + job->hash_start_src_offset_in_bytes,
+ job->msg_len_to_hash_in_bytes, job->auth_tag_output);
+ job->status |= STS_COMPLETED_HMAC;
+ return job;
+ default: /* assume GCM, PON_CRC_BIP or NULL_HASH */
+ job->status |= STS_COMPLETED_HMAC;
+ return job;
+ }
+}
+
+__forceinline
+JOB_AES_HMAC *
+FLUSH_JOB_HASH(MB_MGR *state, JOB_AES_HMAC *job)
+{
+ switch (job->hash_alg) {
+ case SHA1:
+#ifdef HASH_USE_SHAEXT
+ if (state->features & IMB_FEATURE_SHANI)
+ return FLUSH_JOB_HMAC_NI(&state->hmac_sha_1_ooo);
+#endif
+ return FLUSH_JOB_HMAC(&state->hmac_sha_1_ooo);
+ case SHA_224:
+#ifdef HASH_USE_SHAEXT
+ if (state->features & IMB_FEATURE_SHANI)
+ return FLUSH_JOB_HMAC_SHA_224_NI
+ (&state->hmac_sha_224_ooo);
+#endif
+ return FLUSH_JOB_HMAC_SHA_224(&state->hmac_sha_224_ooo);
+ case SHA_256:
+#ifdef HASH_USE_SHAEXT
+ if (state->features & IMB_FEATURE_SHANI)
+ return FLUSH_JOB_HMAC_SHA_256_NI
+ (&state->hmac_sha_256_ooo);
+#endif
+ return FLUSH_JOB_HMAC_SHA_256(&state->hmac_sha_256_ooo);
+ case SHA_384:
+ return FLUSH_JOB_HMAC_SHA_384(&state->hmac_sha_384_ooo);
+ case SHA_512:
+ return FLUSH_JOB_HMAC_SHA_512(&state->hmac_sha_512_ooo);
+ case AES_XCBC:
+ return FLUSH_JOB_AES_XCBC(&state->aes_xcbc_ooo);
+ case MD5:
+ return FLUSH_JOB_HMAC_MD5(&state->hmac_md5_ooo);
+ case CUSTOM_HASH:
+ return FLUSH_JOB_CUSTOM_HASH(job);
+ case AES_CCM:
+ return FLUSH_JOB_AES_CCM_AUTH(&state->aes_ccm_ooo);
+ case AES_CMAC:
+ case AES_CMAC_BITLEN:
+ return FLUSH_JOB_AES_CMAC_AUTH(&state->aes_cmac_ooo);
+ default: /* assume GCM or NULL_HASH */
+ if (!(job->status & STS_COMPLETED_HMAC)) {
+ job->status |= STS_COMPLETED_HMAC;
+ return job;
+ }
+ /* if HMAC is complete then return NULL */
+ return NULL;
+ }
+}
+
+
+/* ========================================================================= */
+/* Job submit & flush functions */
+/* ========================================================================= */
+
+#ifdef DEBUG
+#define DEBUG_PUTS(s) \
+ fputs(s, stderr)
+#ifdef _WIN32
+#define INVALID_PRN(_fmt, ...) \
+ fprintf(stderr, "%s():%d: " _fmt, __FUNCTION__, __LINE__, __VA_ARGS__)
+
+#else
+#define INVALID_PRN(_fmt, ...) \
+ fprintf(stderr, "%s():%d: " _fmt, __func__, __LINE__, __VA_ARGS__)
+#endif
+#else
+#define INVALID_PRN(_fmt, ...)
+#define DEBUG_PUTS(s)
+#endif
+
+__forceinline int
+is_job_invalid(const JOB_AES_HMAC *job)
+{
+ const uint64_t auth_tag_len_fips[] = {
+ 0, /* INVALID selection */
+ 20, /* SHA1 */
+ 28, /* SHA_224 */
+ 32, /* SHA_256 */
+ 48, /* SHA_384 */
+ 64, /* SHA_512 */
+ 12, /* AES_XCBC */
+ 16, /* MD5 */
+ 0, /* NULL_HASH */
+#ifndef NO_GCM
+ 16, /* AES_GMAC */
+#endif
+ 0, /* CUSTOM HASH */
+ 0, /* AES_CCM */
+ 16, /* AES_CMAC */
+ };
+ const uint64_t auth_tag_len_ipsec[] = {
+ 0, /* INVALID selection */
+ 12, /* SHA1 */
+ 14, /* SHA_224 */
+ 16, /* SHA_256 */
+ 24, /* SHA_384 */
+ 32, /* SHA_512 */
+ 12, /* AES_XCBC */
+ 12, /* MD5 */
+ 0, /* NULL_HASH */
+#ifndef NO_GCM
+ 16, /* AES_GMAC */
+#endif
+ 0, /* CUSTOM HASH */
+ 0, /* AES_CCM */
+ 16, /* AES_CMAC */
+ 20, /* PLAIN_SHA1 */
+ 28, /* PLAIN_SHA_224 */
+ 32, /* PLAIN_SHA_256 */
+ 48, /* PLAIN_SHA_384 */
+ 64, /* PLAIN_SHA_512 */
+ 4, /* AES_CMAC 3GPP */
+ };
+
+ /* Maximum length of buffer in PON is 2^14 + 8, since maximum
+ * PLI value is 2^14 - 1 + 1 extra byte of padding + 8 bytes
+ * of XGEM header */
+ const uint64_t max_pon_len = (1 << 14) + 8;
+
+ switch (job->cipher_mode) {
+ case CBC:
+ if (job->src == NULL) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->dst == NULL) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->iv == NULL) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->cipher_direction == ENCRYPT &&
+ job->aes_enc_key_expanded == NULL) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->cipher_direction == DECRYPT &&
+ job->aes_dec_key_expanded == NULL) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->aes_key_len_in_bytes != UINT64_C(16) &&
+ job->aes_key_len_in_bytes != UINT64_C(24) &&
+ job->aes_key_len_in_bytes != UINT64_C(32)) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->msg_len_to_cipher_in_bytes == 0) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->msg_len_to_cipher_in_bytes & UINT64_C(15)) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->iv_len_in_bytes != UINT64_C(16)) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ break;
+ case ECB:
+ if (job->src == NULL) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->dst == NULL) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->aes_enc_key_expanded == NULL) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->aes_key_len_in_bytes != UINT64_C(16) &&
+ job->aes_key_len_in_bytes != UINT64_C(24) &&
+ job->aes_key_len_in_bytes != UINT64_C(32)) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->msg_len_to_cipher_in_bytes == 0) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->msg_len_to_cipher_in_bytes & UINT64_C(15)) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->iv_len_in_bytes != UINT64_C(0)) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ break;
+ case CNTR:
+ case CNTR_BITLEN:
+ if (job->src == NULL) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->dst == NULL) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->iv == NULL) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->aes_enc_key_expanded == NULL) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->aes_key_len_in_bytes != UINT64_C(16) &&
+ job->aes_key_len_in_bytes != UINT64_C(24) &&
+ job->aes_key_len_in_bytes != UINT64_C(32)) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->iv_len_in_bytes != UINT64_C(16) &&
+ job->iv_len_in_bytes != UINT64_C(12)) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ /*
+ * msg_len_to_cipher_in_bits is used with CNTR_BITLEN, but it is
+ * effectively the same field as msg_len_to_cipher_in_bytes,
+ * since it is part of the same union
+ */
+ if (job->msg_len_to_cipher_in_bytes == 0) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ break;
+ case NULL_CIPHER:
+ /*
+ * No checks required for this mode
+ * @note NULL cipher doesn't perform memory copy operation
+ * from source to destination
+ */
+ break;
+ case DOCSIS_SEC_BPI:
+ if (job->src == NULL) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->dst == NULL) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->iv == NULL) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->aes_enc_key_expanded == NULL) {
+ /* it has to be set regardless of direction (AES-CFB) */
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->cipher_direction == DECRYPT &&
+ job->aes_dec_key_expanded == NULL) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->aes_key_len_in_bytes != UINT64_C(16)) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->iv_len_in_bytes != UINT64_C(16)) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->msg_len_to_cipher_in_bytes == 0) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ break;
+#ifndef NO_GCM
+ case GCM:
+ if (job->msg_len_to_cipher_in_bytes != 0 && job->src == NULL) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->msg_len_to_cipher_in_bytes != 0 && job->dst == NULL) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->iv == NULL) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ /* Same key structure used for encrypt and decrypt */
+ if (job->cipher_direction == ENCRYPT &&
+ job->aes_enc_key_expanded == NULL) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->cipher_direction == DECRYPT &&
+ job->aes_dec_key_expanded == NULL) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->aes_key_len_in_bytes != UINT64_C(16) &&
+ job->aes_key_len_in_bytes != UINT64_C(24) &&
+ job->aes_key_len_in_bytes != UINT64_C(32)) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->iv_len_in_bytes != UINT64_C(12)) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->hash_alg != AES_GMAC) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ break;
+#endif /* !NO_GCM */
+ case CUSTOM_CIPHER:
+ /* no checks here */
+ if (job->cipher_func == NULL) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ break;
+ case DES:
+ if (job->src == NULL) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->dst == NULL) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->iv == NULL) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->cipher_direction == ENCRYPT &&
+ job->aes_enc_key_expanded == NULL) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->cipher_direction == DECRYPT &&
+ job->aes_dec_key_expanded == NULL) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->aes_key_len_in_bytes != UINT64_C(8)) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->msg_len_to_cipher_in_bytes == 0) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->msg_len_to_cipher_in_bytes & UINT64_C(7)) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->iv_len_in_bytes != UINT64_C(8)) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ break;
+ case DOCSIS_DES:
+ if (job->src == NULL) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->dst == NULL) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->iv == NULL) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->cipher_direction == ENCRYPT &&
+ job->aes_enc_key_expanded == NULL) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->cipher_direction == DECRYPT &&
+ job->aes_dec_key_expanded == NULL) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->aes_key_len_in_bytes != UINT64_C(8)) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->msg_len_to_cipher_in_bytes == 0) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->iv_len_in_bytes != UINT64_C(8)) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ break;
+ case CCM:
+ if (job->msg_len_to_cipher_in_bytes != 0) {
+ if (job->src == NULL) {
+ INVALID_PRN("cipher_mode:%d\n",
+ job->cipher_mode);
+ return 1;
+ }
+ if (job->dst == NULL) {
+ INVALID_PRN("cipher_mode:%d\n",
+ job->cipher_mode);
+ return 1;
+ }
+ }
+ if (job->iv == NULL) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->aes_enc_key_expanded == NULL) {
+ /* AES-CTR and CBC-MAC use only encryption keys */
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ /* currently only AES-CCM-128 is supported */
+ if (job->aes_key_len_in_bytes != UINT64_C(16)) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ /*
+ * From RFC3610:
+ * Nonce length = 15 - L
+ * Valid L values are: 2 to 8
+ * Then valid nonce lengths 13 to 7 (inclusive).
+ */
+ if (job->iv_len_in_bytes > UINT64_C(13) ||
+ job->iv_len_in_bytes < UINT64_C(7)) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->hash_alg != AES_CCM) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ break;
+ case DES3:
+ if (job->src == NULL) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->dst == NULL) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->iv == NULL) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->aes_key_len_in_bytes != UINT64_C(24)) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->msg_len_to_cipher_in_bytes == 0) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->msg_len_to_cipher_in_bytes & UINT64_C(7)) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->iv_len_in_bytes != UINT64_C(8)) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->cipher_direction == ENCRYPT) {
+ const void * const *ks_ptr =
+ (const void * const *)job->aes_enc_key_expanded;
+
+ if (ks_ptr == NULL) {
+ INVALID_PRN("cipher_mode:%d\n",
+ job->cipher_mode);
+ return 1;
+ }
+ if (ks_ptr[0] == NULL || ks_ptr[1] == NULL ||
+ ks_ptr[2] == NULL) {
+ INVALID_PRN("cipher_mode:%d\n",
+ job->cipher_mode);
+ return 1;
+ }
+ } else {
+ const void * const *ks_ptr =
+ (const void * const *)job->aes_dec_key_expanded;
+
+ if (ks_ptr == NULL) {
+ INVALID_PRN("cipher_mode:%d\n",
+ job->cipher_mode);
+ return 1;
+ }
+ if (ks_ptr[0] == NULL || ks_ptr[1] == NULL ||
+ ks_ptr[2] == NULL) {
+ INVALID_PRN("cipher_mode:%d\n",
+ job->cipher_mode);
+ return 1;
+ }
+ }
+ break;
+ case PON_AES_CNTR:
+ /*
+ * CRC and cipher are done together. A few assumptions:
+ * - CRC and cipher start offsets are the same
+ * - last 4 bytes (32 bits) of the buffer is CRC
+ * - updated CRC value is put into the source buffer
+ * (encryption only)
+ * - CRC length is msg_len_to_cipher_in_bytes - 4 bytes
+ * - msg_len_to_cipher_in_bytes is aligned to 4 bytes
+ * - If msg_len_to_cipher_in_bytes is 0, IV and key pointers
+ * are not required, as encryption is not done
+ */
+ if (job->src == NULL) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->dst == NULL) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+
+ /* source and destination buffer pointers cannot be the same,
+ * as there are always 8 bytes that are not ciphered */
+ if (job->src == job->dst) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ if (job->hash_alg != PON_CRC_BIP) {
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+ /*
+ * If message length to cipher != 0, AES-CTR is performed and
+ * key and IV require to be set properly
+ */
+ if (job->msg_len_to_cipher_in_bytes != UINT64_C(0)) {
+
+ /* message size needs to be aligned to 4 bytes */
+ if ((job->msg_len_to_cipher_in_bytes & 3) != 0) {
+ INVALID_PRN("cipher_mode:%d\n",
+ job->cipher_mode);
+ return 1;
+ }
+
+ /* Substract 8 bytes to maximum length since
+ * XGEM header is not ciphered */
+ if ((job->msg_len_to_cipher_in_bytes >
+ (max_pon_len - 8))) {
+ INVALID_PRN("cipher_mode:%d\n",
+ job->cipher_mode);
+ return 1;
+ }
+
+ if (job->aes_key_len_in_bytes != UINT64_C(16)) {
+ INVALID_PRN("cipher_mode:%d\n",
+ job->cipher_mode);
+ return 1;
+ }
+ if (job->iv_len_in_bytes != UINT64_C(16)) {
+ INVALID_PRN("cipher_mode:%d\n",
+ job->cipher_mode);
+ return 1;
+ }
+ }
+ break;
+ default:
+ INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
+ return 1;
+ }
+
+ switch (job->hash_alg) {
+ case SHA1:
+ case AES_XCBC:
+ case MD5:
+ case SHA_224:
+ case SHA_256:
+ case SHA_384:
+ case SHA_512:
+ if (job->src == NULL) {
+ INVALID_PRN("hash_alg:%d\n", job->hash_alg);
+ return 1;
+ }
+ if (job->auth_tag_output_len_in_bytes !=
+ auth_tag_len_ipsec[job->hash_alg] &&
+ job->auth_tag_output_len_in_bytes !=
+ auth_tag_len_fips[job->hash_alg]) {
+ INVALID_PRN("hash_alg:%d\n", job->hash_alg);
+ return 1;
+ }
+ if (job->msg_len_to_hash_in_bytes == 0) {
+ INVALID_PRN("hash_alg:%d\n", job->hash_alg);
+ return 1;
+ }
+ if (job->auth_tag_output == NULL) {
+ INVALID_PRN("hash_alg:%d\n", job->hash_alg);
+ return 1;
+ }
+ break;
+ case NULL_HASH:
+ break;
+#ifndef NO_GCM
+ case AES_GMAC:
+ if (job->auth_tag_output_len_in_bytes < UINT64_C(1) ||
+ job->auth_tag_output_len_in_bytes > UINT64_C(16)) {
+ INVALID_PRN("hash_alg:%d\n", job->hash_alg);
+ return 1;
+ }
+ if ((job->u.GCM.aad_len_in_bytes > 0) &&
+ (job->u.GCM.aad == NULL)) {
+ INVALID_PRN("hash_alg:%d\n", job->hash_alg);
+ return 1;
+ }
+ if (job->cipher_mode != GCM) {
+ INVALID_PRN("hash_alg:%d\n", job->hash_alg);
+ return 1;
+ }
+ if (job->auth_tag_output == NULL) {
+ INVALID_PRN("hash_alg:%d\n", job->hash_alg);
+ return 1;
+ }
+ /*
+ * msg_len_to_hash_in_bytes not checked against zero.
+ * It is not used for AES-GCM & GMAC - see
+ * SUBMIT_JOB_AES_GCM_ENC and SUBMIT_JOB_AES_GCM_DEC functions.
+ */
+ break;
+#endif /* !NO_GCM */
+ case CUSTOM_HASH:
+ if (job->hash_func == NULL) {
+ INVALID_PRN("hash_alg:%d\n", job->hash_alg);
+ return 1;
+ }
+ break;
+ case AES_CCM:
+ if (job->msg_len_to_hash_in_bytes != 0 && job->src == NULL) {
+ INVALID_PRN("hash_alg:%d\n", job->hash_alg);
+ return 1;
+ }
+ if (job->u.CCM.aad_len_in_bytes > 46) {
+ /* 3 x AES_BLOCK - 2 bytes for AAD len */
+ INVALID_PRN("hash_alg:%d\n", job->hash_alg);
+ return 1;
+ }
+ if ((job->u.CCM.aad_len_in_bytes > 0) &&
+ (job->u.CCM.aad == NULL)) {
+ INVALID_PRN("hash_alg:%d\n", job->hash_alg);
+ return 1;
+ }
+ /* M can be any even number from 4 to 16 */
+ if (job->auth_tag_output_len_in_bytes < UINT64_C(4) ||
+ job->auth_tag_output_len_in_bytes > UINT64_C(16) ||
+ ((job->auth_tag_output_len_in_bytes & 1) != 0)) {
+ INVALID_PRN("hash_alg:%d\n", job->hash_alg);
+ return 1;
+ }
+ if (job->cipher_mode != CCM) {
+ INVALID_PRN("hash_alg:%d\n", job->hash_alg);
+ return 1;
+ }
+ /*
+ * AES-CCM allows for only one message for
+ * cipher and uthentication.
+ * AAD can be used to extend authentication over
+ * clear text fields.
+ */
+ if (job->msg_len_to_cipher_in_bytes !=
+ job->msg_len_to_hash_in_bytes) {
+ INVALID_PRN("hash_alg:%d\n", job->hash_alg);
+ return 1;
+ }
+ if (job->cipher_start_src_offset_in_bytes !=
+ job->hash_start_src_offset_in_bytes) {
+ INVALID_PRN("hash_alg:%d\n", job->hash_alg);
+ return 1;
+ }
+ break;
+ case AES_CMAC:
+ case AES_CMAC_BITLEN:
+ /*
+ * WARNING: When using AES_CMAC_BITLEN, length of message
+ * is passed in bits, using job->msg_len_to_hash_in_bits
+ * (unlike "normal" AES_CMAC, where is passed in bytes,
+ * using job->msg_len_to_hash_in_bytes).
+ */
+ if (job->src == NULL) {
+ INVALID_PRN("hash_alg:%d\n", job->hash_alg);
+ return 1;
+ }
+ if ((job->u.CMAC._key_expanded == NULL) ||
+ (job->u.CMAC._skey1 == NULL) ||
+ (job->u.CMAC._skey2 == NULL)) {
+ INVALID_PRN("hash_alg:%d\n", job->hash_alg);
+ return 1;
+ }
+ /* T is 128 bits but 96 bits is also allowed due to
+ * IPsec use case (RFC 4494) and 32 bits for CMAC 3GPP.
+ */
+ if (job->auth_tag_output_len_in_bytes < UINT64_C(4) ||
+ job->auth_tag_output_len_in_bytes > UINT64_C(16)) {
+ INVALID_PRN("hash_alg:%d\n", job->hash_alg);
+ return 1;
+ }
+ if (job->auth_tag_output == NULL) {
+ INVALID_PRN("hash_alg:%d\n", job->hash_alg);
+ return 1;
+ }
+ break;
+ case PLAIN_SHA1:
+ case PLAIN_SHA_224:
+ case PLAIN_SHA_256:
+ case PLAIN_SHA_384:
+ case PLAIN_SHA_512:
+ if (job->auth_tag_output_len_in_bytes !=
+ auth_tag_len_ipsec[job->hash_alg]) {
+ INVALID_PRN("hash_alg:%d\n", job->hash_alg);
+ return 1;
+ }
+ if (job->src == NULL) {
+ INVALID_PRN("hash_alg:%d\n", job->hash_alg);
+ return 1;
+ }
+ if (job->auth_tag_output == NULL) {
+ INVALID_PRN("hash_alg:%d\n", job->hash_alg);
+ return 1;
+ }
+ break;
+ case PON_CRC_BIP:
+ /*
+ * Authentication tag in PON is BIP 32-bit value only
+ * CRC is done together with cipher,
+ * its initial value is read from the source buffer and
+ * updated value put into the destination buffer.
+ * - msg_len_to_hash_in_bytes is aligned to 4 bytes
+ */
+ if (((job->msg_len_to_hash_in_bytes & UINT64_C(3)) != 0) ||
+ (job->msg_len_to_hash_in_bytes < UINT64_C(8)) ||
+ (job->msg_len_to_hash_in_bytes > max_pon_len)) {
+ /*
+ * Length aligned to 4 bytes (and at least 8 bytes,
+ * including 8-byte XGEM header and no more
+ * than max length)
+ */
+ INVALID_PRN("hash_alg:%d\n", job->hash_alg);
+ return 1;
+ }
+ if (job->auth_tag_output_len_in_bytes != UINT64_C(8)) {
+ /* 64-bits:
+ * - BIP 32-bits
+ * - CRC 32-bits
+ */
+ INVALID_PRN("hash_alg:%d\n", job->hash_alg);
+ return 1;
+ }
+ if (job->cipher_mode != PON_AES_CNTR) {
+ INVALID_PRN("hash_alg:%d\n", job->hash_alg);
+ return 1;
+ }
+ if (job->auth_tag_output == NULL) {
+ INVALID_PRN("hash_alg:%d\n", job->hash_alg);
+ return 1;
+ }
+ break;
+ default:
+ INVALID_PRN("hash_alg:%d\n", job->hash_alg);
+ return 1;
+ }
+ return 0;
+}
+
+__forceinline
+JOB_AES_HMAC *SUBMIT_JOB_AES(MB_MGR *state, JOB_AES_HMAC *job)
+{
+ if (job->cipher_direction == ENCRYPT)
+ job = SUBMIT_JOB_AES_ENC(state, job);
+ else
+ job = SUBMIT_JOB_AES_DEC(state, job);
+
+ return job;
+}
+
+__forceinline
+JOB_AES_HMAC *FLUSH_JOB_AES(MB_MGR *state, JOB_AES_HMAC *job)
+{
+ if (job->cipher_direction == ENCRYPT)
+ job = FLUSH_JOB_AES_ENC(state, job);
+ else
+ job = FLUSH_JOB_AES_DEC(state, job);
+
+ return job;
+}
+
+/* submit a half-completed job, based on the status */
+__forceinline
+JOB_AES_HMAC *RESUBMIT_JOB(MB_MGR *state, JOB_AES_HMAC *job)
+{
+ while (job != NULL && job->status < STS_COMPLETED) {
+ if (job->status == STS_COMPLETED_HMAC)
+ job = SUBMIT_JOB_AES(state, job);
+ else /* assumed job->status = STS_COMPLETED_AES */
+ job = SUBMIT_JOB_HASH(state, job);
+ }
+
+ return job;
+}
+
+__forceinline
+JOB_AES_HMAC *submit_new_job(MB_MGR *state, JOB_AES_HMAC *job)
+{
+ if (job->chain_order == CIPHER_HASH)
+ job = SUBMIT_JOB_AES(state, job);
+ else
+ job = SUBMIT_JOB_HASH(state, job);
+
+ job = RESUBMIT_JOB(state, job);
+ return job;
+}
+
+__forceinline
+void complete_job(MB_MGR *state, JOB_AES_HMAC *job)
+{
+ if (job->chain_order == CIPHER_HASH) {
+ /* while() loop optimized for cipher_hash order */
+ while (job->status < STS_COMPLETED) {
+ JOB_AES_HMAC *tmp = FLUSH_JOB_AES(state, job);
+
+ if (tmp == NULL)
+ tmp = FLUSH_JOB_HASH(state, job);
+
+ (void) RESUBMIT_JOB(state, tmp);
+ }
+ } else {
+ /* while() loop optimized for hash_cipher order */
+ while (job->status < STS_COMPLETED) {
+ JOB_AES_HMAC *tmp = FLUSH_JOB_HASH(state, job);
+
+ if (tmp == NULL)
+ tmp = FLUSH_JOB_AES(state, job);
+
+ (void) RESUBMIT_JOB(state, tmp);
+ }
+ }
+}
+
+__forceinline
+JOB_AES_HMAC *
+submit_job_and_check(MB_MGR *state, const int run_check)
+{
+#ifdef SAFE_PARAM
+ if (state == NULL) {
+ DEBUG_PUTS("submit job and check\n");
+ return NULL;
+ }
+#endif
+
+ JOB_AES_HMAC *job = NULL;
+#ifndef LINUX
+ DECLARE_ALIGNED(uint128_t xmm_save[10], 16);
+
+ SAVE_XMMS(xmm_save);
+#endif
+
+ job = JOBS(state, state->next_job);
+
+ if (run_check) {
+ if (is_job_invalid(job)) {
+ job->status = STS_INVALID_ARGS;
+ } else {
+ job->status = STS_BEING_PROCESSED;
+ job = submit_new_job(state, job);
+ }
+ } else {
+ job->status = STS_BEING_PROCESSED;
+ job = submit_new_job(state, job);
+ }
+
+ if (state->earliest_job < 0) {
+ /* state was previously empty */
+ if (job == NULL)
+ state->earliest_job = state->next_job;
+ ADV_JOBS(&state->next_job);
+ goto exit;
+ }
+
+ ADV_JOBS(&state->next_job);
+
+ if (state->earliest_job == state->next_job) {
+ /* Full */
+ job = JOBS(state, state->earliest_job);
+ complete_job(state, job);
+ ADV_JOBS(&state->earliest_job);
+ goto exit;
+ }
+
+ /* not full */
+ job = JOBS(state, state->earliest_job);
+ if (job->status < STS_COMPLETED) {
+ job = NULL;
+ goto exit;
+ }
+
+ ADV_JOBS(&state->earliest_job);
+exit:
+#ifdef SAFE_DATA
+ CLEAR_SCRATCH_GPS();
+ CLEAR_SCRATCH_SIMD_REGS();
+#endif /* SAFE_DATA */
+
+#ifndef LINUX
+ RESTORE_XMMS(xmm_save);
+#endif
+ return job;
+}
+
+JOB_AES_HMAC *
+SUBMIT_JOB(MB_MGR *state)
+{
+ return submit_job_and_check(state, 1);
+}
+
+JOB_AES_HMAC *
+SUBMIT_JOB_NOCHECK(MB_MGR *state)
+{
+ return submit_job_and_check(state, 0);
+}
+
+JOB_AES_HMAC *
+FLUSH_JOB(MB_MGR *state)
+{
+#ifdef SAFE_PARAM
+ if (state == NULL) {
+ DEBUG_PUTS("flush job\n");
+ return NULL;
+ }
+#endif
+ JOB_AES_HMAC *job;
+#ifndef LINUX
+ DECLARE_ALIGNED(uint128_t xmm_save[10], 16);
+#endif
+
+ if (state->earliest_job < 0)
+ return NULL; /* empty */
+
+#ifndef LINUX
+ SAVE_XMMS(xmm_save);
+#endif
+ job = JOBS(state, state->earliest_job);
+ complete_job(state, job);
+
+ ADV_JOBS(&state->earliest_job);
+
+ if (state->earliest_job == state->next_job)
+ state->earliest_job = -1; /* becomes empty */
+
+#ifdef SAFE_DATA
+ CLEAR_SCRATCH_GPS();
+ CLEAR_SCRATCH_SIMD_REGS();
+#endif /* SAFE_DATA */
+
+#ifndef LINUX
+ RESTORE_XMMS(xmm_save);
+#endif
+ return job;
+}
+
+/* ========================================================================= */
+/* ========================================================================= */
+
+uint32_t
+QUEUE_SIZE(MB_MGR *state)
+{
+#ifdef SAFE_PARAM
+ if (state == NULL) {
+ DEBUG_PUTS("queue size\n");
+ return 0;
+ }
+#endif
+ int a, b;
+
+ if (state->earliest_job < 0)
+ return 0;
+ a = state->next_job / sizeof(JOB_AES_HMAC);
+ b = state->earliest_job / sizeof(JOB_AES_HMAC);
+ return ((a-b) & (MAX_JOBS-1));
+}
+
+JOB_AES_HMAC *
+GET_COMPLETED_JOB(MB_MGR *state)
+{
+#ifdef SAFE_PARAM
+ if (state == NULL) {
+ DEBUG_PUTS("get completed job\n");
+ return NULL;
+ }
+#endif
+ JOB_AES_HMAC *job;
+
+ if (state->earliest_job < 0)
+ return NULL;
+
+ job = JOBS(state, state->earliest_job);
+ if (job->status < STS_COMPLETED)
+ return NULL;
+
+ ADV_JOBS(&state->earliest_job);
+
+ if (state->earliest_job == state->next_job)
+ state->earliest_job = -1;
+
+ return job;
+}
+
+JOB_AES_HMAC *
+GET_NEXT_JOB(MB_MGR *state)
+{
+#ifdef SAFE_PARAM
+ if (state == NULL) {
+ DEBUG_PUTS("get next job\n");
+ return NULL;
+ }
+#endif
+ return JOBS(state, state->next_job);
+}
generated by cgit v1.2.3 (git 2.39.1) at 2024-07-20 23:13:02 +0000