path: root/src/spdk/intel-ipsec-mb/mb_mgr_code.h

/*******************************************************************************
  Copyright (c) 2012-2018, Intel Corporation

  Redistribution and use in source and binary forms, with or without
  modification, are permitted provided that the following conditions are met:

      * Redistributions of source code must retain the above copyright notice,
        this list of conditions and the following disclaimer.
      * Redistributions in binary form must reproduce the above copyright
        notice, this list of conditions and the following disclaimer in the
        documentation and/or other materials provided with the distribution.
      * Neither the name of Intel Corporation nor the names of its contributors
        may be used to endorse or promote products derived from this software
        without specific prior written permission.

  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
  FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*******************************************************************************/

/*
 * This contains the bulk of the mb_mgr code, with #define's to build
 * an SSE, AVX, AVX2 or AVX512 version (see mb_mgr_sse.c, mb_mgr_avx.c, etc.)
 *
 * get_next_job() returns a job object. This must be filled in and returned
 * via submit_job() before get_next_job() is called again.
 *
 * submit_job() and flush_job() returns a job object. This job object ceases
 * to be usable at the next call to get_next_job()
 *
 * Assume JOBS() and ADV_JOBS() from mb_mgr_code.h are available
 */

#include <string.h> /* memcpy(), memset() */

/* ========================================================================= */
/* Lower level "out of order" schedulers */
/* ========================================================================= */

__forceinline
JOB_AES_HMAC *
SUBMIT_JOB_AES128_DEC(JOB_AES_HMAC *job)
{
        AES_CBC_DEC_128(job->src + job->cipher_start_src_offset_in_bytes,
                        job->iv,
                        job->aes_dec_key_expanded,
                        job->dst,
                        job->msg_len_to_cipher_in_bytes & (~15));
        job->status |= STS_COMPLETED_AES;
        return job;
}

__forceinline
JOB_AES_HMAC *
SUBMIT_JOB_AES192_DEC(JOB_AES_HMAC *job)
{
        AES_CBC_DEC_192(job->src + job->cipher_start_src_offset_in_bytes,
                        job->iv,
                        job->aes_dec_key_expanded,
                        job->dst,
                        job->msg_len_to_cipher_in_bytes);
        job->status |= STS_COMPLETED_AES;
        return job;
}

__forceinline
JOB_AES_HMAC *
SUBMIT_JOB_AES256_DEC(JOB_AES_HMAC *job)
{
        AES_CBC_DEC_256(job->src + job->cipher_start_src_offset_in_bytes,
                        job->iv,
                        job->aes_dec_key_expanded,
                        job->dst,
                        job->msg_len_to_cipher_in_bytes);
        job->status |= STS_COMPLETED_AES;
        return job;
}

__forceinline
JOB_AES_HMAC *
SUBMIT_JOB_AES128_CNTR(JOB_AES_HMAC *job)
{
        AES_CNTR_128(job->src + job->cipher_start_src_offset_in_bytes,
                     job->iv,
                     job->aes_enc_key_expanded,
                     job->dst,
                     job->msg_len_to_cipher_in_bytes,
                     job->iv_len_in_bytes);
        job->status |= STS_COMPLETED_AES;
        return job;
}

__forceinline
JOB_AES_HMAC *
SUBMIT_JOB_AES192_CNTR(JOB_AES_HMAC *job)
{
        AES_CNTR_192(job->src + job->cipher_start_src_offset_in_bytes,
                     job->iv,
                     job->aes_enc_key_expanded,
                     job->dst,
                     job->msg_len_to_cipher_in_bytes,
                     job->iv_len_in_bytes);
        job->status |= STS_COMPLETED_AES;
        return job;
}

__forceinline
JOB_AES_HMAC *
SUBMIT_JOB_AES256_CNTR(JOB_AES_HMAC *job)
{
        AES_CNTR_256(job->src + job->cipher_start_src_offset_in_bytes,
                     job->iv,
                     job->aes_enc_key_expanded,
                     job->dst,
                     job->msg_len_to_cipher_in_bytes,
                     job->iv_len_in_bytes);
        job->status |= STS_COMPLETED_AES;
        return job;
}

/* ========================================================================= */
/* AES-CCM */
/* ========================================================================= */

__forceinline
JOB_AES_HMAC *
submit_flush_job_aes_ccm(MB_MGR_CCM_OOO *state, JOB_AES_HMAC *job,
                         const unsigned max_jobs, const int is_submit)
{
        const unsigned lane_blocks_size = 64;
        const unsigned aad_len_size = 2;
        unsigned lane, min_len, min_idx;
        JOB_AES_HMAC *ret_job = NULL;
        uint8_t *pb = NULL;
        unsigned i;

        if (is_submit) {
                /*
                 * SUBMIT
                 * - get a free lane id
                 */
                const unsigned L = AES_BLOCK_SIZE - 1 -
                        (unsigned) job->iv_len_in_bytes;

                lane = state->unused_lanes & 15;
                state->unused_lanes >>= 4;
                pb = &state->init_blocks[lane * lane_blocks_size];

                /*
                 * Build IV for AES-CTR-128.
                 * - byte 0: flags with L'
                 * - bytes 1 to 13: nonce
                 * - zero bytes after nonce (up to byte 15)
                 *
                 * First AES block of init_blocks will always hold this format
                 * throughtout job processing.
                 */
                memset(&pb[8], 0, 8);
                pb[0] = (uint8_t) L - 1; /* flags = L` = L - 1 */
                /* nonce 7 to 13 */
                memcpy(&pb[1], job->iv, job->iv_len_in_bytes);

                if (job->cipher_direction != ENCRYPT) {
                        /* decrypt before authentication */
                        pb[15] = 1;
                        AES_CNTR_128(job->src +
                                     job->cipher_start_src_offset_in_bytes,
                                     pb, job->aes_enc_key_expanded, job->dst,
                                     job->msg_len_to_cipher_in_bytes,
                                     AES_BLOCK_SIZE);
                }

                /* copy job data in and set up inital blocks */
                state->job_in_lane[lane] = job;
                state->lens[lane] = AES_BLOCK_SIZE;
                state->init_done[lane] = 0;
                state->args.in[lane] = pb;
                state->args.keys[lane] = job->aes_enc_key_expanded;
                memset(&state->args.IV[lane], 0, sizeof(state->args.IV[0]));

                /*
                 * Convert AES-CTR IV into BLOCK 0 for CBC-MAC-128:
                 * - correct flags by adding M' (AAD later)
                 * - put message length
                 */
                pb[0] |= ((job->auth_tag_output_len_in_bytes - 2) >> 1) << 3;
                pb[14] = (uint8_t) (job->msg_len_to_hash_in_bytes >> 8);
                pb[15] = (uint8_t) job->msg_len_to_hash_in_bytes;

                /* Make AAD correction and put together AAD blocks, if any */
                if (job->u.CCM.aad_len_in_bytes != 0) {
                        /*
                         * - increment length by length of AAD and
                         *   AAD length size
                         * - add AAD present flag
                         * - copy AAD to the lane initial blocks
                         * - zero trailing block bytes
                         */
                        const unsigned aadl =
                                (unsigned) job->u.CCM.aad_len_in_bytes +
                                aad_len_size;

                        state->lens[lane] +=
                                (aadl + AES_BLOCK_SIZE - 1) &
                                (~(AES_BLOCK_SIZE - 1));
                        pb[0] |= 0x40;
                        pb[AES_BLOCK_SIZE + 0] =
                                (uint8_t) (job->u.CCM.aad_len_in_bytes >> 8);
                        pb[AES_BLOCK_SIZE + 1] =
                                (uint8_t) job->u.CCM.aad_len_in_bytes;
                        memcpy(&pb[AES_BLOCK_SIZE + aad_len_size],
                               job->u.CCM.aad,
                               job->u.CCM.aad_len_in_bytes);
                        memset(&pb[AES_BLOCK_SIZE + aadl], 0,
                               state->lens[lane] - aadl);
                }

                /* enough jobs to start processing? */
                if (state->unused_lanes != 0xf)
                        return NULL;
        } else {
                /*
                 * FLUSH
                 * - find 1st non null job
                 */
                for (lane = 0; lane < max_jobs; lane++)
                        if (state->job_in_lane[lane] != NULL)
                                break;
                if (lane >= max_jobs)
                        return NULL; /* no not null job */
        }

 ccm_round:
        if (is_submit) {
                /*
                 * SUBMIT
                 * - find min common length to process
                 */
                min_idx = 0;
                min_len = state->lens[0];

                for (i = 1; i < max_jobs; i++) {
                        if (min_len > state->lens[i]) {
                                min_idx = i;
                                min_len = state->lens[i];
                        }
                }
        } else {
                /*
                 * FLUSH
                 * - copy good (not null) lane onto empty lanes
                 * - find min common length to process across not null lanes
                 */
                min_idx = lane;
                min_len = state->lens[lane];

                for (i = 0; i < max_jobs; i++) {
                        if (i == lane)
                                continue;

                        if (state->job_in_lane[i] != NULL) {
                                if (min_len > state->lens[i]) {
                                        min_idx = i;
                                        min_len = state->lens[i];
                                }
                        } else {
                                state->args.in[i] = state->args.in[lane];
                                state->args.keys[i] = state->args.keys[lane];
                                state->args.IV[i] = state->args.IV[lane];
                                state->lens[i] = UINT16_MAX;
                                state->init_done[i] = state->init_done[lane];
                        }
                }
        }

        /* subtract min len from all lanes */
        for (i = 0; i < max_jobs; i++)
                state->lens[i] -= min_len;

        /* run the algorythmic code on selected blocks */
        if (min_len != 0)
                AES128_CBC_MAC(&state->args, min_len);

        ret_job = state->job_in_lane[min_idx];
        pb = &state->init_blocks[min_idx * lane_blocks_size];

        if (state->init_done[min_idx] == 0) {
                /*
                 * First block and AAD blocks are done.
                 * Full message blocks are to do.
                 */
                if (ret_job->cipher_direction == ENCRYPT)
                        state->args.in[min_idx] = ret_job->src +
                                ret_job->hash_start_src_offset_in_bytes;
                else
                        state->args.in[min_idx] = ret_job->dst;

                state->init_done[min_idx] = 1;

                if (ret_job->msg_len_to_hash_in_bytes & (~15)) {
                        /* first block + AAD done - process message blocks */
                        state->lens[min_idx] =
                                ret_job->msg_len_to_hash_in_bytes & (~15);
                        goto ccm_round;
                }
        }

        if (state->init_done[min_idx] == 1 &&
            (ret_job->msg_len_to_hash_in_bytes & 15)) {
                /*
                 * First block, AAD, message blocks are done.
                 * Partial message block is still to do.
                 */
                state->init_done[min_idx] = 2;
                state->lens[min_idx] = AES_BLOCK_SIZE;
                memset(&pb[AES_BLOCK_SIZE], 0, AES_BLOCK_SIZE);
                memcpy(&pb[AES_BLOCK_SIZE], state->args.in[min_idx],
                       (size_t) ret_job->msg_len_to_hash_in_bytes & 15);
                state->args.in[min_idx] = &pb[AES_BLOCK_SIZE];
                goto ccm_round;
        }

        /*
         * Final XOR with AES-CNTR on B_0
         * - remove M' and AAD presence bits from flags
         * - set counter to 0
         */
        pb[0] = pb[0] & 7;
        pb[14] = 0;
        pb[15] = 0;

        /*
         * Clever use of AES-CTR mode saves a few ops here.
         * What AES-CCM authentication requires us to do is:
         * AES-CCM: E(KEY,B_0) XOR IV_CBC_MAC
         *
         * And what AES_CTR offers is:
         * AES_CTR: E(KEY, NONCE|COUNTER) XOR PLAIN_TEXT
         *
         * So if:
         * B_0 is passed instead of NONCE|COUNTER and IV instead of PLAIN_TESXT
         * then AES_CTR function is doing pretty much what we need.
         * On top of it can truncate the authentication tag and copy to
         * destination.
         */
        AES_CNTR_128(&state->args.IV[min_idx] /* src = IV */,
                     pb /* nonce/iv = B_0 */,
                     state->args.keys[min_idx],
                     ret_job->auth_tag_output /* dst */,
                     ret_job->auth_tag_output_len_in_bytes /* num_bytes */,
                     AES_BLOCK_SIZE /* nonce/iv len */);

        if (ret_job->cipher_direction == ENCRYPT) {
                /* encrypt after authentication */
                pb[15] = 1; /* start from counter 1, not 0 */
                AES_CNTR_128(ret_job->src +
                             ret_job->cipher_start_src_offset_in_bytes,
                             pb, ret_job->aes_enc_key_expanded, ret_job->dst,
                             ret_job->msg_len_to_cipher_in_bytes,
                             AES_BLOCK_SIZE);
        }

        /* put back processed packet into unused lanes, set job as complete */
        state->unused_lanes = (state->unused_lanes << 4) | min_idx;
        ret_job = state->job_in_lane[min_idx];
        ret_job->status |= (STS_COMPLETED_HMAC|STS_COMPLETED_AES);
        state->job_in_lane[min_idx] = NULL;
        return ret_job;
}

static
JOB_AES_HMAC *
submit_job_aes_ccm_auth_arch(MB_MGR_CCM_OOO *state, JOB_AES_HMAC *job)
{
        return submit_flush_job_aes_ccm(state, job, AES_CCM_MAX_JOBS, 1);
}

static
JOB_AES_HMAC *
flush_job_aes_ccm_auth_arch(MB_MGR_CCM_OOO *state)
{
        return submit_flush_job_aes_ccm(state, NULL, AES_CCM_MAX_JOBS, 0);
}
/* ========================================================================= */
/* AES-CMAC */
/* ========================================================================= */

/*
 * Implementation follows Figure 2.3 from RFC 4493
 *
 * +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 * +                   Algorithm AES-CMAC                              +
 * +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 * +                                                                   +
 * +   Input    : K    ( 128-bit key )                                 +
 * +            : M    ( message to be authenticated )                 +
 * +            : len  ( length of the message in octets )             +
 * +   Output   : T    ( message authentication code )                 +
 * +                                                                   +
 * +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 * +   Constants: const_Zero is 0x00000000000000000000000000000000     +
 * +              const_Bsize is 16                                    +
 * +                                                                   +
 * +   Variables: K1, K2 for 128-bit subkeys                           +
 * +              M_i is the i-th block (i=1..ceil(len/const_Bsize))   +
 * +              M_last is the last block xor-ed with K1 or K2        +
 * +              n      for number of blocks to be processed          +
 * +              r      for number of octets of last block            +
 * +              flag   for denoting if last block is complete or not +
 * +                                                                   +
 * +   Step 1.  (K1,K2) := Generate_Subkey(K);                         +
 * +   Step 2.  n := ceil(len/const_Bsize);                            +
 * +   Step 3.  if n = 0                                               +
 * +            then                                                   +
 * +                 n := 1;                                           +
 * +                 flag := false;                                    +
 * +            else                                                   +
 * +                 if len mod const_Bsize is 0                       +
 * +                 then flag := true;                                +
 * +                 else flag := false;                               +
 * +                                                                   +
 * +   Step 4.  if flag is true                                        +
 * +            then M_last := M_n XOR K1;                             +
 * +            else M_last := padding(M_n) XOR K2;                    +
 * +   Step 5.  X := const_Zero;                                       +
 * +   Step 6.  for i := 1 to n-1 do                                   +
 * +                begin                                              +
 * +                  Y := X XOR M_i;                                  +
 * +                  X := AES-128(K,Y);                               +
 * +                end                                                +
 * +            Y := M_last XOR X;                                     +
 * +            T := AES-128(K,Y);                                     +
 * +   Step 7.  return T;                                              +
 * +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 */

__forceinline
JOB_AES_HMAC *
submit_flush_job_aes_cmac(MB_MGR_CMAC_OOO *state, JOB_AES_HMAC *job,
                         const unsigned max_jobs, const int is_submit)
{
        const unsigned lane_scratch_size = 16;
        unsigned lane, min_len, min_idx;
        JOB_AES_HMAC *ret_job = NULL;
        uint8_t *M_last = NULL;
        unsigned i;

        if (is_submit) {
                /*
                 * SUBMIT
                 * - get a free lane id
                 */
                unsigned n = (unsigned) (job->msg_len_to_hash_in_bytes + 15) /
                        AES_BLOCK_SIZE;
                const unsigned r = (unsigned) job->msg_len_to_hash_in_bytes &
                        (AES_BLOCK_SIZE - 1);
                const uint64_t *p_key;
                const uint8_t *job_src =
                        job->src + job->hash_start_src_offset_in_bytes;
                uint64_t *p_dst;
                int flag;

                lane = state->unused_lanes & 15;
                state->unused_lanes >>= 4;

                /* copy job into the lane */
                state->job_in_lane[lane] = job;
                state->args.keys[lane] = job->u.CMAC._key_expanded;
                memset(&state->args.IV[lane], 0, sizeof(state->args.IV[0]));

                M_last = &state->scratch[lane * lane_scratch_size];
                p_dst = (uint64_t *) M_last;

                if (n == 0) {
                        /* only one partial block */
                        state->init_done[lane] = 1;
                        state->args.in[lane] = M_last;
                        state->lens[lane] = AES_BLOCK_SIZE;
                        n = 1;
                        flag = 0;
                } else {
                        /* one or more blocks, potentially partial */
                        state->init_done[lane] = 0;
                        state->args.in[lane] = job_src;
                        state->lens[lane] = (n - 1) * AES_BLOCK_SIZE;
                        flag = (r == 0);
                }

                if (flag) {
                        p_key = job->u.CMAC._skey1;
                        memcpy(M_last, job_src + ((n - 1) * AES_BLOCK_SIZE),
                               AES_BLOCK_SIZE);
                } else  {
                        p_key = job->u.CMAC._skey2;
                        memcpy(M_last, job_src + ((n - 1) * AES_BLOCK_SIZE), r);
                        M_last[r] = 0x80;
                        memset(&M_last[r + 1], 0, AES_BLOCK_SIZE - r - 1);
                }

                p_dst[0] = p_dst[0] ^ p_key[0];
                p_dst[1] = p_dst[1] ^ p_key[1];

                /* enough jobs to start processing? */
                if (state->unused_lanes != 0xf)
                        return NULL;
        } else {
                /*
                 * FLUSH
                 * - find 1st non null job
                 */
                for (lane = 0; lane < max_jobs; lane++)
                        if (state->job_in_lane[lane] != NULL)
                                break;
                if (lane >= max_jobs)
                        return NULL; /* no not null job */
        }

 cmac_round:
        if (is_submit) {
                /*
                 * SUBMIT
                 * - find min common length to process
                 */
                min_idx = 0;
                min_len = state->lens[0];

                for (i = 1; i < max_jobs; i++) {
                        if (min_len > state->lens[i]) {
                                min_idx = i;
                                min_len = state->lens[i];
                        }
                }
        } else {
                /*
                 * FLUSH
                 * - copy good (not null) lane onto empty lanes
                 * - find min common length to process across not null lanes
                 */
                min_idx = lane;
                min_len = state->lens[lane];

                for (i = 0; i < max_jobs; i++) {
                        if (i == lane)
                                continue;

                        if (state->job_in_lane[i] != NULL) {
                                if (min_len > state->lens[i]) {
                                        min_idx = i;
                                        min_len = state->lens[i];
                                }
                        } else {
                                state->args.in[i] = state->args.in[lane];
                                state->args.keys[i] = state->args.keys[lane];
                                state->args.IV[i] = state->args.IV[lane];
                                state->lens[i] = UINT16_MAX;
                                state->init_done[i] = state->init_done[lane];
                        }
                }
        }

        /* subtract min len from all lanes */
        for (i = 0; i < max_jobs; i++)
                state->lens[i] -= min_len;

        /* run the algorythmic code on selected blocks */
        if (min_len != 0)
                AES128_CBC_MAC(&state->args, min_len);

        ret_job = state->job_in_lane[min_idx];
        M_last = &state->scratch[min_idx * lane_scratch_size];

        if (state->init_done[min_idx] == 0) {
                /*
                 * Finish step 6
                 * Y := M_last XOR X;
                 * T := AES-128(K,Y);
                 */
                state->init_done[min_idx] = 1;
                state->args.in[min_idx] = M_last;
                state->lens[min_idx] = AES_BLOCK_SIZE;
                goto cmac_round;
        }

        if (ret_job->auth_tag_output_len_in_bytes == 16)
                memcpy(ret_job->auth_tag_output, &state->args.IV[min_idx], 16);
        else
                memcpy(ret_job->auth_tag_output, &state->args.IV[min_idx], 12);


        /* put back processed packet into unused lanes, set job as complete */
        state->unused_lanes = (state->unused_lanes << 4) | min_idx;
        ret_job = state->job_in_lane[min_idx];
        ret_job->status |= STS_COMPLETED_HMAC;
        state->job_in_lane[min_idx] = NULL;
        return ret_job;
}

static
JOB_AES_HMAC *
submit_job_aes_cmac_auth_arch(MB_MGR_CMAC_OOO *state, JOB_AES_HMAC *job)
{
        return submit_flush_job_aes_cmac(state, job, AES_CMAC_MAX_JOBS, 1);
}

static
JOB_AES_HMAC *
flush_job_aes_cmac_auth_arch(MB_MGR_CMAC_OOO *state)
{
        return submit_flush_job_aes_cmac(state, NULL, AES_CMAC_MAX_JOBS, 0);
}

/* ========================================================================= */
/* AES-GCM */
/* ========================================================================= */

#ifndef NO_GCM
__forceinline
JOB_AES_HMAC *
SUBMIT_JOB_AES_GCM_DEC(JOB_AES_HMAC *job)
{
        DECLARE_ALIGNED(struct gcm_context_data ctx, 16);

        if (16 == job->aes_key_len_in_bytes)
                AES_GCM_DEC_128(job->aes_dec_key_expanded, &ctx, job->dst,
                                job->src +
                                job->cipher_start_src_offset_in_bytes,
                                job->msg_len_to_cipher_in_bytes,
                                job->iv,
                                job->u.GCM.aad, job->u.GCM.aad_len_in_bytes,
                                job->auth_tag_output,
                                job->auth_tag_output_len_in_bytes);
        else if (24 == job->aes_key_len_in_bytes)
                AES_GCM_DEC_192(job->aes_dec_key_expanded, &ctx, job->dst,
                                job->src +
                                job->cipher_start_src_offset_in_bytes,
                                job->msg_len_to_cipher_in_bytes,
                                job->iv,
                                job->u.GCM.aad, job->u.GCM.aad_len_in_bytes,
                                job->auth_tag_output,
                                job->auth_tag_output_len_in_bytes);
        else
                AES_GCM_DEC_256(job->aes_dec_key_expanded, &ctx, job->dst,
                                job->src +
                                job->cipher_start_src_offset_in_bytes,
                                job->msg_len_to_cipher_in_bytes,
                                job->iv,
                                job->u.GCM.aad, job->u.GCM.aad_len_in_bytes,
                                job->auth_tag_output,
                                job->auth_tag_output_len_in_bytes);

        job->status = STS_COMPLETED;
        return job;
}

__forceinline
JOB_AES_HMAC *
SUBMIT_JOB_AES_GCM_ENC(JOB_AES_HMAC *job)
{
        DECLARE_ALIGNED(struct gcm_context_data ctx, 16);

        if (16 == job->aes_key_len_in_bytes)
                AES_GCM_ENC_128(job->aes_enc_key_expanded, &ctx, job->dst,
                                job->src +
                                job->cipher_start_src_offset_in_bytes,
                                job->msg_len_to_cipher_in_bytes, job->iv,
                                job->u.GCM.aad, job->u.GCM.aad_len_in_bytes,
                                job->auth_tag_output,
                                job->auth_tag_output_len_in_bytes);
        else if (24 == job->aes_key_len_in_bytes)
                AES_GCM_ENC_192(job->aes_enc_key_expanded, &ctx, job->dst,
                                job->src +
                                job->cipher_start_src_offset_in_bytes,
                                job->msg_len_to_cipher_in_bytes, job->iv,
                                job->u.GCM.aad, job->u.GCM.aad_len_in_bytes,
                                job->auth_tag_output,
                                job->auth_tag_output_len_in_bytes);
        else
                AES_GCM_ENC_256(job->aes_enc_key_expanded, &ctx, job->dst,
                                job->src +
                                job->cipher_start_src_offset_in_bytes,
                                job->msg_len_to_cipher_in_bytes, job->iv,
                                job->u.GCM.aad, job->u.GCM.aad_len_in_bytes,
                                job->auth_tag_output,
                                job->auth_tag_output_len_in_bytes);

        job->status = STS_COMPLETED;
        return job;
}
#endif /* !NO_GCM */

/* ========================================================================= */
/* Custom hash / cipher */
/* ========================================================================= */

__forceinline
JOB_AES_HMAC *
JOB_CUSTOM_CIPHER(JOB_AES_HMAC *job)
{
        if (!(job->status & STS_COMPLETED_AES)) {
                if (job->cipher_func(job))
                        job->status = STS_INTERNAL_ERROR;
                else
                        job->status |= STS_COMPLETED_AES;
        }
        return job;
}

__forceinline
JOB_AES_HMAC *
SUBMIT_JOB_CUSTOM_CIPHER(JOB_AES_HMAC *job)
{
        return JOB_CUSTOM_CIPHER(job);
}

__forceinline
JOB_AES_HMAC *
FLUSH_JOB_CUSTOM_CIPHER(JOB_AES_HMAC *job)
{
        return JOB_CUSTOM_CIPHER(job);
}

__forceinline
JOB_AES_HMAC *
JOB_CUSTOM_HASH(JOB_AES_HMAC *job)
{
        if (!(job->status & STS_COMPLETED_HMAC)) {
                if (job->hash_func(job))
                        job->status = STS_INTERNAL_ERROR;
                else
                        job->status |= STS_COMPLETED_HMAC;
        }
        return job;
}

__forceinline
JOB_AES_HMAC *
SUBMIT_JOB_CUSTOM_HASH(JOB_AES_HMAC *job)
{
        return JOB_CUSTOM_HASH(job);
}

__forceinline
JOB_AES_HMAC *
FLUSH_JOB_CUSTOM_HASH(JOB_AES_HMAC *job)
{
        return JOB_CUSTOM_HASH(job);
}

/* ========================================================================= */
/* DOCSIS AES (AES128 CBC + AES128 CFB) */
/* ========================================================================= */

#define AES_BLOCK_SIZE 16

/**
 * @brief Encrypts/decrypts the last partial block for DOCSIS SEC v3.1 BPI
 *
 * The last partial block is encrypted/decrypted using AES CFB128.
 * IV is always the next last ciphered block.
 *
 * @note It is assumed that length is bigger than one AES 128 block.
 *
 * @param job desriptor of performed crypto operation
 * @return It always returns value passed in \a job
 */
__forceinline
JOB_AES_HMAC *
DOCSIS_LAST_BLOCK(JOB_AES_HMAC *job)
{
        const void *iv = NULL;
        uint64_t offset = 0;
        uint64_t partial_bytes = 0;

        if (job == NULL)
                return job;

        IMB_ASSERT((job->cipher_direction == DECRYPT) ||
                   (job->status & STS_COMPLETED_AES));

        partial_bytes = job->msg_len_to_cipher_in_bytes & (AES_BLOCK_SIZE - 1);
        offset = job->msg_len_to_cipher_in_bytes & (~(AES_BLOCK_SIZE - 1));

        if (!partial_bytes)
                return job;

        /* in either case IV has to be next last ciphered block */
        if (job->cipher_direction == ENCRYPT)
                iv = job->dst + offset - AES_BLOCK_SIZE;
        else
                iv = job->src + job->cipher_start_src_offset_in_bytes +
                        offset - AES_BLOCK_SIZE;

        IMB_ASSERT(partial_bytes <= AES_BLOCK_SIZE);
        AES_CFB_128_ONE(job->dst + offset,
                        job->src + job->cipher_start_src_offset_in_bytes +
                        offset,
                        iv, job->aes_enc_key_expanded, partial_bytes);

        return job;
}

/**
 * @brief Encrypts/decrypts the first and only partial block for
 *        DOCSIS SEC v3.1 BPI
 *
 * The first partial block is encrypted/decrypted using AES CFB128.
 *
 * @param job desriptor of performed crypto operation
 * @return It always returns value passed in \a job
 */
__forceinline
JOB_AES_HMAC *
DOCSIS_FIRST_BLOCK(JOB_AES_HMAC *job)
{
        IMB_ASSERT(!(job->status & STS_COMPLETED_AES));
        IMB_ASSERT(job->msg_len_to_cipher_in_bytes <= AES_BLOCK_SIZE);
        AES_CFB_128_ONE(job->dst,
                        job->src + job->cipher_start_src_offset_in_bytes,
                        job->iv, job->aes_enc_key_expanded,
                        job->msg_len_to_cipher_in_bytes);
        job->status |= STS_COMPLETED_AES;
        return job;
}

/* ========================================================================= */
/* DES, 3DES and DOCSIS DES (DES CBC + DES CFB) */
/* ========================================================================= */

/**
 * @brief DOCSIS DES cipher encryption
 *
 * @param job desriptor of performed crypto operation
 * @return It always returns value passed in \a job
 */
__forceinline
JOB_AES_HMAC *
DOCSIS_DES_ENC(JOB_AES_HMAC *job)
{
        IMB_ASSERT(!(job->status & STS_COMPLETED_AES));
        docsis_des_enc_basic(job->src + job->cipher_start_src_offset_in_bytes,
                             job->dst,
                             (int) job->msg_len_to_cipher_in_bytes,
                             job->aes_enc_key_expanded,
                             (const uint64_t *)job->iv);
        job->status |= STS_COMPLETED_AES;
        return job;
}

/**
 * @brief DOCSIS DES cipher decryption
 *
 * @param job desriptor of performed crypto operation
 * @return It always returns value passed in \a job
 */
__forceinline
JOB_AES_HMAC *
DOCSIS_DES_DEC(JOB_AES_HMAC *job)
{
        IMB_ASSERT(!(job->status & STS_COMPLETED_AES));
        docsis_des_dec_basic(job->src + job->cipher_start_src_offset_in_bytes,
                             job->dst,
                             (int) job->msg_len_to_cipher_in_bytes,
                             job->aes_dec_key_expanded,
                             (const uint64_t *)job->iv);
        job->status |= STS_COMPLETED_AES;
        return job;
}

/**
 * @brief DES cipher encryption
 *
 * @param job desriptor of performed crypto operation
 * @return It always returns value passed in \a job
 */
__forceinline
JOB_AES_HMAC *
DES_CBC_ENC(JOB_AES_HMAC *job)
{
        IMB_ASSERT(!(job->status & STS_COMPLETED_AES));
        des_enc_cbc_basic(job->src + job->cipher_start_src_offset_in_bytes,
                          job->dst,
                          job->msg_len_to_cipher_in_bytes &
                          (~(DES_BLOCK_SIZE - 1)),
                          job->aes_enc_key_expanded, (const uint64_t *)job->iv);
        job->status |= STS_COMPLETED_AES;
        return job;
}

/**
 * @brief DES cipher decryption
 *
 * @param job desriptor of performed crypto operation
 * @return It always returns value passed in \a job
 */
__forceinline
JOB_AES_HMAC *
DES_CBC_DEC(JOB_AES_HMAC *job)
{
        IMB_ASSERT(!(job->status & STS_COMPLETED_AES));
        des_dec_cbc_basic(job->src + job->cipher_start_src_offset_in_bytes,
                          job->dst,
                          job->msg_len_to_cipher_in_bytes &
                          (~(DES_BLOCK_SIZE - 1)),
                          job->aes_dec_key_expanded, (const uint64_t *)job->iv);
        job->status |= STS_COMPLETED_AES;
        return job;
}

/**
 * @brief 3DES cipher encryption
 *
 * @param job desriptor of performed crypto operation
 * @return It always returns value passed in \a job
 */
__forceinline
JOB_AES_HMAC *
DES3_CBC_ENC(JOB_AES_HMAC *job)
{
        const void * const *ks_ptr =
                (const void * const *)job->aes_enc_key_expanded;

        IMB_ASSERT(!(job->status & STS_COMPLETED_AES));
        des3_enc_cbc_basic(job->src + job->cipher_start_src_offset_in_bytes,
                           job->dst,
                           job->msg_len_to_cipher_in_bytes &
                           (~(DES_BLOCK_SIZE - 1)),
                           ks_ptr[0], ks_ptr[1], ks_ptr[2],
                           (const uint64_t *)job->iv);
        job->status |= STS_COMPLETED_AES;
        return job;
}

/**
 * @brief 3DES cipher decryption
 *
 * @param job desriptor of performed crypto operation
 * @return It always returns value passed in \a job
 */
__forceinline
JOB_AES_HMAC *
DES3_CBC_DEC(JOB_AES_HMAC *job)
{
        const void * const *ks_ptr =
                (const void * const *)job->aes_dec_key_expanded;

        IMB_ASSERT(!(job->status & STS_COMPLETED_AES));
        des3_dec_cbc_basic(job->src + job->cipher_start_src_offset_in_bytes,
                           job->dst,
                           job->msg_len_to_cipher_in_bytes &
                           (~(DES_BLOCK_SIZE - 1)),
                           ks_ptr[0], ks_ptr[1], ks_ptr[2],
                           (const uint64_t *)job->iv);
        job->status |= STS_COMPLETED_AES;
        return job;
}

/* ========================================================================= */
/* Cipher submit & flush functions */
/* ========================================================================= */

__forceinline
JOB_AES_HMAC *
SUBMIT_JOB_AES_ENC(MB_MGR *state, JOB_AES_HMAC *job)
{
        if (CBC == job->cipher_mode) {
                if (16 == job->aes_key_len_in_bytes) {
                        return SUBMIT_JOB_AES128_ENC(&state->aes128_ooo, job);
                } else if (24 == job->aes_key_len_in_bytes) {
                        return SUBMIT_JOB_AES192_ENC(&state->aes192_ooo, job);
                } else { /* assume 32 */
                        return SUBMIT_JOB_AES256_ENC(&state->aes256_ooo, job);
                }
        } else if (CNTR == job->cipher_mode) {
                if (16 == job->aes_key_len_in_bytes) {
                        return SUBMIT_JOB_AES128_CNTR(job);
                } else if (24 == job->aes_key_len_in_bytes) {
                        return SUBMIT_JOB_AES192_CNTR(job);
                } else { /* assume 32 */
                        return SUBMIT_JOB_AES256_CNTR(job);
                }
        } else if (DOCSIS_SEC_BPI == job->cipher_mode) {
                if (job->msg_len_to_cipher_in_bytes >= AES_BLOCK_SIZE) {
                        JOB_AES_HMAC *tmp;

                        tmp = SUBMIT_JOB_AES128_ENC(&state->docsis_sec_ooo,
                                                    job);
                        return DOCSIS_LAST_BLOCK(tmp);
                } else
                        return DOCSIS_FIRST_BLOCK(job);
#ifndef NO_GCM
        } else if (GCM == job->cipher_mode) {
                return SUBMIT_JOB_AES_GCM_ENC(job);
#endif /* NO_GCM */
        } else if (CUSTOM_CIPHER == job->cipher_mode) {
                return SUBMIT_JOB_CUSTOM_CIPHER(job);
        } else if (DES == job->cipher_mode) {
#ifdef SUBMIT_JOB_DES_CBC_ENC
                return SUBMIT_JOB_DES_CBC_ENC(&state->des_enc_ooo, job);
#else
                return DES_CBC_ENC(job);
#endif /* SUBMIT_JOB_DES_CBC_ENC */
        } else if (DOCSIS_DES == job->cipher_mode) {
#ifdef SUBMIT_JOB_DOCSIS_DES_ENC
                return SUBMIT_JOB_DOCSIS_DES_ENC(&state->docsis_des_enc_ooo,
                                                 job);
#else
                return DOCSIS_DES_ENC(job);
#endif /* SUBMIT_JOB_DOCSIS_DES_ENC */
        } else if (DES3 == job->cipher_mode) {
#ifdef SUBMIT_JOB_3DES_CBC_ENC
                return SUBMIT_JOB_3DES_CBC_ENC(&state->des3_enc_ooo, job);
#else
                return DES3_CBC_ENC(job);
#endif
        } else { /* assume NUL_CIPHER or CCM */
                job->status |= STS_COMPLETED_AES;
                return job;
        }
}

__forceinline
JOB_AES_HMAC *
FLUSH_JOB_AES_ENC(MB_MGR *state, JOB_AES_HMAC *job)
{
        if (CBC == job->cipher_mode) {
                if (16 == job->aes_key_len_in_bytes) {
                        return FLUSH_JOB_AES128_ENC(&state->aes128_ooo);
                } else if (24 == job->aes_key_len_in_bytes) {
                        return FLUSH_JOB_AES192_ENC(&state->aes192_ooo);
                } else  { /* assume 32 */
                        return FLUSH_JOB_AES256_ENC(&state->aes256_ooo);
                }
        } else if (DOCSIS_SEC_BPI == job->cipher_mode) {
                JOB_AES_HMAC *tmp;

                tmp = FLUSH_JOB_AES128_ENC(&state->docsis_sec_ooo);
                return DOCSIS_LAST_BLOCK(tmp);
#ifdef FLUSH_JOB_DES_CBC_ENC
        } else if (DES == job->cipher_mode) {
                return FLUSH_JOB_DES_CBC_ENC(&state->des_enc_ooo);
#endif /* FLUSH_JOB_DES_CBC_ENC */
#ifdef FLUSH_JOB_3DES_CBC_ENC
        } else if (DES3 == job->cipher_mode) {
                return FLUSH_JOB_3DES_CBC_ENC(&state->des3_enc_ooo);
#endif /* FLUSH_JOB_3DES_CBC_ENC */
#ifdef FLUSH_JOB_DOCSIS_DES_ENC
        } else if (DOCSIS_DES == job->cipher_mode) {
                return FLUSH_JOB_DOCSIS_DES_ENC(&state->docsis_des_enc_ooo);
#endif /* FLUSH_JOB_DOCSIS_DES_ENC */
        } else if (CUSTOM_CIPHER == job->cipher_mode) {
                return FLUSH_JOB_CUSTOM_CIPHER(job);
        } else { /* assume CNTR, CCM or NULL_CIPHER */
                return NULL;
        }
}

__forceinline
JOB_AES_HMAC *
SUBMIT_JOB_AES_DEC(MB_MGR *state, JOB_AES_HMAC *job)
{
        if (CBC == job->cipher_mode) {
                if (16 == job->aes_key_len_in_bytes) {
                        return SUBMIT_JOB_AES128_DEC(job);
                } else if (24 == job->aes_key_len_in_bytes) {
                        return SUBMIT_JOB_AES192_DEC(job);
                } else { /* assume 32 */
                        return SUBMIT_JOB_AES256_DEC(job);
                }
        } else if (CNTR == job->cipher_mode) {
                if (16 == job->aes_key_len_in_bytes) {
                        return SUBMIT_JOB_AES128_CNTR(job);
                } else if (24 == job->aes_key_len_in_bytes) {
                        return SUBMIT_JOB_AES192_CNTR(job);
                } else { /* assume 32 */
                        return SUBMIT_JOB_AES256_CNTR(job);
                }
        } else if (DOCSIS_SEC_BPI == job->cipher_mode) {
                if (job->msg_len_to_cipher_in_bytes >= AES_BLOCK_SIZE) {
                        DOCSIS_LAST_BLOCK(job);
                        return SUBMIT_JOB_AES128_DEC(job);
                } else {
                        return DOCSIS_FIRST_BLOCK(job);
                }
#ifndef NO_GCM
        } else if (GCM == job->cipher_mode) {
                return SUBMIT_JOB_AES_GCM_DEC(job);
#endif /* NO_GCM */
        } else if (DES == job->cipher_mode) {
#ifdef SUBMIT_JOB_DES_CBC_DEC
                return SUBMIT_JOB_DES_CBC_DEC(&state->des_dec_ooo, job);
#else
                (void) state;
                return DES_CBC_DEC(job);
#endif /* SUBMIT_JOB_DES_CBC_DEC */
        } else if (DOCSIS_DES == job->cipher_mode) {
#ifdef SUBMIT_JOB_DOCSIS_DES_DEC
                return SUBMIT_JOB_DOCSIS_DES_DEC(&state->docsis_des_dec_ooo,
                                                 job);
#else
                return DOCSIS_DES_DEC(job);
#endif /* SUBMIT_JOB_DOCSIS_DES_DEC */
        } else if (DES3 == job->cipher_mode) {
#ifdef SUBMIT_JOB_3DES_CBC_DEC
                return SUBMIT_JOB_3DES_CBC_DEC(&state->des3_dec_ooo, job);
#else
                return DES3_CBC_DEC(job);
#endif
        } else if (CUSTOM_CIPHER == job->cipher_mode) {
                return SUBMIT_JOB_CUSTOM_CIPHER(job);
        } else {
                /* assume NULL cipher or CCM */
                job->status |= STS_COMPLETED_AES;
                return job;
        }
}

__forceinline
JOB_AES_HMAC *
FLUSH_JOB_AES_DEC(MB_MGR *state, JOB_AES_HMAC *job)
{
#ifdef FLUSH_JOB_DES_CBC_DEC
        if (DES == job->cipher_mode)
                return FLUSH_JOB_DES_CBC_DEC(&state->des_dec_ooo);
#endif /* FLUSH_JOB_DES_CBC_DEC */
#ifdef FLUSH_JOB_3DES_CBC_DEC
        if (DES3 == job->cipher_mode)
                return FLUSH_JOB_3DES_CBC_DEC(&state->des3_dec_ooo);
#endif /* FLUSH_JOB_3DES_CBC_DEC */
#ifdef FLUSH_JOB_DOCSIS_DES_DEC
        if (DOCSIS_DES == job->cipher_mode)
                return FLUSH_JOB_DOCSIS_DES_DEC(&state->docsis_des_dec_ooo);
#endif /* FLUSH_JOB_DOCSIS_DES_DEC */
        (void) state;
        return SUBMIT_JOB_AES_DEC(state, job);
}

/* ========================================================================= */
/* Hash submit & flush functions */
/* ========================================================================= */

__forceinline
JOB_AES_HMAC *
SUBMIT_JOB_HASH(MB_MGR *state, JOB_AES_HMAC *job)
{
#ifdef VERBOSE
        printf("--------Enter SUBMIT_JOB_HASH --------------\n");
#endif
        switch (job->hash_alg) {
        case SHA1:
#ifdef HASH_USE_SHAEXT
                if (state->features & IMB_FEATURE_SHANI)
                        return SUBMIT_JOB_HMAC_NI(&state->hmac_sha_1_ooo, job);
#endif
                return SUBMIT_JOB_HMAC(&state->hmac_sha_1_ooo, job);
        case SHA_224:
#ifdef HASH_USE_SHAEXT
                if (state->features & IMB_FEATURE_SHANI)
                        return SUBMIT_JOB_HMAC_SHA_224_NI
                                (&state->hmac_sha_224_ooo, job);
#endif
                return SUBMIT_JOB_HMAC_SHA_224(&state->hmac_sha_224_ooo, job);
        case SHA_256:
#ifdef HASH_USE_SHAEXT
                if (state->features & IMB_FEATURE_SHANI)
                        return SUBMIT_JOB_HMAC_SHA_256_NI
                                (&state->hmac_sha_256_ooo, job);
#endif
                return SUBMIT_JOB_HMAC_SHA_256(&state->hmac_sha_256_ooo, job);
        case SHA_384:
                return SUBMIT_JOB_HMAC_SHA_384(&state->hmac_sha_384_ooo, job);
        case SHA_512:
                return SUBMIT_JOB_HMAC_SHA_512(&state->hmac_sha_512_ooo, job);
        case AES_XCBC:
                return SUBMIT_JOB_AES_XCBC(&state->aes_xcbc_ooo, job);
        case MD5:
                return SUBMIT_JOB_HMAC_MD5(&state->hmac_md5_ooo, job);
        case CUSTOM_HASH:
                return SUBMIT_JOB_CUSTOM_HASH(job);
        case AES_CCM:
                return SUBMIT_JOB_AES_CCM_AUTH(&state->aes_ccm_ooo, job);
        case AES_CMAC:
                return SUBMIT_JOB_AES_CMAC_AUTH(&state->aes_cmac_ooo, job);
        default: /* assume NULL_HASH */
                job->status |= STS_COMPLETED_HMAC;
                return job;
        }
}

__forceinline
JOB_AES_HMAC *
FLUSH_JOB_HASH(MB_MGR *state, JOB_AES_HMAC *job)
{
        switch (job->hash_alg) {
        case SHA1:
#ifdef HASH_USE_SHAEXT
                if (state->features & IMB_FEATURE_SHANI)
                        return FLUSH_JOB_HMAC_NI(&state->hmac_sha_1_ooo);
#endif
                return FLUSH_JOB_HMAC(&state->hmac_sha_1_ooo);
        case SHA_224:
#ifdef HASH_USE_SHAEXT
                if (state->features & IMB_FEATURE_SHANI)
                        return FLUSH_JOB_HMAC_SHA_224_NI
                                (&state->hmac_sha_224_ooo);
#endif
                return FLUSH_JOB_HMAC_SHA_224(&state->hmac_sha_224_ooo);
        case SHA_256:
#ifdef HASH_USE_SHAEXT
                if (state->features & IMB_FEATURE_SHANI)
                        return FLUSH_JOB_HMAC_SHA_256_NI
                                (&state->hmac_sha_256_ooo);
#endif
                return FLUSH_JOB_HMAC_SHA_256(&state->hmac_sha_256_ooo);
        case SHA_384:
                return FLUSH_JOB_HMAC_SHA_384(&state->hmac_sha_384_ooo);
        case SHA_512:
                return FLUSH_JOB_HMAC_SHA_512(&state->hmac_sha_512_ooo);
        case AES_XCBC:
                return FLUSH_JOB_AES_XCBC(&state->aes_xcbc_ooo);
        case MD5:
                return FLUSH_JOB_HMAC_MD5(&state->hmac_md5_ooo);
        case CUSTOM_HASH:
                return FLUSH_JOB_CUSTOM_HASH(job);
        case AES_CCM:
                return FLUSH_JOB_AES_CCM_AUTH(&state->aes_ccm_ooo);
        case AES_CMAC:
                return FLUSH_JOB_AES_CMAC_AUTH(&state->aes_cmac_ooo);
        default: /* assume NULL_HASH */
                if (!(job->status & STS_COMPLETED_HMAC)) {
                        job->status |= STS_COMPLETED_HMAC;
                        return job;
                }
                /* if HMAC is complete then return NULL */
                return NULL;
        }
}


/* ========================================================================= */
/* Job submit & flush functions */
/* ========================================================================= */

#ifdef DEBUG
#ifdef _WIN32
#define INVALID_PRN(_fmt, ...)                                          \
        fprintf(stderr, "%s():%d: " _fmt, __FUNCTION__, __LINE__, __VA_ARGS__)
#else
#define INVALID_PRN(_fmt, ...)                                          \
        fprintf(stderr, "%s():%d: " _fmt, __func__, __LINE__, __VA_ARGS__)
#endif
#else
#define INVALID_PRN(_fmt, ...)
#endif

__forceinline int
is_job_invalid(const JOB_AES_HMAC *job)
{
        const uint64_t auth_tag_len_max[] = {
                0,  /* INVALID selection */
                12, /* SHA1 */
                14, /* SHA_224 */
                16, /* SHA_256 */
                24, /* SHA_384 */
                32, /* SHA_512 */
                12, /* AES_XCBC */
                12, /* MD5 */
                0,  /* NULL_HASH */
                16, /* AES_GMAC */
                0,  /* CUSTOM HASH */
                0,  /* AES_CCM */
                16, /* AES_CMAC */
        };

        switch (job->cipher_mode) {
        case CBC:
                if (job->cipher_direction == ENCRYPT &&
                    job->aes_enc_key_expanded == NULL) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                if (job->cipher_direction == DECRYPT &&
                    job->aes_dec_key_expanded == NULL) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                if (job->aes_key_len_in_bytes != UINT64_C(16) &&
                    job->aes_key_len_in_bytes != UINT64_C(24) &&
                    job->aes_key_len_in_bytes != UINT64_C(32)) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                if (job->msg_len_to_cipher_in_bytes == 0) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                if (job->msg_len_to_cipher_in_bytes & UINT64_C(15)) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                if (job->iv_len_in_bytes != UINT64_C(16)) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                break;
        case CNTR:
                if (job->aes_enc_key_expanded == NULL) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                if (job->aes_key_len_in_bytes != UINT64_C(16) &&
                    job->aes_key_len_in_bytes != UINT64_C(24) &&
                    job->aes_key_len_in_bytes != UINT64_C(32)) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                if (job->iv_len_in_bytes != UINT64_C(16) &&
                    job->iv_len_in_bytes != UINT64_C(12)) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                if (job->msg_len_to_cipher_in_bytes == 0) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                break;
        case NULL_CIPHER:
                /* NULL_CIPHER only allowed in HASH_CIPHER */
                if (job->chain_order != HASH_CIPHER)
                        return 1;
                /* XXX: not copy src to dst */
                break;
        case DOCSIS_SEC_BPI:
                if (job->aes_enc_key_expanded == NULL) {
                        /* it has to be set regardless of direction (AES-CFB) */
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                if (job->cipher_direction == DECRYPT &&
                    job->aes_dec_key_expanded == NULL) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                if (job->aes_key_len_in_bytes != UINT64_C(16)) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                if (job->iv_len_in_bytes != UINT64_C(16)) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                if (job->msg_len_to_cipher_in_bytes == 0) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                break;
#ifndef NO_GCM
        case GCM:
                /* Same key structure used for encrypt and decrypt */
                if (job->cipher_direction == ENCRYPT &&
                    job->aes_enc_key_expanded == NULL) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                if (job->cipher_direction == DECRYPT &&
                    job->aes_dec_key_expanded == NULL) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                if (job->aes_key_len_in_bytes != UINT64_C(16) &&
                    job->aes_key_len_in_bytes != UINT64_C(24) &&
                    job->aes_key_len_in_bytes != UINT64_C(32)) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                if (job->iv_len_in_bytes != UINT64_C(12)) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                if (job->hash_alg != AES_GMAC) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                if (job->msg_len_to_cipher_in_bytes == 0) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                break;
#endif /* !NO_GCM */
        case CUSTOM_CIPHER:
                /* no checks here */
                if (job->cipher_func == NULL) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                break;
        case DES:
                if (job->cipher_direction == ENCRYPT &&
                    job->aes_enc_key_expanded == NULL) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                if (job->cipher_direction == DECRYPT &&
                    job->aes_dec_key_expanded == NULL) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                if (job->aes_key_len_in_bytes != UINT64_C(8)) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                if (job->msg_len_to_cipher_in_bytes == 0) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                if (job->msg_len_to_cipher_in_bytes & UINT64_C(7)) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                if (job->iv_len_in_bytes != UINT64_C(8)) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                break;
        case DOCSIS_DES:
                if (job->cipher_direction == ENCRYPT &&
                    job->aes_enc_key_expanded == NULL) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                if (job->cipher_direction == DECRYPT &&
                    job->aes_dec_key_expanded == NULL) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                if (job->aes_key_len_in_bytes != UINT64_C(8)) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                if (job->msg_len_to_cipher_in_bytes == 0) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                if (job->iv_len_in_bytes != UINT64_C(8)) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                break;
        case CCM:
                if (job->aes_enc_key_expanded == NULL) {
                        /* AES-CTR and CBC-MAC use only encryption keys */
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                /* currently only AES-CCM-128 is only supported */
                if (job->aes_key_len_in_bytes != UINT64_C(16)) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                /*
                 * From RFC3610:
                 *     Nonce length = 15 - L
                 *     Valid L values are: 2 to 8
                 * Then valid nonce lengths 13 to 7 (inclusive).
                 */
                if (job->iv_len_in_bytes > UINT64_C(13) ||
                    job->iv_len_in_bytes < UINT64_C(7)) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                if (job->msg_len_to_cipher_in_bytes == 0) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                if (job->hash_alg != AES_CCM) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                break;
        case DES3:
                if (job->aes_key_len_in_bytes != UINT64_C(8)) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                if (job->msg_len_to_cipher_in_bytes == 0) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                if (job->msg_len_to_cipher_in_bytes & UINT64_C(7)) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                if (job->iv_len_in_bytes != UINT64_C(8)) {
                        INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                        return 1;
                }
                if (job->cipher_direction == ENCRYPT) {
                        const void * const *ks_ptr =
                                (const void * const *)job->aes_enc_key_expanded;

                        if (ks_ptr == NULL) {
                                INVALID_PRN("cipher_mode:%d\n",
                                            job->cipher_mode);
                                return 1;
                        }
                        if (ks_ptr[0] == NULL || ks_ptr[1] == NULL ||
                            ks_ptr[2] == NULL) {
                                INVALID_PRN("cipher_mode:%d\n",
                                            job->cipher_mode);
                                return 1;
                        }
                } else {
                        const void * const *ks_ptr =
                                (const void * const *)job->aes_dec_key_expanded;

                        if (ks_ptr == NULL) {
                                INVALID_PRN("cipher_mode:%d\n",
                                            job->cipher_mode);
                                return 1;
                        }
                        if (ks_ptr[0] == NULL || ks_ptr[1] == NULL ||
                            ks_ptr[2] == NULL) {
                                INVALID_PRN("cipher_mode:%d\n",
                                            job->cipher_mode);
                                return 1;
                        }
                }
                break;
        default:
                INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
                return 1;
        }

        switch (job->hash_alg) {
        case SHA1:
        case AES_XCBC:
        case MD5:
        case SHA_224:
        case SHA_256:
        case SHA_384:
        case SHA_512:
                if (job->auth_tag_output_len_in_bytes !=
                    auth_tag_len_max[job->hash_alg]) {
                        INVALID_PRN("hash_alg:%d\n", job->hash_alg);
                        return 1;
                }
                if (job->msg_len_to_hash_in_bytes == 0) {
                        INVALID_PRN("hash_alg:%d\n", job->hash_alg);
                        return 1;
                }
                if (job->auth_tag_output == NULL) {
                        INVALID_PRN("hash_alg:%d\n", job->hash_alg);
                        return 1;
                }
                break;
        case NULL_HASH:
                break;
#ifndef NO_GCM
        case AES_GMAC:
                if (job->auth_tag_output_len_in_bytes != UINT64_C(8) &&
                    job->auth_tag_output_len_in_bytes != UINT64_C(12) &&
                    job->auth_tag_output_len_in_bytes != UINT64_C(16)) {
                        INVALID_PRN("hash_alg:%d\n", job->hash_alg);
                                return 1;
                }
                if (job->cipher_mode != GCM) {
                        INVALID_PRN("hash_alg:%d\n", job->hash_alg);
                                return 1;
                }
                if (job->auth_tag_output == NULL) {
                        INVALID_PRN("hash_alg:%d\n", job->hash_alg);
                        return 1;
                }
                /*
                 * msg_len_to_hash_in_bytes not checked against zero.
                 * It is not used for AES-GCM & GMAC - see
                 * SUBMIT_JOB_AES_GCM_ENC and SUBMIT_JOB_AES_GCM_DEC functions.
                 */
                break;
#endif /* !NO_GCM */
        case CUSTOM_HASH:
                if (job->hash_func == NULL) {
                        INVALID_PRN("hash_alg:%d\n", job->hash_alg);
                        return 1;
                }
                break;
        case AES_CCM:
                if (job->u.CCM.aad_len_in_bytes > 46) {
                        /* 3 x AES_BLOCK - 2 bytes for AAD len */
                        INVALID_PRN("hash_alg:%d\n", job->hash_alg);
                        return 1;
                }
                if ((job->u.CCM.aad_len_in_bytes > 0) &&
                    (job->u.CCM.aad == NULL)) {
                        INVALID_PRN("hash_alg:%d\n", job->hash_alg);
                        return 1;
                }
                /* M can be any even number from 4 to 16 */
                if (job->auth_tag_output_len_in_bytes < UINT64_C(4) ||
                    job->auth_tag_output_len_in_bytes > UINT64_C(16) ||
                    ((job->auth_tag_output_len_in_bytes & 1) != 0)) {
                        INVALID_PRN("hash_alg:%d\n", job->hash_alg);
                                return 1;
                }
                if (job->cipher_mode != CCM) {
                        INVALID_PRN("hash_alg:%d\n", job->hash_alg);
                        return 1;
                }
                /*
                 * AES-CCM allows for only one message for
                 * cipher and uthentication.
                 * AAD can be used to extend authentication over
                 * clear text fields.
                 */
                if (job->msg_len_to_cipher_in_bytes !=
                    job->msg_len_to_hash_in_bytes) {
                        INVALID_PRN("hash_alg:%d\n", job->hash_alg);
                        return 1;
                }
                if (job->cipher_start_src_offset_in_bytes !=
                    job->hash_start_src_offset_in_bytes) {
                        INVALID_PRN("hash_alg:%d\n", job->hash_alg);
                        return 1;
                }
                break;
        case AES_CMAC:
                if ((job->u.CMAC._key_expanded == NULL) ||
                    (job->u.CMAC._skey1 == NULL) ||
                    (job->u.CMAC._skey2 == NULL)) {
                        INVALID_PRN("hash_alg:%d\n", job->hash_alg);
                        return 1;
                }
                /*
                 * T is 128 bits but 96 bits is also allowed due to
                 * IPsec use case (RFC 4494)
                 */
                if (job->auth_tag_output_len_in_bytes != UINT64_C(16) &&
                    job->auth_tag_output_len_in_bytes != UINT64_C(12)) {
                        INVALID_PRN("hash_alg:%d\n", job->hash_alg);
                        return 1;
                }
                if (job->auth_tag_output == NULL) {
                        INVALID_PRN("hash_alg:%d\n", job->hash_alg);
                        return 1;
                }
                break;
        default:
                INVALID_PRN("hash_alg:%d\n", job->hash_alg);
                return 1;
        }

        switch (job->chain_order) {
        case CIPHER_HASH:
                if (job->cipher_direction != ENCRYPT) {
                        INVALID_PRN("chain_order:%d\n", job->chain_order);
                        return 1;
                }
                break;
        case HASH_CIPHER:
                if (job->cipher_mode != NULL_CIPHER) {
                        if (job->cipher_direction != DECRYPT) {
                                INVALID_PRN("chain_order:%d\n",
                                            job->chain_order);
                                return 1;
                        }
                }
                break;
        default:
                INVALID_PRN("chain_order:%d\n", job->chain_order);
                return 1;
        }

        return 0;
}

__forceinline
JOB_AES_HMAC *submit_new_job(MB_MGR *state, JOB_AES_HMAC *job)
{
        if (job->chain_order == CIPHER_HASH) {
                /* assume job->cipher_direction == ENCRYPT */
                job = SUBMIT_JOB_AES_ENC(state, job);
                if (job) {
                        job = SUBMIT_JOB_HASH(state, job);
                        if (job && (job->chain_order == HASH_CIPHER))
                                SUBMIT_JOB_AES_DEC(state, job);
                } /* end if job */
        } else { /* job->chain_order == HASH_CIPHER */
                /* assume job->cipher_direction == DECRYPT */
                job = SUBMIT_JOB_HASH(state, job);
                if (job && (job->chain_order == HASH_CIPHER))
                        SUBMIT_JOB_AES_DEC(state, job);
        }
        return job;
}

__forceinline
void complete_job(MB_MGR *state, JOB_AES_HMAC *job)
{
        JOB_AES_HMAC *tmp = NULL;

        while (job->status < STS_COMPLETED) {
                if (job->chain_order == CIPHER_HASH) {
                        /* assume job->cipher_direction == ENCRYPT */
                        tmp = FLUSH_JOB_AES_ENC(state, job);
                        if (tmp)
                                tmp = SUBMIT_JOB_HASH(state, tmp);
                        else
                                tmp = FLUSH_JOB_HASH(state, job);
                        if (tmp && (tmp->chain_order == HASH_CIPHER))
                                SUBMIT_JOB_AES_DEC(state, tmp);
                } else { /* job->chain_order == HASH_CIPHER */
                        /* assume job->cipher_direction == DECRYPT */
                        tmp = FLUSH_JOB_HASH(state, job);
                        if (tmp == NULL)
                                tmp = FLUSH_JOB_AES_DEC(state, job);
                        else
                                if (tmp->chain_order == HASH_CIPHER)
                                        SUBMIT_JOB_AES_DEC(state, tmp);
                }
        }
}

__forceinline
JOB_AES_HMAC *
submit_job_and_check(MB_MGR *state, const int run_check)
{
        JOB_AES_HMAC *job = NULL;
#ifndef LINUX
        DECLARE_ALIGNED(uint128_t xmm_save[10], 16);

        SAVE_XMMS(xmm_save);
#endif

        job = JOBS(state, state->next_job);

        if (run_check) {
                if (is_job_invalid(job)) {
                        job->status = STS_INVALID_ARGS;
                } else {
                        job->status = STS_BEING_PROCESSED;
                        job = submit_new_job(state, job);
                }
        } else {
                job->status = STS_BEING_PROCESSED;
                job = submit_new_job(state, job);
        }

        if (state->earliest_job < 0) {
                /* state was previously empty */
                state->earliest_job = state->next_job;
                ADV_JOBS(&state->next_job);
#ifndef LINUX
                RESTORE_XMMS(xmm_save);
#endif
                return NULL;	/* if we were empty, nothing to return */
        }

        ADV_JOBS(&state->next_job);

        if (state->earliest_job == state->next_job) {
                /* Full */
                job = JOBS(state, state->earliest_job);
                complete_job(state, job);
                ADV_JOBS(&state->earliest_job);
#ifndef LINUX
                RESTORE_XMMS(xmm_save);
#endif
                return job;
        }

        /* not full */
#ifndef LINUX
        RESTORE_XMMS(xmm_save);
#endif
        job = JOBS(state, state->earliest_job);
        if (job->status < STS_COMPLETED)
                return NULL;

        ADV_JOBS(&state->earliest_job);
        return job;
}

IMB_DLL_EXPORT
JOB_AES_HMAC *
SUBMIT_JOB(MB_MGR *state)
{
        return submit_job_and_check(state, 1);
}

IMB_DLL_EXPORT
JOB_AES_HMAC *
SUBMIT_JOB_NOCHECK(MB_MGR *state)
{
        return submit_job_and_check(state, 0);
}

IMB_DLL_EXPORT
JOB_AES_HMAC *
FLUSH_JOB(MB_MGR *state)
{
        JOB_AES_HMAC *job;
#ifndef LINUX
        DECLARE_ALIGNED(uint128_t xmm_save[10], 16);
#endif

        if (state->earliest_job < 0)
                return NULL; /* empty */

#ifndef LINUX
        SAVE_XMMS(xmm_save);
#endif
        job = JOBS(state, state->earliest_job);
        complete_job(state, job);

        ADV_JOBS(&state->earliest_job);

        if (state->earliest_job == state->next_job)
                state->earliest_job = -1; /* becomes empty */

#ifndef LINUX
        RESTORE_XMMS(xmm_save);
#endif
        return job;
}

/* ========================================================================= */
/* ========================================================================= */

IMB_DLL_EXPORT
uint32_t
QUEUE_SIZE(MB_MGR *state)
{
        int a, b;

        if (state->earliest_job < 0)
                return 0;
        a = state->next_job / sizeof(JOB_AES_HMAC);
        b = state->earliest_job / sizeof(JOB_AES_HMAC);
        return ((a-b) & (MAX_JOBS-1));
}