diff options
Diffstat (limited to 'src/spdk/intel-ipsec-mb/avx512/mb_mgr_avx512.c')
| -rw-r--r-- | src/spdk/intel-ipsec-mb/avx512/mb_mgr_avx512.c | 1066 |
1 files changed, 1066 insertions, 0 deletions
diff --git a/src/spdk/intel-ipsec-mb/avx512/mb_mgr_avx512.c b/src/spdk/intel-ipsec-mb/avx512/mb_mgr_avx512.c new file mode 100644 index 000000000..bd1aaef63 --- /dev/null +++ b/src/spdk/intel-ipsec-mb/avx512/mb_mgr_avx512.c @@ -0,0 +1,1066 @@ +/******************************************************************************* + Copyright (c) 2012-2019, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#define AVX512 +#define CLEAR_SCRATCH_SIMD_REGS clear_scratch_zmms + +#include "intel-ipsec-mb.h" +#include "include/kasumi_internal.h" +#include "include/zuc_internal.h" +#include "include/snow3g.h" + +#include "save_xmms.h" +#include "asm.h" +#include "des.h" +#include "gcm.h" +#include "cpu_feature.h" +#include "noaesni.h" + +JOB_AES_HMAC *submit_job_aes128_enc_avx(MB_MGR_AES_OOO *state, + JOB_AES_HMAC *job); +JOB_AES_HMAC *flush_job_aes128_enc_avx(MB_MGR_AES_OOO *state); + +JOB_AES_HMAC *submit_job_aes192_enc_avx(MB_MGR_AES_OOO *state, + JOB_AES_HMAC *job); +JOB_AES_HMAC *flush_job_aes192_enc_avx(MB_MGR_AES_OOO *state); + +JOB_AES_HMAC *submit_job_aes256_enc_avx(MB_MGR_AES_OOO *state, + JOB_AES_HMAC *job); +JOB_AES_HMAC *flush_job_aes256_enc_avx(MB_MGR_AES_OOO *state); + +JOB_AES_HMAC *submit_job_aes_xcbc_avx(MB_MGR_AES_XCBC_OOO *state, + JOB_AES_HMAC *job); +JOB_AES_HMAC *flush_job_aes_xcbc_avx(MB_MGR_AES_XCBC_OOO *state); + +JOB_AES_HMAC *submit_job_aes128_enc_vaes_avx512(MB_MGR_AES_OOO *state, + JOB_AES_HMAC *job); + +JOB_AES_HMAC *flush_job_aes128_enc_vaes_avx512(MB_MGR_AES_OOO *state); + +JOB_AES_HMAC *submit_job_aes192_enc_vaes_avx512(MB_MGR_AES_OOO *state, + JOB_AES_HMAC *job); + +JOB_AES_HMAC *flush_job_aes192_enc_vaes_avx512(MB_MGR_AES_OOO *state); + +JOB_AES_HMAC *submit_job_aes256_enc_vaes_avx512(MB_MGR_AES_OOO *state, + JOB_AES_HMAC *job); + +JOB_AES_HMAC *flush_job_aes256_enc_vaes_avx512(MB_MGR_AES_OOO *state); + +JOB_AES_HMAC *submit_job_des_cbc_enc_avx512(MB_MGR_DES_OOO *state, + JOB_AES_HMAC *job); +JOB_AES_HMAC *flush_job_des_cbc_enc_avx512(MB_MGR_DES_OOO *state); + +JOB_AES_HMAC *submit_job_des_cbc_dec_avx512(MB_MGR_DES_OOO *state, + JOB_AES_HMAC *job); +JOB_AES_HMAC *flush_job_des_cbc_dec_avx512(MB_MGR_DES_OOO *state); + +JOB_AES_HMAC *submit_job_3des_cbc_enc_avx512(MB_MGR_DES_OOO *state, + JOB_AES_HMAC *job); +JOB_AES_HMAC *flush_job_3des_cbc_enc_avx512(MB_MGR_DES_OOO *state); + +JOB_AES_HMAC *submit_job_3des_cbc_dec_avx512(MB_MGR_DES_OOO *state, + JOB_AES_HMAC *job); +JOB_AES_HMAC *flush_job_3des_cbc_dec_avx512(MB_MGR_DES_OOO *state); + +JOB_AES_HMAC *submit_job_docsis_des_enc_avx512(MB_MGR_DES_OOO *state, + JOB_AES_HMAC *job); +JOB_AES_HMAC *flush_job_docsis_des_enc_avx512(MB_MGR_DES_OOO *state); + +JOB_AES_HMAC *submit_job_docsis_des_dec_avx512(MB_MGR_DES_OOO *state, + JOB_AES_HMAC *job); +JOB_AES_HMAC *flush_job_docsis_des_dec_avx512(MB_MGR_DES_OOO *state); + +JOB_AES_HMAC *submit_job_aes_cntr_avx(JOB_AES_HMAC *job); + +JOB_AES_HMAC *submit_job_aes_cntr_bit_avx(JOB_AES_HMAC *job); + +#define SAVE_XMMS save_xmms_avx +#define RESTORE_XMMS restore_xmms_avx + +#define SUBMIT_JOB_AES128_ENC submit_job_aes128_enc_avx512 +#define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_avx512 +#define FLUSH_JOB_AES128_ENC flush_job_aes128_enc_avx512 + +#define SUBMIT_JOB_AES192_ENC submit_job_aes192_enc_avx512 +#define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_avx512 +#define FLUSH_JOB_AES192_ENC flush_job_aes192_enc_avx512 + +#define SUBMIT_JOB_AES256_ENC submit_job_aes256_enc_avx512 +#define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_avx512 +#define FLUSH_JOB_AES256_ENC flush_job_aes256_enc_avx512 + +#define SUBMIT_JOB_AES_ECB_128_ENC submit_job_aes_ecb_128_enc_avx +#define SUBMIT_JOB_AES_ECB_128_DEC submit_job_aes_ecb_128_dec_avx +#define SUBMIT_JOB_AES_ECB_192_ENC submit_job_aes_ecb_192_enc_avx +#define SUBMIT_JOB_AES_ECB_192_DEC submit_job_aes_ecb_192_dec_avx +#define SUBMIT_JOB_AES_ECB_256_ENC submit_job_aes_ecb_256_enc_avx +#define SUBMIT_JOB_AES_ECB_256_DEC submit_job_aes_ecb_256_dec_avx + +#define SUBMIT_JOB_AES_CNTR submit_job_aes_cntr_avx512 +#define SUBMIT_JOB_AES_CNTR_BIT submit_job_aes_cntr_bit_avx512 + +#define AES_CBC_DEC_128 aes_cbc_dec_128_avx512 +#define AES_CBC_DEC_192 aes_cbc_dec_192_avx512 +#define AES_CBC_DEC_256 aes_cbc_dec_256_avx512 + +#define AES_CNTR_128 aes_cntr_128_avx +#define AES_CNTR_192 aes_cntr_192_avx +#define AES_CNTR_256 aes_cntr_256_avx + +#define AES_CNTR_CCM_128 aes_cntr_ccm_128_avx + +#define AES_ECB_ENC_128 aes_ecb_enc_128_avx +#define AES_ECB_ENC_192 aes_ecb_enc_192_avx +#define AES_ECB_ENC_256 aes_ecb_enc_256_avx +#define AES_ECB_DEC_128 aes_ecb_dec_128_avx +#define AES_ECB_DEC_192 aes_ecb_dec_192_avx +#define AES_ECB_DEC_256 aes_ecb_dec_256_avx + +#define SUBMIT_JOB_PON_ENC submit_job_pon_enc_avx +#define SUBMIT_JOB_PON_DEC submit_job_pon_dec_avx +#define SUBMIT_JOB_PON_ENC_NO_CTR submit_job_pon_enc_no_ctr_avx +#define SUBMIT_JOB_PON_DEC_NO_CTR submit_job_pon_dec_no_ctr_avx + +#define SUBMIT_JOB_AES_XCBC submit_job_aes_xcbc_avx +#define FLUSH_JOB_AES_XCBC flush_job_aes_xcbc_avx + +#define SUBMIT_JOB_DES_CBC_ENC submit_job_des_cbc_enc_avx512 +#define FLUSH_JOB_DES_CBC_ENC flush_job_des_cbc_enc_avx512 + +#define SUBMIT_JOB_DES_CBC_DEC submit_job_des_cbc_dec_avx512 +#define FLUSH_JOB_DES_CBC_DEC flush_job_des_cbc_dec_avx512 + +#define SUBMIT_JOB_3DES_CBC_ENC submit_job_3des_cbc_enc_avx512 +#define FLUSH_JOB_3DES_CBC_ENC flush_job_3des_cbc_enc_avx512 + +#define SUBMIT_JOB_3DES_CBC_DEC submit_job_3des_cbc_dec_avx512 +#define FLUSH_JOB_3DES_CBC_DEC flush_job_3des_cbc_dec_avx512 + +#define SUBMIT_JOB_DOCSIS_DES_ENC submit_job_docsis_des_enc_avx512 +#define FLUSH_JOB_DOCSIS_DES_ENC flush_job_docsis_des_enc_avx512 + +#define SUBMIT_JOB_DOCSIS_DES_DEC submit_job_docsis_des_dec_avx512 +#define FLUSH_JOB_DOCSIS_DES_DEC flush_job_docsis_des_dec_avx512 + +#define SUBMIT_JOB_AES_ENC SUBMIT_JOB_AES_ENC_AVX512 +#define FLUSH_JOB_AES_ENC FLUSH_JOB_AES_ENC_AVX512 +#define SUBMIT_JOB_AES_DEC SUBMIT_JOB_AES_DEC_AVX512 + +JOB_AES_HMAC *submit_job_hmac_avx512(MB_MGR_HMAC_SHA_1_OOO *state, + JOB_AES_HMAC *job); +JOB_AES_HMAC *flush_job_hmac_avx512(MB_MGR_HMAC_SHA_1_OOO *state); + +JOB_AES_HMAC *submit_job_hmac_sha_224_avx512(MB_MGR_HMAC_SHA_256_OOO *state, + JOB_AES_HMAC *job); +JOB_AES_HMAC *flush_job_hmac_sha_224_avx512(MB_MGR_HMAC_SHA_256_OOO *state); + +JOB_AES_HMAC *submit_job_hmac_sha_256_avx512(MB_MGR_HMAC_SHA_256_OOO *state, + JOB_AES_HMAC *job); +JOB_AES_HMAC *flush_job_hmac_sha_256_avx512(MB_MGR_HMAC_SHA_256_OOO *state); + +JOB_AES_HMAC *submit_job_hmac_sha_384_avx512(MB_MGR_HMAC_SHA_512_OOO *state, + JOB_AES_HMAC *job); +JOB_AES_HMAC *flush_job_hmac_sha_384_avx512(MB_MGR_HMAC_SHA_512_OOO *state); + +JOB_AES_HMAC *submit_job_hmac_sha_512_avx512(MB_MGR_HMAC_SHA_512_OOO *state, + JOB_AES_HMAC *job); +JOB_AES_HMAC *flush_job_hmac_sha_512_avx512(MB_MGR_HMAC_SHA_512_OOO *state); + +JOB_AES_HMAC *submit_job_hmac_md5_avx2(MB_MGR_HMAC_MD5_OOO *state, + JOB_AES_HMAC *job); +JOB_AES_HMAC *flush_job_hmac_md5_avx2(MB_MGR_HMAC_MD5_OOO *state); + +JOB_AES_HMAC *submit_job_aes_cmac_auth_avx(MB_MGR_CMAC_OOO *state, + JOB_AES_HMAC *job); + +JOB_AES_HMAC *flush_job_aes_cmac_auth_avx(MB_MGR_CMAC_OOO *state); + +JOB_AES_HMAC *submit_job_aes_ccm_auth_avx(MB_MGR_CCM_OOO *state, + JOB_AES_HMAC *job); + +JOB_AES_HMAC *flush_job_aes_ccm_auth_avx(MB_MGR_CCM_OOO *state); + +#define SUBMIT_JOB_HMAC submit_job_hmac_avx512 +#define FLUSH_JOB_HMAC flush_job_hmac_avx512 +#define SUBMIT_JOB_HMAC_SHA_224 submit_job_hmac_sha_224_avx512 +#define FLUSH_JOB_HMAC_SHA_224 flush_job_hmac_sha_224_avx512 +#define SUBMIT_JOB_HMAC_SHA_256 submit_job_hmac_sha_256_avx512 +#define FLUSH_JOB_HMAC_SHA_256 flush_job_hmac_sha_256_avx512 +#define SUBMIT_JOB_HMAC_SHA_384 submit_job_hmac_sha_384_avx512 +#define FLUSH_JOB_HMAC_SHA_384 flush_job_hmac_sha_384_avx512 +#define SUBMIT_JOB_HMAC_SHA_512 submit_job_hmac_sha_512_avx512 +#define FLUSH_JOB_HMAC_SHA_512 flush_job_hmac_sha_512_avx512 +#define SUBMIT_JOB_HMAC_MD5 submit_job_hmac_md5_avx2 +#define FLUSH_JOB_HMAC_MD5 flush_job_hmac_md5_avx2 + +#ifndef NO_GCM +#define AES_GCM_DEC_128 aes_gcm_dec_128_avx512 +#define AES_GCM_ENC_128 aes_gcm_enc_128_avx512 +#define AES_GCM_DEC_192 aes_gcm_dec_192_avx512 +#define AES_GCM_ENC_192 aes_gcm_enc_192_avx512 +#define AES_GCM_DEC_256 aes_gcm_dec_256_avx512 +#define AES_GCM_ENC_256 aes_gcm_enc_256_avx512 + +#define AES_GCM_DEC_128_VAES aes_gcm_dec_128_vaes_avx512 +#define AES_GCM_ENC_128_VAES aes_gcm_enc_128_vaes_avx512 +#define AES_GCM_DEC_192_VAES aes_gcm_dec_192_vaes_avx512 +#define AES_GCM_ENC_192_VAES aes_gcm_enc_192_vaes_avx512 +#define AES_GCM_DEC_256_VAES aes_gcm_dec_256_vaes_avx512 +#define AES_GCM_ENC_256_VAES aes_gcm_enc_256_vaes_avx512 + +#define SUBMIT_JOB_AES_GCM_DEC submit_job_aes_gcm_dec_avx512 +#define FLUSH_JOB_AES_GCM_DEC flush_job_aes_gcm_avx512 +#define SUBMIT_JOB_AES_GCM_ENC submit_job_aes_gcm_enc_avx512 +#define FLUSH_JOB_AES_GCM_ENC flush_job_aes_gcm_avx512 +#endif /* NO_GCM */ + +/* ====================================================================== */ + +#define SUBMIT_JOB submit_job_avx512 +#define FLUSH_JOB flush_job_avx512 +#define QUEUE_SIZE queue_size_avx512 +#define SUBMIT_JOB_NOCHECK submit_job_nocheck_avx512 +#define GET_NEXT_JOB get_next_job_avx512 +#define GET_COMPLETED_JOB get_completed_job_avx512 + +/* ====================================================================== */ + +#define SUBMIT_JOB_HASH SUBMIT_JOB_HASH_AVX512 +#define FLUSH_JOB_HASH FLUSH_JOB_HASH_AVX512 + +/* ====================================================================== */ + +#define AES_CFB_128_ONE aes_cfb_128_one_avx512 + +void aes128_cbc_mac_x8(AES_ARGS *args, uint64_t len); + +#define AES128_CBC_MAC aes128_cbc_mac_x8 + +#define FLUSH_JOB_AES_CCM_AUTH flush_job_aes_ccm_auth_avx +#define SUBMIT_JOB_AES_CCM_AUTH submit_job_aes_ccm_auth_avx + +#define FLUSH_JOB_AES_CMAC_AUTH flush_job_aes_cmac_auth_avx +#define SUBMIT_JOB_AES_CMAC_AUTH submit_job_aes_cmac_auth_avx + +/* ====================================================================== */ + +/* + * GCM submit / flush API for AVX512 arch + */ +#ifndef NO_GCM +static JOB_AES_HMAC * +plain_submit_gcm_dec_avx512(MB_MGR *state, JOB_AES_HMAC *job) +{ + DECLARE_ALIGNED(struct gcm_context_data ctx, 16); + (void) state; + + if (16 == job->aes_key_len_in_bytes) + AES_GCM_DEC_128(job->aes_dec_key_expanded, &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, + job->u.GCM.aad, job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + else if (24 == job->aes_key_len_in_bytes) + AES_GCM_DEC_192(job->aes_dec_key_expanded, &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, + job->u.GCM.aad, job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + else /* assume 32 bytes */ + AES_GCM_DEC_256(job->aes_dec_key_expanded, &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, + job->u.GCM.aad, job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + + job->status = STS_COMPLETED; + return job; +} + +static JOB_AES_HMAC * +plain_submit_gcm_enc_avx512(MB_MGR *state, JOB_AES_HMAC *job) +{ + DECLARE_ALIGNED(struct gcm_context_data ctx, 16); + (void) state; + + if (16 == job->aes_key_len_in_bytes) + AES_GCM_ENC_128(job->aes_enc_key_expanded, &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, job->iv, + job->u.GCM.aad, job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + else if (24 == job->aes_key_len_in_bytes) + AES_GCM_ENC_192(job->aes_enc_key_expanded, &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, job->iv, + job->u.GCM.aad, job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + else /* assume 32 bytes */ + AES_GCM_ENC_256(job->aes_enc_key_expanded, &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, job->iv, + job->u.GCM.aad, job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + + job->status = STS_COMPLETED; + return job; +} + +static JOB_AES_HMAC * +vaes_submit_gcm_dec_avx512(MB_MGR *state, JOB_AES_HMAC *job) +{ + DECLARE_ALIGNED(struct gcm_context_data ctx, 16); + (void) state; + + if (16 == job->aes_key_len_in_bytes) + AES_GCM_DEC_128_VAES(job->aes_dec_key_expanded, &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + else if (24 == job->aes_key_len_in_bytes) + AES_GCM_DEC_192_VAES(job->aes_dec_key_expanded, &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + else /* assume 32 bytes */ + AES_GCM_DEC_256_VAES(job->aes_dec_key_expanded, &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + + job->status = STS_COMPLETED; + return job; +} + +static JOB_AES_HMAC * +vaes_submit_gcm_enc_avx512(MB_MGR *state, JOB_AES_HMAC *job) +{ + DECLARE_ALIGNED(struct gcm_context_data ctx, 16); + (void) state; + + if (16 == job->aes_key_len_in_bytes) + AES_GCM_ENC_128_VAES(job->aes_enc_key_expanded, &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, job->iv, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + else if (24 == job->aes_key_len_in_bytes) + AES_GCM_ENC_192_VAES(job->aes_enc_key_expanded, &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, job->iv, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + else /* assume 32 bytes */ + AES_GCM_ENC_256_VAES(job->aes_enc_key_expanded, &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, job->iv, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + + job->status = STS_COMPLETED; + return job; +} + +static JOB_AES_HMAC * +flush_job_aes_gcm_avx512(MB_MGR *state, JOB_AES_HMAC *job) +{ + (void) state; + (void) job; + return NULL; +} + +static JOB_AES_HMAC *(*submit_job_aes_gcm_enc_avx512) + (MB_MGR *state, JOB_AES_HMAC *job) = plain_submit_gcm_enc_avx512; + +static JOB_AES_HMAC *(*submit_job_aes_gcm_dec_avx512) + (MB_MGR *state, JOB_AES_HMAC *job) = plain_submit_gcm_dec_avx512; + +#endif /* NO_GCM */ + +static JOB_AES_HMAC *(*submit_job_aes_cntr_avx512) + (JOB_AES_HMAC *job) = submit_job_aes_cntr_avx; +static JOB_AES_HMAC *(*submit_job_aes_cntr_bit_avx512) + (JOB_AES_HMAC *job) = submit_job_aes_cntr_bit_avx; + +static JOB_AES_HMAC * +vaes_submit_cntr_avx512(JOB_AES_HMAC *job) +{ + if (16 == job->aes_key_len_in_bytes) + aes_cntr_128_submit_vaes_avx512(job); + else if (24 == job->aes_key_len_in_bytes) + aes_cntr_192_submit_vaes_avx512(job); + else /* assume 32 bytes */ + aes_cntr_256_submit_vaes_avx512(job); + + job->status |= STS_COMPLETED_AES; + return job; +} + +static JOB_AES_HMAC * +vaes_submit_cntr_bit_avx512(JOB_AES_HMAC *job) +{ + if (16 == job->aes_key_len_in_bytes) + aes_cntr_bit_128_submit_vaes_avx512(job); + else if (24 == job->aes_key_len_in_bytes) + aes_cntr_bit_192_submit_vaes_avx512(job); + else /* assume 32 bytes */ + aes_cntr_bit_256_submit_vaes_avx512(job); + + job->status |= STS_COMPLETED_AES; + return job; +} + +/* ====================================================================== */ + +static JOB_AES_HMAC * +(*submit_job_aes128_enc_avx512) + (MB_MGR_AES_OOO *state, JOB_AES_HMAC *job) = submit_job_aes128_enc_avx; + +static JOB_AES_HMAC * +(*submit_job_aes192_enc_avx512) + (MB_MGR_AES_OOO *state, JOB_AES_HMAC *job) = submit_job_aes192_enc_avx; + +static JOB_AES_HMAC * +(*submit_job_aes256_enc_avx512) + (MB_MGR_AES_OOO *state, JOB_AES_HMAC *job) = submit_job_aes256_enc_avx; + +static JOB_AES_HMAC * +(*flush_job_aes128_enc_avx512) + (MB_MGR_AES_OOO *state) = flush_job_aes128_enc_avx; + +static JOB_AES_HMAC * +(*flush_job_aes192_enc_avx512) + (MB_MGR_AES_OOO *state) = flush_job_aes192_enc_avx; + +static JOB_AES_HMAC * +(*flush_job_aes256_enc_avx512) + (MB_MGR_AES_OOO *state) = flush_job_aes256_enc_avx; + +static void +(*aes_cbc_dec_128_avx512) (const void *in, const uint8_t *IV, + const void *keys, void *out, + uint64_t len_bytes) = aes_cbc_dec_128_avx; +static void +(*aes_cbc_dec_192_avx512) (const void *in, const uint8_t *IV, + const void *keys, void *out, + uint64_t len_bytes) = aes_cbc_dec_192_avx; +static void +(*aes_cbc_dec_256_avx512) (const void *in, const uint8_t *IV, + const void *keys, void *out, + uint64_t len_bytes) = aes_cbc_dec_256_avx; + +void +init_mb_mgr_avx512(MB_MGR *state) +{ + unsigned int j, vaes_support = 0; + uint8_t *p; + size_t size; + + state->features = cpu_feature_adjust(state->flags, + cpu_feature_detect()); + + if (!(state->features & IMB_FEATURE_AESNI)) { + init_mb_mgr_sse_no_aesni(state); + return; + } + if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { + vaes_support = 1; + aes_cbc_dec_128_avx512 = aes_cbc_dec_128_vaes_avx512; + aes_cbc_dec_192_avx512 = aes_cbc_dec_192_vaes_avx512; + aes_cbc_dec_256_avx512 = aes_cbc_dec_256_vaes_avx512; + submit_job_aes128_enc_avx512 = + submit_job_aes128_enc_vaes_avx512; + flush_job_aes128_enc_avx512 = + flush_job_aes128_enc_vaes_avx512; + submit_job_aes192_enc_avx512 = + submit_job_aes192_enc_vaes_avx512; + flush_job_aes192_enc_avx512 = + flush_job_aes192_enc_vaes_avx512; + submit_job_aes256_enc_avx512 = + submit_job_aes256_enc_vaes_avx512; + flush_job_aes256_enc_avx512 = + flush_job_aes256_enc_vaes_avx512; + } + + /* Init AES out-of-order fields */ + if (vaes_support) { + /* init 16 lanes */ + memset(state->aes128_ooo.lens, 0, + sizeof(state->aes128_ooo.lens)); + memset(state->aes128_ooo.job_in_lane, 0, + sizeof(state->aes128_ooo.job_in_lane)); + state->aes128_ooo.unused_lanes = 0xFEDCBA9876543210; + state->aes128_ooo.num_lanes_inuse = 0; + + memset(state->aes192_ooo.lens, 0, + sizeof(state->aes192_ooo.lens)); + memset(state->aes192_ooo.job_in_lane, 0, + sizeof(state->aes192_ooo.job_in_lane)); + state->aes192_ooo.unused_lanes = 0xFEDCBA9876543210; + state->aes192_ooo.num_lanes_inuse = 0; + + memset(state->aes256_ooo.lens, 0, + sizeof(state->aes256_ooo.lens)); + memset(state->aes256_ooo.job_in_lane, 0, + sizeof(state->aes256_ooo.job_in_lane)); + state->aes256_ooo.unused_lanes = 0xFEDCBA9876543210; + state->aes256_ooo.num_lanes_inuse = 0; + } else { + /* init 8 lanes */ + memset(state->aes128_ooo.lens, 0xFF, + sizeof(state->aes128_ooo.lens)); + memset(&state->aes128_ooo.lens[0], 0, + sizeof(state->aes128_ooo.lens[0]) * 8); + memset(state->aes128_ooo.job_in_lane, 0, + sizeof(state->aes128_ooo.job_in_lane)); + state->aes128_ooo.unused_lanes = 0xF76543210; + state->aes128_ooo.num_lanes_inuse = 0; + + memset(state->aes192_ooo.lens, 0xFF, + sizeof(state->aes192_ooo.lens)); + memset(&state->aes192_ooo.lens[0], 0, + sizeof(state->aes192_ooo.lens[0]) * 8); + memset(state->aes192_ooo.job_in_lane, 0, + sizeof(state->aes192_ooo.job_in_lane)); + state->aes192_ooo.unused_lanes = 0xF76543210; + state->aes192_ooo.num_lanes_inuse = 0; + + memset(&state->aes256_ooo.lens, 0xFF, + sizeof(state->aes256_ooo.lens)); + memset(&state->aes256_ooo.lens[0], 0, + sizeof(state->aes256_ooo.lens[0]) * 8); + memset(state->aes256_ooo.job_in_lane, 0, + sizeof(state->aes256_ooo.job_in_lane)); + state->aes256_ooo.unused_lanes = 0xF76543210; + state->aes256_ooo.num_lanes_inuse = 0; + } + + + /* DOCSIS SEC BPI (AES CBC + AES CFB for partial block) + * uses same settings as AES128 CBC. + */ + if (vaes_support) { + /* init 16 lanes */ + memset(state->docsis_sec_ooo.lens, 0, + sizeof(state->docsis_sec_ooo.lens)); + memset(state->docsis_sec_ooo.job_in_lane, 0, + sizeof(state->docsis_sec_ooo.job_in_lane)); + state->docsis_sec_ooo.unused_lanes = 0xFEDCBA9876543210; + state->docsis_sec_ooo.num_lanes_inuse = 0; + } else { + /* init 8 lanes */ + memset(state->docsis_sec_ooo.lens, 0xFF, + sizeof(state->docsis_sec_ooo.lens)); + memset(&state->docsis_sec_ooo.lens[0], 0, + sizeof(state->docsis_sec_ooo.lens[0]) * 8); + memset(state->docsis_sec_ooo.job_in_lane, 0, + sizeof(state->docsis_sec_ooo.job_in_lane)); + state->docsis_sec_ooo.unused_lanes = 0xF76543210; + state->docsis_sec_ooo.num_lanes_inuse = 0; + } + + + /* DES, 3DES and DOCSIS DES (DES CBC + DES CFB for partial block) */ + /* - separate DES OOO for encryption */ + for (j = 0; j < AVX512_NUM_DES_LANES; j++) { + state->des_enc_ooo.lens[j] = 0; + state->des_enc_ooo.job_in_lane[j] = NULL; + } + state->des_enc_ooo.unused_lanes = 0xFEDCBA9876543210; + state->des_enc_ooo.num_lanes_inuse = 0; + memset(&state->des_enc_ooo.args, 0, sizeof(state->des_enc_ooo.args)); + + /* - separate DES OOO for decryption */ + for (j = 0; j < AVX512_NUM_DES_LANES; j++) { + state->des_dec_ooo.lens[j] = 0; + state->des_dec_ooo.job_in_lane[j] = NULL; + } + state->des_dec_ooo.unused_lanes = 0xFEDCBA9876543210; + state->des_dec_ooo.num_lanes_inuse = 0; + memset(&state->des_dec_ooo.args, 0, sizeof(state->des_dec_ooo.args)); + + /* - separate 3DES OOO for encryption */ + for (j = 0; j < AVX512_NUM_DES_LANES; j++) { + state->des3_enc_ooo.lens[j] = 0; + state->des3_enc_ooo.job_in_lane[j] = NULL; + } + state->des3_enc_ooo.unused_lanes = 0xFEDCBA9876543210; + state->des3_enc_ooo.num_lanes_inuse = 0; + memset(&state->des3_enc_ooo.args, 0, sizeof(state->des3_enc_ooo.args)); + + /* - separate 3DES OOO for decryption */ + for (j = 0; j < AVX512_NUM_DES_LANES; j++) { + state->des3_dec_ooo.lens[j] = 0; + state->des3_dec_ooo.job_in_lane[j] = NULL; + } + state->des3_dec_ooo.unused_lanes = 0xFEDCBA9876543210; + state->des3_dec_ooo.num_lanes_inuse = 0; + memset(&state->des3_dec_ooo.args, 0, sizeof(state->des3_dec_ooo.args)); + + /* - separate DOCSIS DES OOO for encryption */ + for (j = 0; j < AVX512_NUM_DES_LANES; j++) { + state->docsis_des_enc_ooo.lens[j] = 0; + state->docsis_des_enc_ooo.job_in_lane[j] = NULL; + } + state->docsis_des_enc_ooo.unused_lanes = 0xFEDCBA9876543210; + state->docsis_des_enc_ooo.num_lanes_inuse = 0; + memset(&state->docsis_des_enc_ooo.args, 0, + sizeof(state->docsis_des_enc_ooo.args)); + + /* - separate DES OOO for decryption */ + for (j = 0; j < AVX512_NUM_DES_LANES; j++) { + state->docsis_des_dec_ooo.lens[j] = 0; + state->docsis_des_dec_ooo.job_in_lane[j] = NULL; + } + state->docsis_des_dec_ooo.unused_lanes = 0xFEDCBA9876543210; + state->docsis_des_dec_ooo.num_lanes_inuse = 0; + memset(&state->docsis_des_dec_ooo.args, 0, + sizeof(state->docsis_des_dec_ooo.args)); + + /* Init HMAC/SHA1 out-of-order fields */ + state->hmac_sha_1_ooo.lens[0] = 0; + state->hmac_sha_1_ooo.lens[1] = 0; + state->hmac_sha_1_ooo.lens[2] = 0; + state->hmac_sha_1_ooo.lens[3] = 0; + state->hmac_sha_1_ooo.lens[4] = 0; + state->hmac_sha_1_ooo.lens[5] = 0; + state->hmac_sha_1_ooo.lens[6] = 0; + state->hmac_sha_1_ooo.lens[7] = 0; + state->hmac_sha_1_ooo.lens[8] = 0; + state->hmac_sha_1_ooo.lens[9] = 0; + state->hmac_sha_1_ooo.lens[10] = 0; + state->hmac_sha_1_ooo.lens[11] = 0; + state->hmac_sha_1_ooo.lens[12] = 0; + state->hmac_sha_1_ooo.lens[13] = 0; + state->hmac_sha_1_ooo.lens[14] = 0; + state->hmac_sha_1_ooo.lens[15] = 0; + state->hmac_sha_1_ooo.unused_lanes = 0xFEDCBA9876543210; + state->hmac_sha_1_ooo.num_lanes_inuse = 0; + for (j = 0; j < AVX512_NUM_SHA1_LANES; j++) { + state->hmac_sha_1_ooo.ldata[j].job_in_lane = NULL; + state->hmac_sha_1_ooo.ldata[j].extra_block[64] = 0x80; + memset(state->hmac_sha_1_ooo.ldata[j].extra_block + 65, + 0x00, + 64 + 7); + p = state->hmac_sha_1_ooo.ldata[j].outer_block; + memset(p + 5*4 + 1, + 0x00, + 64 - 5*4 - 1 - 2); + p[5 * 4] = 0x80; + p[64 - 2] = 0x02; + p[64 - 1] = 0xA0; + } + + /* Init HMAC/SHA224 out-of-order fields */ + state->hmac_sha_224_ooo.lens[0] = 0; + state->hmac_sha_224_ooo.lens[1] = 0; + state->hmac_sha_224_ooo.lens[2] = 0; + state->hmac_sha_224_ooo.lens[3] = 0; + state->hmac_sha_224_ooo.lens[4] = 0; + state->hmac_sha_224_ooo.lens[5] = 0; + state->hmac_sha_224_ooo.lens[6] = 0; + state->hmac_sha_224_ooo.lens[7] = 0; + state->hmac_sha_224_ooo.lens[8] = 0; + state->hmac_sha_224_ooo.lens[9] = 0; + state->hmac_sha_224_ooo.lens[10] = 0; + state->hmac_sha_224_ooo.lens[11] = 0; + state->hmac_sha_224_ooo.lens[12] = 0; + state->hmac_sha_224_ooo.lens[13] = 0; + state->hmac_sha_224_ooo.lens[14] = 0; + state->hmac_sha_224_ooo.lens[15] = 0; + state->hmac_sha_224_ooo.unused_lanes = 0xFEDCBA9876543210; + state->hmac_sha_224_ooo.num_lanes_inuse = 0; + /* sha256 and sha224 are very similar except for + * digest constants and output size + */ + for (j = 0; j < AVX512_NUM_SHA256_LANES; j++) { + state->hmac_sha_224_ooo.ldata[j].job_in_lane = NULL; + + p = state->hmac_sha_224_ooo.ldata[j].extra_block; + size = sizeof(state->hmac_sha_224_ooo.ldata[j].extra_block); + memset (p, 0x00, size); + p[64] = 0x80; + + p = state->hmac_sha_224_ooo.ldata[j].outer_block; + size = sizeof(state->hmac_sha_224_ooo.ldata[j].outer_block); + memset(p, 0x00, size); + p[7 * 4] = 0x80; /* digest 7 words long */ + p[64 - 2] = 0x02; /* length in little endian = 0x02E0 */ + p[64 - 1] = 0xE0; + } + + /* Init HMAC/SHA256 out-of-order fields */ + state->hmac_sha_256_ooo.lens[0] = 0; + state->hmac_sha_256_ooo.lens[1] = 0; + state->hmac_sha_256_ooo.lens[2] = 0; + state->hmac_sha_256_ooo.lens[3] = 0; + state->hmac_sha_256_ooo.lens[4] = 0; + state->hmac_sha_256_ooo.lens[5] = 0; + state->hmac_sha_256_ooo.lens[6] = 0; + state->hmac_sha_256_ooo.lens[7] = 0; + state->hmac_sha_256_ooo.lens[8] = 0; + state->hmac_sha_256_ooo.lens[9] = 0; + state->hmac_sha_256_ooo.lens[10] = 0; + state->hmac_sha_256_ooo.lens[11] = 0; + state->hmac_sha_256_ooo.lens[12] = 0; + state->hmac_sha_256_ooo.lens[13] = 0; + state->hmac_sha_256_ooo.lens[14] = 0; + state->hmac_sha_256_ooo.lens[15] = 0; + state->hmac_sha_256_ooo.unused_lanes = 0xFEDCBA9876543210; + state->hmac_sha_256_ooo.num_lanes_inuse = 0; + for (j = 0; j < AVX512_NUM_SHA256_LANES; j++) { + state->hmac_sha_256_ooo.ldata[j].job_in_lane = NULL; + state->hmac_sha_256_ooo.ldata[j].extra_block[64] = 0x80; + memset(state->hmac_sha_256_ooo.ldata[j].extra_block + 65, + 0x00, + 64 + 7); + /* hmac related */ + p = state->hmac_sha_256_ooo.ldata[j].outer_block; + memset(p + 8*4 + 1, + 0x00, + 64 - 8*4 - 1 - 2); + p[8 * 4] = 0x80; /* 8 digest words */ + p[64 - 2] = 0x03; /* length */ + p[64 - 1] = 0x00; + } + + /* Init HMAC/SHA384 out-of-order fields */ + state->hmac_sha_384_ooo.lens[0] = 0; + state->hmac_sha_384_ooo.lens[1] = 0; + state->hmac_sha_384_ooo.lens[2] = 0; + state->hmac_sha_384_ooo.lens[3] = 0; + state->hmac_sha_384_ooo.lens[4] = 0; + state->hmac_sha_384_ooo.lens[5] = 0; + state->hmac_sha_384_ooo.lens[6] = 0; + state->hmac_sha_384_ooo.lens[7] = 0; + state->hmac_sha_384_ooo.unused_lanes = 0xF76543210; + for (j = 0; j < AVX512_NUM_SHA512_LANES; j++) { + MB_MGR_HMAC_SHA_512_OOO *ctx = &state->hmac_sha_384_ooo; + + ctx->ldata[j].job_in_lane = NULL; + ctx->ldata[j].extra_block[SHA_384_BLOCK_SIZE] = 0x80; + memset(ctx->ldata[j].extra_block + (SHA_384_BLOCK_SIZE + 1), + 0x00, SHA_384_BLOCK_SIZE + 7); + p = ctx->ldata[j].outer_block; + /* special end point because this length is constant */ + memset(p + SHA384_DIGEST_SIZE_IN_BYTES + 1, 0x00, + SHA_384_BLOCK_SIZE - + SHA384_DIGEST_SIZE_IN_BYTES - 1 - 2); + /* mark the end */ + p[SHA384_DIGEST_SIZE_IN_BYTES] = 0x80; + /* hmac outer block length always of fixed size, + * it is OKey length, a whole message block length, 1024 bits, + * with padding plus the length of the inner digest, + * which is 384 bits, 1408 bits == 0x0580. + * The input message block needs to be converted to big endian + * within the sha implementation before use. + */ + p[SHA_384_BLOCK_SIZE - 2] = 0x05; + p[SHA_384_BLOCK_SIZE - 1] = 0x80; + } + + /* Init HMAC/SHA512 out-of-order fields */ + state->hmac_sha_512_ooo.lens[0] = 0; + state->hmac_sha_512_ooo.lens[1] = 0; + state->hmac_sha_512_ooo.lens[2] = 0; + state->hmac_sha_512_ooo.lens[3] = 0; + state->hmac_sha_512_ooo.lens[4] = 0; + state->hmac_sha_512_ooo.lens[5] = 0; + state->hmac_sha_512_ooo.lens[6] = 0; + state->hmac_sha_512_ooo.lens[7] = 0; + state->hmac_sha_512_ooo.unused_lanes = 0xF76543210; + for (j = 0; j < AVX512_NUM_SHA512_LANES; j++) { + MB_MGR_HMAC_SHA_512_OOO *ctx = &state->hmac_sha_512_ooo; + + ctx->ldata[j].job_in_lane = NULL; + ctx->ldata[j].extra_block[SHA_512_BLOCK_SIZE] = 0x80; + memset(ctx->ldata[j].extra_block + (SHA_512_BLOCK_SIZE + 1), + 0x00, SHA_512_BLOCK_SIZE + 7); + p = ctx->ldata[j].outer_block; + /* special end point because this length is constant */ + memset(p + SHA512_DIGEST_SIZE_IN_BYTES + 1, 0x00, + SHA_512_BLOCK_SIZE - + SHA512_DIGEST_SIZE_IN_BYTES - 1 - 2); + /* mark the end */ + p[SHA512_DIGEST_SIZE_IN_BYTES] = 0x80; + /* hmac outer block length always of fixed size, + * it is OKey length, a whole message block length, 1024 bits, + * with padding plus the length of the inner digest, + * which is 512 bits, 1536 bits == 0x600. + * The input message block needs to be converted to big endian + * within the sha implementation before use. + */ + p[SHA_512_BLOCK_SIZE - 2] = 0x06; + p[SHA_512_BLOCK_SIZE - 1] = 0x00; + } + + /* Init HMAC/MD5 out-of-order fields */ + state->hmac_md5_ooo.lens[0] = 0; + state->hmac_md5_ooo.lens[1] = 0; + state->hmac_md5_ooo.lens[2] = 0; + state->hmac_md5_ooo.lens[3] = 0; + state->hmac_md5_ooo.lens[4] = 0; + state->hmac_md5_ooo.lens[5] = 0; + state->hmac_md5_ooo.lens[6] = 0; + state->hmac_md5_ooo.lens[7] = 0; + state->hmac_md5_ooo.lens[8] = 0; + state->hmac_md5_ooo.lens[9] = 0; + state->hmac_md5_ooo.lens[10] = 0; + state->hmac_md5_ooo.lens[11] = 0; + state->hmac_md5_ooo.lens[12] = 0; + state->hmac_md5_ooo.lens[13] = 0; + state->hmac_md5_ooo.lens[14] = 0; + state->hmac_md5_ooo.lens[15] = 0; + state->hmac_md5_ooo.unused_lanes = 0xFEDCBA9876543210; + state->hmac_md5_ooo.num_lanes_inuse = 0; + for (j = 0; j < AVX512_NUM_MD5_LANES; j++) { + state->hmac_md5_ooo.ldata[j].job_in_lane = NULL; + + p = state->hmac_md5_ooo.ldata[j].extra_block; + size = sizeof(state->hmac_md5_ooo.ldata[j].extra_block); + memset (p, 0x00, size); + p[64] = 0x80; + + p = state->hmac_md5_ooo.ldata[j].outer_block; + size = sizeof(state->hmac_md5_ooo.ldata[j].outer_block); + memset(p, 0x00, size); + p[4 * 4] = 0x80; + p[64 - 7] = 0x02; + p[64 - 8] = 0x80; + } + + /* Init AES/XCBC OOO fields */ + state->aes_xcbc_ooo.lens[0] = 0; + state->aes_xcbc_ooo.lens[1] = 0; + state->aes_xcbc_ooo.lens[2] = 0; + state->aes_xcbc_ooo.lens[3] = 0; + state->aes_xcbc_ooo.lens[4] = 0; + state->aes_xcbc_ooo.lens[5] = 0; + state->aes_xcbc_ooo.lens[6] = 0; + state->aes_xcbc_ooo.lens[7] = 0; + state->aes_xcbc_ooo.unused_lanes = 0xF76543210; + for (j = 0; j < 8 ; j++) { + state->aes_xcbc_ooo.ldata[j].job_in_lane = NULL; + state->aes_xcbc_ooo.ldata[j].final_block[16] = 0x80; + memset(state->aes_xcbc_ooo.ldata[j].final_block + 17, 0x00, 15); + } + + /* Init AES-CCM auth out-of-order fields */ + for (j = 0; j < 8; j++) { + state->aes_ccm_ooo.init_done[j] = 0; + state->aes_ccm_ooo.lens[j] = 0; + state->aes_ccm_ooo.job_in_lane[j] = NULL; + } + state->aes_ccm_ooo.unused_lanes = 0xF76543210; + + /* Init AES-CMAC auth out-of-order fields */ + for (j = 0; j < 8; j++) { + state->aes_cmac_ooo.init_done[j] = 0; + state->aes_cmac_ooo.lens[j] = 0; + state->aes_cmac_ooo.job_in_lane[j] = NULL; + } + state->aes_cmac_ooo.unused_lanes = 0xF76543210; + + /* Init "in order" components */ + state->next_job = 0; + state->earliest_job = -1; + + /* set handlers */ + state->get_next_job = get_next_job_avx512; + state->submit_job = submit_job_avx512; + state->submit_job_nocheck = submit_job_nocheck_avx512; + state->get_completed_job = get_completed_job_avx512; + state->flush_job = flush_job_avx512; + state->queue_size = queue_size_avx512; + state->keyexp_128 = aes_keyexp_128_avx512; + state->keyexp_192 = aes_keyexp_192_avx512; + state->keyexp_256 = aes_keyexp_256_avx512; + state->cmac_subkey_gen_128 = aes_cmac_subkey_gen_avx512; + state->xcbc_keyexp = aes_xcbc_expand_key_avx512; + state->des_key_sched = des_key_schedule; + state->sha1_one_block = sha1_one_block_avx512; + state->sha1 = sha1_avx512; + state->sha224_one_block = sha224_one_block_avx512; + state->sha224 = sha224_avx512; + state->sha256_one_block = sha256_one_block_avx512; + state->sha256 = sha256_avx512; + state->sha384_one_block = sha384_one_block_avx512; + state->sha384 = sha384_avx512; + state->sha512_one_block = sha512_one_block_avx512; + state->sha512 = sha512_avx512; + state->md5_one_block = md5_one_block_avx512; + state->aes128_cfb_one = aes_cfb_128_one_avx512; + + state->eea3_1_buffer = zuc_eea3_1_buffer_avx; + state->eea3_4_buffer = zuc_eea3_4_buffer_avx; + state->eea3_n_buffer = zuc_eea3_n_buffer_avx; + state->eia3_1_buffer = zuc_eia3_1_buffer_avx; + + state->f8_1_buffer = kasumi_f8_1_buffer_avx; + state->f8_1_buffer_bit = kasumi_f8_1_buffer_bit_avx; + state->f8_2_buffer = kasumi_f8_2_buffer_avx; + state->f8_3_buffer = kasumi_f8_3_buffer_avx; + state->f8_4_buffer = kasumi_f8_4_buffer_avx; + state->f8_n_buffer = kasumi_f8_n_buffer_avx; + state->f9_1_buffer = kasumi_f9_1_buffer_avx; + state->f9_1_buffer_user = kasumi_f9_1_buffer_user_avx; + state->kasumi_init_f8_key_sched = kasumi_init_f8_key_sched_avx; + state->kasumi_init_f9_key_sched = kasumi_init_f9_key_sched_avx; + state->kasumi_key_sched_size = kasumi_key_sched_size_avx; + + state->snow3g_f8_1_buffer_bit = snow3g_f8_1_buffer_bit_avx2; + state->snow3g_f8_1_buffer = snow3g_f8_1_buffer_avx2; + state->snow3g_f8_2_buffer = snow3g_f8_2_buffer_avx2; + state->snow3g_f8_4_buffer = snow3g_f8_4_buffer_avx2; + state->snow3g_f8_8_buffer = snow3g_f8_8_buffer_avx2; + state->snow3g_f8_n_buffer = snow3g_f8_n_buffer_avx2; + state->snow3g_f8_8_buffer_multikey = snow3g_f8_8_buffer_multikey_avx2; + state->snow3g_f8_n_buffer_multikey = snow3g_f8_n_buffer_multikey_avx2; + state->snow3g_f9_1_buffer = snow3g_f9_1_buffer_avx2; + state->snow3g_init_key_sched = snow3g_init_key_sched_avx2; + state->snow3g_key_sched_size = snow3g_key_sched_size_avx2; + + if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { + submit_job_aes_cntr_avx512 = vaes_submit_cntr_avx512; + submit_job_aes_cntr_bit_avx512 = vaes_submit_cntr_bit_avx512; + } +#ifndef NO_GCM + if ((state->features & (IMB_FEATURE_VAES | IMB_FEATURE_VPCLMULQDQ)) == + (IMB_FEATURE_VAES | IMB_FEATURE_VPCLMULQDQ)) { + state->gcm128_enc = aes_gcm_enc_128_vaes_avx512; + state->gcm192_enc = aes_gcm_enc_192_vaes_avx512; + state->gcm256_enc = aes_gcm_enc_256_vaes_avx512; + state->gcm128_dec = aes_gcm_dec_128_vaes_avx512; + state->gcm192_dec = aes_gcm_dec_192_vaes_avx512; + state->gcm256_dec = aes_gcm_dec_256_vaes_avx512; + state->gcm128_init = aes_gcm_init_128_vaes_avx512; + state->gcm192_init = aes_gcm_init_192_vaes_avx512; + state->gcm256_init = aes_gcm_init_256_vaes_avx512; + state->gcm128_enc_update = aes_gcm_enc_128_update_vaes_avx512; + state->gcm192_enc_update = aes_gcm_enc_192_update_vaes_avx512; + state->gcm256_enc_update = aes_gcm_enc_256_update_vaes_avx512; + state->gcm128_dec_update = aes_gcm_dec_128_update_vaes_avx512; + state->gcm192_dec_update = aes_gcm_dec_192_update_vaes_avx512; + state->gcm256_dec_update = aes_gcm_dec_256_update_vaes_avx512; + state->gcm128_enc_finalize = + aes_gcm_enc_128_finalize_vaes_avx512; + state->gcm192_enc_finalize = + aes_gcm_enc_192_finalize_vaes_avx512; + state->gcm256_enc_finalize = + aes_gcm_enc_256_finalize_vaes_avx512; + state->gcm128_dec_finalize = + aes_gcm_dec_128_finalize_vaes_avx512; + state->gcm192_dec_finalize = + aes_gcm_dec_192_finalize_vaes_avx512; + state->gcm256_dec_finalize = + aes_gcm_dec_256_finalize_vaes_avx512; + state->gcm128_precomp = aes_gcm_precomp_128_vaes_avx512; + state->gcm192_precomp = aes_gcm_precomp_192_vaes_avx512; + state->gcm256_precomp = aes_gcm_precomp_256_vaes_avx512; + state->gcm128_pre = aes_gcm_pre_128_vaes_avx512; + state->gcm192_pre = aes_gcm_pre_192_vaes_avx512; + state->gcm256_pre = aes_gcm_pre_256_vaes_avx512; + + submit_job_aes_gcm_enc_avx512 = vaes_submit_gcm_enc_avx512; + submit_job_aes_gcm_dec_avx512 = vaes_submit_gcm_dec_avx512; + } else { + state->gcm128_enc = aes_gcm_enc_128_avx512; + state->gcm192_enc = aes_gcm_enc_192_avx512; + state->gcm256_enc = aes_gcm_enc_256_avx512; + state->gcm128_dec = aes_gcm_dec_128_avx512; + state->gcm192_dec = aes_gcm_dec_192_avx512; + state->gcm256_dec = aes_gcm_dec_256_avx512; + state->gcm128_init = aes_gcm_init_128_avx512; + state->gcm192_init = aes_gcm_init_192_avx512; + state->gcm256_init = aes_gcm_init_256_avx512; + state->gcm128_enc_update = aes_gcm_enc_128_update_avx512; + state->gcm192_enc_update = aes_gcm_enc_192_update_avx512; + state->gcm256_enc_update = aes_gcm_enc_256_update_avx512; + state->gcm128_dec_update = aes_gcm_dec_128_update_avx512; + state->gcm192_dec_update = aes_gcm_dec_192_update_avx512; + state->gcm256_dec_update = aes_gcm_dec_256_update_avx512; + state->gcm128_enc_finalize = aes_gcm_enc_128_finalize_avx512; + state->gcm192_enc_finalize = aes_gcm_enc_192_finalize_avx512; + state->gcm256_enc_finalize = aes_gcm_enc_256_finalize_avx512; + state->gcm128_dec_finalize = aes_gcm_dec_128_finalize_avx512; + state->gcm192_dec_finalize = aes_gcm_dec_192_finalize_avx512; + state->gcm256_dec_finalize = aes_gcm_dec_256_finalize_avx512; + state->gcm128_precomp = aes_gcm_precomp_128_avx512; + state->gcm192_precomp = aes_gcm_precomp_192_avx512; + state->gcm256_precomp = aes_gcm_precomp_256_avx512; + state->gcm128_pre = aes_gcm_pre_128_avx512; + state->gcm192_pre = aes_gcm_pre_192_avx512; + state->gcm256_pre = aes_gcm_pre_256_avx512; + } +#endif +} + +#include "mb_mgr_code.h" |