Adding upstream version 14.2.21.upstream/14.2.21upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 1419 insertions, 0 deletions

diff --git a/src/spdk/intel-ipsec-mb/intel-ipsec-mb.h b/src/spdk/intel-ipsec-mb/intel-ipsec-mb.h
new file mode 100644
index 00000000..09d5f079
--- /dev/null
+++ b/src/spdk/intel-ipsec-mb/intel-ipsec-mb.h
@@ -0,0 +1,1419 @@
+/*******************************************************************************
+  Copyright (c) 2012-2018, Intel Corporation
+
+  Redistribution and use in source and binary forms, with or without
+  modification, are permitted provided that the following conditions are met:
+
+      * Redistributions of source code must retain the above copyright notice,
+        this list of conditions and the following disclaimer.
+      * Redistributions in binary form must reproduce the above copyright
+        notice, this list of conditions and the following disclaimer in the
+        documentation and/or other materials provided with the distribution.
+      * Neither the name of Intel Corporation nor the names of its contributors
+        may be used to endorse or promote products derived from this software
+        without specific prior written permission.
+
+  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+  FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+  DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+  OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*******************************************************************************/
+
+#ifndef IMB_IPSEC_MB_H
+#define IMB_IPSEC_MB_H
+
+#include <stdlib.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* 128-bit data type that is not in sdtint.h */
+typedef struct {
+        uint64_t low;
+        uint64_t high;
+} uint128_t;
+
+/*
+ * Macros for aligning data structures and function inlines
+ */
+#ifdef __linux__
+/* Linux */
+#define DECLARE_ALIGNED(decl, alignval) \
+        decl __attribute__((aligned(alignval)))
+#define __forceinline \
+        static inline __attribute__((always_inline))
+
+#if __GNUC__ >= 4
+#define IMB_DLL_EXPORT __attribute__((visibility("default")))
+#define IMB_DLL_LOCAL  __attribute__((visibility("hidden")))
+#else /* GNU C 4.0 and later */
+#define IMB_DLL_EXPORT
+#define IMB_DLL_LOCAL
+#endif /* different C compiler */
+
+#else
+/* Windows */
+#define DECLARE_ALIGNED(decl, alignval) \
+        __declspec(align(alignval)) decl
+#define __forceinline \
+        static __forceinline
+
+/* Windows DLL export is done via DEF file */
+#define IMB_DLL_EXPORT
+#define IMB_DLL_LOCAL
+#endif
+
+/*
+ * Custom ASSERT and DIM macros
+ */
+#ifdef DEBUG
+#include <assert.h>
+#define IMB_ASSERT(x) assert(x)
+#else
+#define IMB_ASSERT(x)
+#endif
+
+#ifndef IMB_DIM
+#define IMB_DIM(x) (sizeof(x) / sizeof(x[0]))
+#endif
+
+/*
+ * Algorithm constants
+ */
+
+#define DES_KEY_SCHED_SIZE (16 * 8) /* 16 rounds x 8 bytes */
+#define DES_BLOCK_SIZE 8
+
+#define AES_BLOCK_SIZE 16
+
+#define NUM_MD5_DIGEST_WORDS     4
+#define NUM_SHA_DIGEST_WORDS     5
+#define NUM_SHA_256_DIGEST_WORDS 8
+#define NUM_SHA_224_DIGEST_WORDS 7
+#define NUM_SHA_512_DIGEST_WORDS 8
+#define NUM_SHA_384_DIGEST_WORDS 6
+
+#define SHA384_DIGEST_WORD_SIZE   8
+#define SHA512_DIGEST_WORD_SIZE   8
+
+#define SHA384_DIGEST_SIZE_IN_BYTES \
+        (NUM_SHA_384_DIGEST_WORDS * SHA384_DIGEST_WORD_SIZE)
+#define SHA512_DIGEST_SIZE_IN_BYTES \
+        (NUM_SHA_512_DIGEST_WORDS * SHA512_DIGEST_WORD_SIZE)
+
+#define SHA1_BLOCK_SIZE 64    /* 512 bits is 64 byte blocks */
+#define SHA_256_BLOCK_SIZE 64 /* 512 bits is 64 byte blocks */
+#define SHA_384_BLOCK_SIZE 128
+#define SHA_512_BLOCK_SIZE 128
+
+/* Number of lanes AVX512, AVX2, AVX and SSE */
+#define AVX512_NUM_SHA1_LANES   16
+#define AVX512_NUM_SHA256_LANES 16
+#define AVX512_NUM_SHA512_LANES 8
+#define AVX512_NUM_MD5_LANES    32
+#define AVX512_NUM_DES_LANES    16
+
+#define AVX2_NUM_SHA1_LANES     8
+#define AVX2_NUM_SHA256_LANES   8
+#define AVX2_NUM_SHA512_LANES   4
+#define AVX2_NUM_MD5_LANES      16
+
+#define AVX_NUM_SHA1_LANES      4
+#define AVX_NUM_SHA256_LANES    4
+#define AVX_NUM_SHA512_LANES    2
+#define AVX_NUM_MD5_LANES       8
+
+#define SSE_NUM_SHA1_LANES   AVX_NUM_SHA1_LANES
+#define SSE_NUM_SHA256_LANES AVX_NUM_SHA256_LANES
+#define SSE_NUM_SHA512_LANES AVX_NUM_SHA512_LANES
+#define SSE_NUM_MD5_LANES    AVX_NUM_MD5_LANES
+
+/*
+ *  Each row is sized to hold enough lanes for AVX2, AVX1 and SSE use a subset
+ * of each row. Thus one row is not adjacent in memory to its neighboring rows
+ * in the case of SSE and AVX1.
+ */
+#define MD5_DIGEST_SZ    (NUM_MD5_DIGEST_WORDS * AVX512_NUM_MD5_LANES)
+#define SHA1_DIGEST_SZ   (NUM_SHA_DIGEST_WORDS * AVX512_NUM_SHA1_LANES)
+#define SHA256_DIGEST_SZ (NUM_SHA_256_DIGEST_WORDS * AVX512_NUM_SHA256_LANES)
+#define SHA512_DIGEST_SZ (NUM_SHA_512_DIGEST_WORDS * AVX512_NUM_SHA512_LANES)
+
+/*
+ * Job structure definitions
+ */
+
+typedef enum {
+        STS_BEING_PROCESSED = 0,
+        STS_COMPLETED_AES =   1,
+        STS_COMPLETED_HMAC =  2,
+        STS_COMPLETED =       3, /* COMPLETED_AES | COMPLETED_HMAC */
+        STS_INVALID_ARGS =    4,
+        STS_INTERNAL_ERROR,
+        STS_ERROR
+} JOB_STS;
+
+typedef enum {
+        CBC = 1,
+        CNTR,
+        NULL_CIPHER,
+        DOCSIS_SEC_BPI,
+#ifndef NO_GCM
+        GCM,
+#endif /* !NO_GCM */
+        CUSTOM_CIPHER,
+        DES,
+        DOCSIS_DES,
+        CCM,
+        DES3
+} JOB_CIPHER_MODE;
+
+typedef enum {
+        ENCRYPT = 1,
+        DECRYPT
+} JOB_CIPHER_DIRECTION;
+
+typedef enum {
+        SHA1 = 1,
+        SHA_224,
+        SHA_256,
+        SHA_384,
+        SHA_512,
+        AES_XCBC,
+        MD5,
+        NULL_HASH,
+#ifndef NO_GCM
+        AES_GMAC,
+#endif /* !NO_GCM */
+        CUSTOM_HASH,
+        AES_CCM,
+        AES_CMAC,
+} JOB_HASH_ALG;
+
+typedef enum {
+        CIPHER_HASH = 1,
+        HASH_CIPHER
+} JOB_CHAIN_ORDER;
+
+typedef enum {
+        AES_128_BYTES = 16,
+        AES_192_BYTES = 24,
+        AES_256_BYTES = 32
+} AES_KEY_SIZE_BYTES;
+
+typedef struct JOB_AES_HMAC {
+        /*
+         * For AES, aes_enc_key_expanded and aes_dec_key_expanded are
+         * expected to point to expanded keys structure.
+         * - AES-CTR and AES-CCM, only aes_enc_key_expanded is used
+         * - DOCSIS (AES-CBC + AES-CFB), both pointers are used
+         *   aes_enc_key_expanded has to be set always for the partial block
+         *
+         * For DES, aes_enc_key_expanded and aes_dec_key_expanded are
+         * expected to point to DES key schedule.
+         * - same key schedule used for enc and dec operations
+         *
+         * For 3DES, aes_enc_key_expanded and aes_dec_key_expanded are
+         * expected to point to an array of 3 pointers for
+         * the corresponding 3 key schedules.
+         * - same key schedule used for enc and dec operations
+         */
+        const void *aes_enc_key_expanded;  /* 16-byte aligned pointer. */
+        const void *aes_dec_key_expanded;
+        uint64_t aes_key_len_in_bytes; /* 16, 24 and 32 byte (128, 192 and
+                                        * 256-bit) keys supported */
+        const uint8_t *src; /* Input. May be cipher text or plaintext.
+                             * In-place ciphering allowed. */
+        uint8_t *dst; /*Output. May be cipher text or plaintext.
+                       * In-place ciphering allowed, i.e. dst = src. */
+        uint64_t cipher_start_src_offset_in_bytes;
+        uint64_t msg_len_to_cipher_in_bytes; /* Max len = 65472 bytes.
+                                              * IPSec case, the maximum cipher
+                                              * length would be:
+                                              * 65535 -
+                                              * 20 (outer IP header) -
+                                              * 24 (ESP header + IV) -
+                                              * 12 (supported ICV length) */
+        uint64_t hash_start_src_offset_in_bytes;
+        uint64_t msg_len_to_hash_in_bytes; /* Max len = 65496 bytes.
+                                            * (Max cipher len +
+                                            * 24 bytes ESP header) */
+        const uint8_t *iv; /* AES IV. */
+        uint64_t iv_len_in_bytes; /* AES IV length in bytes. */
+        uint8_t *auth_tag_output; /* HMAC Tag output. This may point to
+                                   * a location in the src buffer
+                                   * (for in place)*/
+        uint64_t auth_tag_output_len_in_bytes; /* Authentication (i.e. HMAC) tag
+                                                * output length in bytes
+                                                * (may be a truncated value) */
+
+        /* Start algorithm-specific fields */
+        union {
+                struct _HMAC_specific_fields {
+                        /* Hashed result of HMAC key xor'd with ipad (0x36). */
+                        const uint8_t *_hashed_auth_key_xor_ipad;
+                        /* Hashed result of HMAC key xor'd with opad (0x5c). */
+                        const uint8_t *_hashed_auth_key_xor_opad;
+                } HMAC;
+                struct _AES_XCBC_specific_fields {
+                        /* 16-byte aligned pointers */
+                        const uint32_t *_k1_expanded;
+                        const uint8_t *_k2;
+                        const uint8_t *_k3;
+                } XCBC;
+                struct _AES_CCM_specific_fields {
+                        /* Additional Authentication Data (AAD) */
+                        const void *aad;
+                        uint64_t aad_len_in_bytes; /* Length of AAD */
+                } CCM;
+                struct _AES_CMAC_specific_fields {
+                        const void *_key_expanded; /* 16-byte aligned */
+                        const void *_skey1;
+                        const void *_skey2;
+                } CMAC;
+#ifndef NO_GCM
+                struct _AES_GCM_specific_fields {
+                        /* Additional Authentication Data (AAD) */
+                        const void *aad;
+                        uint64_t aad_len_in_bytes;    /* Length of AAD */
+                } GCM;
+#endif /* !NO_GCM */
+        } u;
+
+        JOB_STS status;
+        JOB_CIPHER_MODE cipher_mode; /* CBC, CNTR, DES, GCM etc. */
+        JOB_CIPHER_DIRECTION cipher_direction; /* Encrypt/decrypt */
+        /* Ignored as the direction is implied by the chain _order field. */
+        JOB_HASH_ALG hash_alg; /* SHA-1 or others... */
+        JOB_CHAIN_ORDER chain_order; /* CIPHER_HASH or HASH_CIPHER */
+
+        void *user_data;
+        void *user_data2;
+
+        /*
+         * stateless custom cipher and hash
+         *   Return:
+         *     success: 0
+         *     fail:    other
+         */
+        int (*cipher_func)(struct JOB_AES_HMAC *);
+        int (*hash_func)(struct JOB_AES_HMAC *);
+} JOB_AES_HMAC;
+
+/*
+ * Argument structures for various algorithms
+ */
+typedef struct {
+        const uint8_t *in[8];
+        uint8_t *out[8];
+        const uint32_t *keys[8];
+        DECLARE_ALIGNED(uint128_t IV[8], 32);
+} AES_ARGS_x8;
+
+typedef struct {
+        DECLARE_ALIGNED(uint32_t digest[SHA1_DIGEST_SZ], 32);
+        uint8_t *data_ptr[AVX512_NUM_SHA1_LANES];
+} SHA1_ARGS;
+
+typedef struct {
+        DECLARE_ALIGNED(uint32_t digest[SHA256_DIGEST_SZ], 32);
+        uint8_t *data_ptr[AVX512_NUM_SHA256_LANES];
+} SHA256_ARGS;
+
+typedef struct {
+        DECLARE_ALIGNED(uint64_t digest[SHA512_DIGEST_SZ], 32);
+        uint8_t *data_ptr[AVX512_NUM_SHA512_LANES];
+}  SHA512_ARGS;
+
+typedef struct {
+        DECLARE_ALIGNED(uint32_t digest[MD5_DIGEST_SZ], 32);
+        uint8_t *data_ptr[AVX512_NUM_MD5_LANES];
+} MD5_ARGS;
+
+typedef struct {
+        const uint8_t *in[8];
+        const uint32_t *keys[8];
+        DECLARE_ALIGNED(uint128_t ICV[8], 32);
+} AES_XCBC_ARGS_x8;
+
+typedef struct {
+        const uint8_t *in[AVX512_NUM_DES_LANES];
+        uint8_t *out[AVX512_NUM_DES_LANES];
+        const uint8_t *keys[AVX512_NUM_DES_LANES];
+        uint32_t IV[AVX512_NUM_DES_LANES * 2]; /* uint32_t is more handy here */
+        uint32_t partial_len[AVX512_NUM_DES_LANES];
+        uint32_t block_len[AVX512_NUM_DES_LANES];
+        const uint8_t *last_in[AVX512_NUM_DES_LANES];
+        uint8_t *last_out[AVX512_NUM_DES_LANES];
+} DES_ARGS_x16;
+
+/* AES out-of-order scheduler fields */
+typedef struct {
+        AES_ARGS_x8 args;
+        DECLARE_ALIGNED(uint16_t lens[8], 16);
+        /* each nibble is index (0...7) of an unused lane,
+         * the last nibble is set to F as a flag
+         */
+        uint64_t unused_lanes;
+        JOB_AES_HMAC *job_in_lane[8];
+} MB_MGR_AES_OOO;
+
+/* AES XCBC out-of-order scheduler fields */
+typedef struct {
+        DECLARE_ALIGNED(uint8_t final_block[2 * 16], 32);
+        JOB_AES_HMAC *job_in_lane;
+        uint64_t final_done;
+} XCBC_LANE_DATA;
+
+typedef struct {
+        AES_XCBC_ARGS_x8 args;
+        DECLARE_ALIGNED(uint16_t lens[8], 16);
+        /* each byte is index (0...3) of unused lanes
+         * byte 4 is set to FF as a flag
+         */
+        uint64_t unused_lanes;
+        XCBC_LANE_DATA ldata[8];
+} MB_MGR_AES_XCBC_OOO;
+
+/* AES-CCM out-of-order scheduler structure */
+typedef struct {
+        AES_ARGS_x8 args; /* need to re-use AES arguments */
+        DECLARE_ALIGNED(uint16_t lens[8], 16);
+        DECLARE_ALIGNED(uint16_t init_done[8], 16);
+        /* each byte is index (0...3) of unused lanes
+         * byte 4 is set to FF as a flag
+         */
+        uint64_t unused_lanes;
+        JOB_AES_HMAC *job_in_lane[8];
+        DECLARE_ALIGNED(uint8_t init_blocks[8 * (4 * 16)], 32);
+} MB_MGR_CCM_OOO;
+
+
+/* AES-CMAC out-of-order scheduler structure */
+typedef struct {
+        AES_ARGS_x8 args; /* need to re-use AES arguments */
+        DECLARE_ALIGNED(uint16_t lens[8], 16);
+        DECLARE_ALIGNED(uint16_t init_done[8], 16);
+        /* each byte is index (0...3) of unused lanes
+         * byte 4 is set to FF as a flag
+         */
+        uint64_t unused_lanes;
+        JOB_AES_HMAC *job_in_lane[8];
+        DECLARE_ALIGNED(uint8_t scratch[8 * 16], 32);
+} MB_MGR_CMAC_OOO;
+
+
+/* DES out-of-order scheduler fields */
+typedef struct {
+        DES_ARGS_x16 args;
+        DECLARE_ALIGNED(uint16_t lens[16], 16);
+        /* each nibble is index (0...7) of unused lanes
+         * nibble 8 is set to F as a flag
+         */
+        uint64_t unused_lanes;
+        JOB_AES_HMAC *job_in_lane[16];
+        uint32_t num_lanes_inuse;
+} MB_MGR_DES_OOO;
+
+
+/* HMAC-SHA1 and HMAC-SHA256/224 */
+typedef struct {
+        /* YMM aligned access to extra_block */
+        DECLARE_ALIGNED(uint8_t extra_block[2 * SHA1_BLOCK_SIZE+8], 32);
+        JOB_AES_HMAC *job_in_lane;
+        uint8_t outer_block[64];
+        uint32_t outer_done;
+        uint32_t extra_blocks; /* num extra blocks (1 or 2) */
+        uint32_t size_offset;  /* offset in extra_block to start of
+                                * size field */
+        uint32_t start_offset; /* offset to start of data */
+} HMAC_SHA1_LANE_DATA;
+
+/* HMAC-SHA512/384 */
+typedef struct {
+        DECLARE_ALIGNED(uint8_t extra_block[2 * SHA_512_BLOCK_SIZE + 16], 32);
+        uint8_t outer_block[SHA_512_BLOCK_SIZE];
+        JOB_AES_HMAC *job_in_lane;
+        uint32_t outer_done;
+        uint32_t extra_blocks; /* num extra blocks (1 or 2) */
+        uint32_t size_offset;  /* offset in extra_block to start of
+                                * size field */
+        uint32_t start_offset; /* offset to start of data */
+} HMAC_SHA512_LANE_DATA;
+
+/*
+ * unused_lanes contains a list of unused lanes stored as bytes or as
+ * nibbles depending on the arch. The end of list is either FF or F.
+ */
+typedef struct {
+        SHA1_ARGS args;
+        DECLARE_ALIGNED(uint16_t lens[16], 32);
+        uint64_t unused_lanes;
+        HMAC_SHA1_LANE_DATA ldata[AVX512_NUM_SHA1_LANES];
+        uint32_t num_lanes_inuse;
+} MB_MGR_HMAC_SHA_1_OOO;
+
+typedef struct {
+        SHA256_ARGS args;
+        DECLARE_ALIGNED(uint16_t lens[16], 16);
+        uint64_t unused_lanes;
+        HMAC_SHA1_LANE_DATA ldata[AVX512_NUM_SHA256_LANES];
+        uint32_t num_lanes_inuse;
+} MB_MGR_HMAC_SHA_256_OOO;
+
+typedef struct {
+        SHA512_ARGS args;
+        DECLARE_ALIGNED(uint16_t lens[8], 16);
+        uint64_t unused_lanes;
+        HMAC_SHA512_LANE_DATA ldata[AVX512_NUM_SHA512_LANES];
+} MB_MGR_HMAC_SHA_512_OOO;
+
+/* MD5-HMAC out-of-order scheduler fields */
+typedef struct {
+        MD5_ARGS args;
+        DECLARE_ALIGNED(uint16_t lens[AVX512_NUM_MD5_LANES], 16);
+        /*
+         * In the avx2 case, all 16 nibbles of unused lanes are used.
+         * In that case num_lanes_inuse is used to detect the end of the list
+         */
+        uint64_t unused_lanes;
+        HMAC_SHA1_LANE_DATA ldata[AVX512_NUM_MD5_LANES];
+        uint32_t num_lanes_inuse;
+} MB_MGR_HMAC_MD5_OOO;
+
+/* ========================================================================== */
+/* API data type definitions */
+struct MB_MGR;
+
+typedef void (*init_mb_mgr_t)(struct MB_MGR *);
+typedef JOB_AES_HMAC *(*get_next_job_t)(struct MB_MGR *);
+typedef JOB_AES_HMAC *(*submit_job_t)(struct MB_MGR *);
+typedef JOB_AES_HMAC *(*get_completed_job_t)(struct MB_MGR *);
+typedef JOB_AES_HMAC *(*flush_job_t)(struct MB_MGR *);
+typedef uint32_t (*queue_size_t)(struct MB_MGR *);
+typedef void (*keyexp_t)(const void *, void *, void *);
+typedef void (*cmac_subkey_gen_t)(const void *, void *, void *);
+typedef void (*hash_one_block_t)(const void *, void *);
+typedef void (*xcbc_keyexp_t)(const void *, void *, void *, void *);
+typedef int (*des_keysched_t)(uint64_t *, const void *);
+
+/* ========================================================================== */
+/* Multi-buffer manager flags passed to alloc_mb_mgr() */
+
+#define IMB_FLAG_SHANI_OFF (1ULL << 0) /* disable use of SHANI extension */
+
+/* ========================================================================== */
+/* Multi-buffer manager features - valid after call to init_mb_mgr() */
+
+#define IMB_FEATURE_SHANI  (1ULL << 0) /* if set SHANI extensions is used */
+
+/* ========================================================================== */
+/* TOP LEVEL (MB_MGR) Data structure fields */
+
+#define MAX_JOBS 128
+
+typedef struct MB_MGR {
+        /*
+         * flags - passed to alloc_mb_mgr()
+         * features - reflects features of multi-buffer instance
+         */
+        uint64_t flags;
+        uint64_t features;
+
+        /*
+         * ARCH handlers / API
+         * Careful as changes here can break ABI compatibility
+         */
+        get_next_job_t          get_next_job;
+        submit_job_t            submit_job;
+        submit_job_t            submit_job_nocheck;
+        get_completed_job_t     get_completed_job;
+        flush_job_t             flush_job;
+        queue_size_t            queue_size;
+        keyexp_t                keyexp_128;
+        keyexp_t                keyexp_192;
+        keyexp_t                keyexp_256;
+        cmac_subkey_gen_t       cmac_subkey_gen_128;
+        xcbc_keyexp_t           xcbc_keyexp;
+        des_keysched_t          des_key_sched;
+        hash_one_block_t        sha1_one_block;
+        hash_one_block_t        sha224_one_block;
+        hash_one_block_t        sha256_one_block;
+        hash_one_block_t        sha384_one_block;
+        hash_one_block_t        sha512_one_block;
+        hash_one_block_t        md5_one_block;
+
+        /* in-order scheduler fields */
+        int              earliest_job; /* byte offset, -1 if none */
+        int              next_job;     /* byte offset */
+        JOB_AES_HMAC     jobs[MAX_JOBS];
+
+        /* out of order managers */
+        DECLARE_ALIGNED(MB_MGR_AES_OOO aes128_ooo, 64);
+        DECLARE_ALIGNED(MB_MGR_AES_OOO aes192_ooo, 64);
+        DECLARE_ALIGNED(MB_MGR_AES_OOO aes256_ooo, 64);
+        DECLARE_ALIGNED(MB_MGR_AES_OOO docsis_sec_ooo, 64);
+        DECLARE_ALIGNED(MB_MGR_DES_OOO des_enc_ooo, 64);
+        DECLARE_ALIGNED(MB_MGR_DES_OOO des_dec_ooo, 64);
+        DECLARE_ALIGNED(MB_MGR_DES_OOO des3_enc_ooo, 64);
+        DECLARE_ALIGNED(MB_MGR_DES_OOO des3_dec_ooo, 64);
+        DECLARE_ALIGNED(MB_MGR_DES_OOO docsis_des_enc_ooo, 64);
+        DECLARE_ALIGNED(MB_MGR_DES_OOO docsis_des_dec_ooo, 64);
+
+        DECLARE_ALIGNED(MB_MGR_HMAC_SHA_1_OOO hmac_sha_1_ooo, 64);
+        DECLARE_ALIGNED(MB_MGR_HMAC_SHA_256_OOO hmac_sha_224_ooo, 64);
+        DECLARE_ALIGNED(MB_MGR_HMAC_SHA_256_OOO hmac_sha_256_ooo, 64);
+        DECLARE_ALIGNED(MB_MGR_HMAC_SHA_512_OOO hmac_sha_384_ooo, 64);
+        DECLARE_ALIGNED(MB_MGR_HMAC_SHA_512_OOO hmac_sha_512_ooo, 64);
+        DECLARE_ALIGNED(MB_MGR_HMAC_MD5_OOO hmac_md5_ooo, 64);
+        DECLARE_ALIGNED(MB_MGR_AES_XCBC_OOO aes_xcbc_ooo, 64);
+        DECLARE_ALIGNED(MB_MGR_CCM_OOO aes_ccm_ooo, 64);
+        DECLARE_ALIGNED(MB_MGR_CMAC_OOO aes_cmac_ooo, 64);
+} MB_MGR;
+
+/* ========================================================================== */
+/* API definitions */
+
+/*
+ * get_next_job returns a job object. This must be filled in and returned
+ * via submit_job before get_next_job is called again.
+ * After submit_job is called, one should call get_completed_job() at least
+ * once (and preferably until it returns NULL).
+ * get_completed_job and flush_job returns a job object. This job object ceases
+ * to be usable at the next call to get_next_job
+ */
+IMB_DLL_EXPORT MB_MGR *alloc_mb_mgr(uint64_t flags);
+IMB_DLL_EXPORT void free_mb_mgr(MB_MGR *state);
+
+IMB_DLL_EXPORT void init_mb_mgr_avx(MB_MGR *state);
+IMB_DLL_EXPORT JOB_AES_HMAC *submit_job_avx(MB_MGR *state);
+IMB_DLL_EXPORT JOB_AES_HMAC *submit_job_nocheck_avx(MB_MGR *state);
+IMB_DLL_EXPORT JOB_AES_HMAC *flush_job_avx(MB_MGR *state);
+IMB_DLL_EXPORT uint32_t queue_size_avx(MB_MGR *state);
+
+IMB_DLL_EXPORT void init_mb_mgr_avx2(MB_MGR *state);
+IMB_DLL_EXPORT JOB_AES_HMAC *submit_job_avx2(MB_MGR *state);
+IMB_DLL_EXPORT JOB_AES_HMAC *submit_job_nocheck_avx2(MB_MGR *state);
+IMB_DLL_EXPORT JOB_AES_HMAC *flush_job_avx2(MB_MGR *state);
+IMB_DLL_EXPORT uint32_t queue_size_avx2(MB_MGR *state);
+
+IMB_DLL_EXPORT void init_mb_mgr_avx512(MB_MGR *state);
+IMB_DLL_EXPORT JOB_AES_HMAC *submit_job_avx512(MB_MGR *state);
+IMB_DLL_EXPORT JOB_AES_HMAC *submit_job_nocheck_avx512(MB_MGR *state);
+IMB_DLL_EXPORT JOB_AES_HMAC *flush_job_avx512(MB_MGR *state);
+IMB_DLL_EXPORT uint32_t queue_size_avx512(MB_MGR *state);
+
+IMB_DLL_EXPORT void init_mb_mgr_sse(MB_MGR *state);
+IMB_DLL_EXPORT JOB_AES_HMAC *submit_job_sse(MB_MGR *state);
+IMB_DLL_EXPORT JOB_AES_HMAC *submit_job_nocheck_sse(MB_MGR *state);
+IMB_DLL_EXPORT JOB_AES_HMAC *flush_job_sse(MB_MGR *state);
+IMB_DLL_EXPORT uint32_t queue_size_sse(MB_MGR *state);
+
+#define get_completed_job_avx  get_completed_job_sse
+#define get_next_job_avx       get_next_job_sse
+
+#define get_completed_job_avx2 get_completed_job_sse
+#define get_next_job_avx2      get_next_job_sse
+
+#define get_completed_job_avx512 get_completed_job_sse
+#define get_next_job_avx512      get_next_job_sse
+
+/*
+ * JOBS() and ADV_JOBS() also used in mb_mgr_code.h
+ * index in JOBS array using byte offset rather than object index
+ */
+__forceinline
+JOB_AES_HMAC *JOBS(MB_MGR *state, const int offset)
+{
+        char *cp = (char *)state->jobs;
+
+        return (JOB_AES_HMAC *)(cp + offset);
+}
+
+__forceinline
+void ADV_JOBS(int *ptr)
+{
+        *ptr += sizeof(JOB_AES_HMAC);
+        if (*ptr >= (int) (MAX_JOBS * sizeof(JOB_AES_HMAC)))
+                *ptr = 0;
+}
+
+__forceinline
+JOB_AES_HMAC *
+get_completed_job_sse(MB_MGR *state)
+{
+        JOB_AES_HMAC *job;
+
+        if (state->earliest_job < 0)
+                return NULL;
+
+        job = JOBS(state, state->earliest_job);
+        if (job->status < STS_COMPLETED)
+                return NULL;
+
+        ADV_JOBS(&state->earliest_job);
+
+        if (state->earliest_job == state->next_job)
+                state->earliest_job = -1;
+
+        return job;
+}
+
+__forceinline
+JOB_AES_HMAC *
+get_next_job_sse(MB_MGR *state)
+{
+        return JOBS(state, state->next_job);
+}
+
+/*
+ * Wrapper macros to call arch API's set up
+ * at init phase of multi-buffer manager.
+ *
+ * For example, after calling init_mb_mgr_sse(&mgr)
+ * The 'mgr' structure be set up so that:
+ *   mgr.get_next_job will point to get_next_job_sse(),
+ *   mgr.submit_job will point to submit_job_sse(),
+ *   mgr.submit_job_nocheck will point to submit_job_nocheck_sse(),
+ *   mgr.get_completed_job will point to get_completed_job_sse(),
+ *   mgr.flush_job will point to flush_job_sse(),
+ *   mgr.queue_size will point to queue_size_sse()
+ *   mgr.keyexp_128 will point to aes_keyexp_128_sse()
+ *   mgr.keyexp_192 will point to aes_keyexp_192_sse()
+ *   mgr.keyexp_256 will point to aes_keyexp_256_sse()
+ *
+ * Direct use of arch API's may result in better performance.
+ * Using below indirect interface may produce slightly worse performance but
+ * it can simplify application implementation.
+ * LibTestApp provides example of using the indirect interface.
+ */
+#define IMB_GET_NEXT_JOB(_mgr)       ((_mgr)->get_next_job((_mgr)))
+#define IMB_SUBMIT_JOB(_mgr)         ((_mgr)->submit_job((_mgr)))
+#define IMB_SUBMIT_JOB_NOCHECK(_mgr) ((_mgr)->submit_job_nocheck((_mgr)))
+#define IMB_GET_COMPLETED_JOB(_mgr)  ((_mgr)->get_completed_job((_mgr)))
+#define IMB_FLUSH_JOB(_mgr)          ((_mgr)->flush_job((_mgr)))
+#define IMB_QUEUE_SIZE(_mgr)         ((_mgr)->queue_size((_mgr)))
+#define IMB_AES_KEYEXP_128(_mgr, _raw, _enc, _dec)      \
+        ((_mgr)->keyexp_128((_raw), (_enc), (_dec)))
+#define IMB_AES_KEYEXP_192(_mgr, _raw, _enc, _dec)      \
+        ((_mgr)->keyexp_192((_raw), (_enc), (_dec)))
+#define IMB_AES_KEYEXP_256(_mgr, _raw, _enc, _dec)      \
+        ((_mgr)->keyexp_256((_raw), (_enc), (_dec)))
+#define IMB_AES_CMAC_SUBKEY_GEN_128(_mgr, _key_exp, _k1, _k2)   \
+        ((_mgr)->cmac_subkey_gen_128((_key_exp), (_k1), (_k2)))
+#define IMB_AES_XCBC_KEYEXP(_mgr, _key, _k1_exp, _k2, _k3)      \
+        ((_mgr)->xcbc_keyexp((_key), (_k1_exp), (_k2), (_k3)))
+#define IMB_DES_KEYSCHED(_mgr, _ks, _key)       \
+        ((_mgr)->des_key_sched((_ks), (_key)))
+#define IMB_SHA1_ONE_BLOCK(_mgr, _data, _digest)        \
+        ((_mgr)->sha1_one_block((_data), (_digest)))
+#define IMB_SHA224_ONE_BLOCK(_mgr, _data, _digest)      \
+        ((_mgr)->sha224_one_block((_data), (_digest)))
+#define IMB_SHA256_ONE_BLOCK(_mgr, _data, _digest)      \
+        ((_mgr)->sha256_one_block((_data), (_digest)))
+#define IMB_SHA384_ONE_BLOCK(_mgr, _data, _digest)      \
+        ((_mgr)->sha384_one_block((_data), (_digest)))
+#define IMB_SHA512_ONE_BLOCK(_mgr, _data, _digest)      \
+        ((_mgr)->sha512_one_block((_data), (_digest)))
+#define IMB_MD5_ONE_BLOCK(_mgr, _data, _digest)         \
+        ((_mgr)->md5_one_block((_data), (_digest)))
+
+/* Auxiliary functions */
+
+/**
+ * @brief DES key schedule set up
+ *
+ * \a ks buffer needs to accomodate \a DES_KEY_SCHED_SIZE (128) bytes of data.
+ *
+ * @param ks destination buffer to accomodate DES key schedule
+ * @param key a pointer to an 8 byte DES key
+ *
+ * @return Operation status
+ * @retval 0 success
+ * @retval !0 error
+ */
+IMB_DLL_EXPORT int
+des_key_schedule(uint64_t *ks, const void *key);
+
+/* SSE */
+IMB_DLL_EXPORT void sha1_one_block_sse(const void *data, void *digest);
+IMB_DLL_EXPORT void sha224_one_block_sse(const void *data, void *digest);
+IMB_DLL_EXPORT void sha256_one_block_sse(const void *data, void *digest);
+IMB_DLL_EXPORT void sha384_one_block_sse(const void *data, void *digest);
+IMB_DLL_EXPORT void sha512_one_block_sse(const void *data, void *digest);
+IMB_DLL_EXPORT void md5_one_block_sse(const void *data, void *digest);
+IMB_DLL_EXPORT void aes_keyexp_128_sse(const void *key, void *enc_exp_keys,
+                                       void *dec_exp_keys);
+IMB_DLL_EXPORT void aes_keyexp_192_sse(const void *key, void *enc_exp_keys,
+                                       void *dec_exp_keys);
+IMB_DLL_EXPORT void aes_keyexp_256_sse(const void *key, void *enc_exp_keys,
+                                       void *dec_exp_keys);
+IMB_DLL_EXPORT void aes_xcbc_expand_key_sse(const void *key, void *k1_exp,
+                                            void *k2, void *k3);
+IMB_DLL_EXPORT void aes_keyexp_128_enc_sse(const void *key,
+                                           void *enc_exp_keys);
+IMB_DLL_EXPORT void aes_keyexp_192_enc_sse(const void *key,
+                                           void *enc_exp_keys);
+IMB_DLL_EXPORT void aes_keyexp_256_enc_sse(const void *key,
+                                           void *enc_exp_keys);
+IMB_DLL_EXPORT void aes_cmac_subkey_gen_sse(const void *key_exp, void *key1,
+                                            void *key2);
+IMB_DLL_EXPORT void aes_cfb_128_one_sse(void *out, const void *in,
+                                        const void *iv, const void *keys,
+                                        uint64_t len);
+/* AVX */
+IMB_DLL_EXPORT void sha1_one_block_avx(const void *data, void *digest);
+IMB_DLL_EXPORT void sha224_one_block_avx(const void *data, void *digest);
+IMB_DLL_EXPORT void sha256_one_block_avx(const void *data, void *digest);
+IMB_DLL_EXPORT void sha384_one_block_avx(const void *data, void *digest);
+IMB_DLL_EXPORT void sha512_one_block_avx(const void *data, void *digest);
+#define md5_one_block_avx       md5_one_block_sse
+IMB_DLL_EXPORT void aes_keyexp_128_avx(const void *key, void *enc_exp_keys,
+                                       void *dec_exp_keys);
+IMB_DLL_EXPORT void aes_keyexp_192_avx(const void *key, void *enc_exp_keys,
+                                       void *dec_exp_keys);
+IMB_DLL_EXPORT void aes_keyexp_256_avx(const void *key, void *enc_exp_keys,
+                                       void *dec_exp_keys);
+IMB_DLL_EXPORT void aes_xcbc_expand_key_avx(const void *key, void *k1_exp,
+                                            void *k2, void *k3);
+IMB_DLL_EXPORT void aes_keyexp_128_enc_avx(const void *key,
+                                           void *enc_exp_keys);
+IMB_DLL_EXPORT void aes_keyexp_192_enc_avx(const void *key,
+                                           void *enc_exp_keys);
+IMB_DLL_EXPORT void aes_keyexp_256_enc_avx(const void *key,
+                                           void *enc_exp_keys);
+IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx(const void *key_exp, void *key1,
+                                            void *key2);
+IMB_DLL_EXPORT void aes_cfb_128_one_avx(void *out, const void *in,
+                                        const void *iv, const void *keys,
+                                        uint64_t len);
+
+/* AVX2 */
+#define sha1_one_block_avx2      sha1_one_block_avx
+#define sha224_one_block_avx2    sha224_one_block_avx
+#define sha256_one_block_avx2    sha256_one_block_avx
+#define sha384_one_block_avx2    sha384_one_block_avx
+#define sha512_one_block_avx2    sha512_one_block_avx
+#define md5_one_block_avx2       md5_one_block_avx
+#define aes_keyexp_128_avx2      aes_keyexp_128_avx
+#define aes_keyexp_192_avx2      aes_keyexp_192_avx
+#define aes_keyexp_256_avx2      aes_keyexp_256_avx
+#define aes_xcbc_expand_key_avx2 aes_xcbc_expand_key_avx
+#define aes_keyexp_128_enc_avx2  aes_keyexp_128_enc_avx
+#define aes_keyexp_192_enc_avx2  aes_keyexp_192_enc_avx
+#define aes_keyexp_256_enc_avx2  aes_keyexp_256_enc_avx
+#define aes_cmac_subkey_gen_avx2 aes_cmac_subkey_gen_avx
+#define aes_cfb_128_one_avx2     aes_cfb_128_one_avx
+
+/* AVX512 */
+#define sha1_one_block_avx512      sha1_one_block_avx2
+#define sha224_one_block_avx512    sha224_one_block_avx2
+#define sha256_one_block_avx512    sha256_one_block_avx2
+#define sha384_one_block_avx512    sha384_one_block_avx2
+#define sha512_one_block_avx512    sha512_one_block_avx2
+#define md5_one_block_avx512       md5_one_block_avx2
+#define aes_keyexp_128_avx512      aes_keyexp_128_avx2
+#define aes_keyexp_192_avx512      aes_keyexp_192_avx2
+#define aes_keyexp_256_avx512      aes_keyexp_256_avx2
+#define aes_xcbc_expand_key_avx512 aes_xcbc_expand_key_avx2
+#define aes_keyexp_128_enc_avx512  aes_keyexp_128_enc_avx2
+#define aes_keyexp_192_enc_avx512  aes_keyexp_192_enc_avx2
+#define aes_keyexp_256_enc_avx512  aes_keyexp_256_enc_avx2
+#define aes_cmac_subkey_gen_avx512 aes_cmac_subkey_gen_avx2
+#define aes_cfb_128_one_avx512     aes_cfb_128_one_avx2
+
+/*
+ * Direct GCM API.
+ * Note that GCM is also availabe through job API.
+ */
+#ifndef NO_GCM
+/* Authenticated Tag Length in bytes.
+ * Valid values are 16 (most likely), 12 or 8. */
+#define MAX_TAG_LEN (16)
+/*
+ * IV data is limited to 16 bytes as follows:
+ * 12 bytes is provided by an application -
+ *    pre-counter block j0: 4 byte salt (from Security Association)
+ *    concatenated with 8 byte Initialization Vector (from IPSec ESP
+ *    Payload).
+ * 4 byte value 0x00000001 is padded automatically by the library -
+ *    there is no need to add these 4 bytes on application side anymore.
+ */
+#define GCM_IV_DATA_LEN (12)
+
+#define LONGEST_TESTED_AAD_LENGTH (2 * 1024)
+
+/* Key lengths of 128 and 256 supported */
+#define GCM_128_KEY_LEN (16)
+#define GCM_192_KEY_LEN (24)
+#define GCM_256_KEY_LEN (32)
+
+#define GCM_BLOCK_LEN   16
+#define GCM_ENC_KEY_LEN 16
+#define GCM_KEY_SETS    (15) /*exp key + 14 exp round keys*/
+
+/**
+ * @brief holds intermediate key data needed to improve performance
+ *
+ * gcm_key_data hold internal key information used by gcm128, gcm192 and gcm256.
+ */
+#ifdef __WIN32
+__declspec(align(16))
+#endif /* WIN32 */
+struct gcm_key_data {
+        uint8_t expanded_keys[GCM_ENC_KEY_LEN * GCM_KEY_SETS];
+        /* storage for HashKey mod poly */
+        uint8_t shifted_hkey_1[GCM_ENC_KEY_LEN]; /* HashKey<<1 mod poly */
+        uint8_t shifted_hkey_2[GCM_ENC_KEY_LEN]; /* HashKey^2<<1 mod poly */
+        uint8_t shifted_hkey_3[GCM_ENC_KEY_LEN]; /* HashKey^3<<1 mod poly */
+        uint8_t shifted_hkey_4[GCM_ENC_KEY_LEN]; /* HashKey^4<<1 mod poly */
+        uint8_t shifted_hkey_5[GCM_ENC_KEY_LEN]; /* HashKey^5<<1 mod poly */
+        uint8_t shifted_hkey_6[GCM_ENC_KEY_LEN]; /* HashKey^6<<1 mod poly */
+        uint8_t shifted_hkey_7[GCM_ENC_KEY_LEN]; /* HashKey^7<<1 mod poly */
+        uint8_t shifted_hkey_8[GCM_ENC_KEY_LEN]; /* HashKey^8<<1 mod poly */
+        /*
+         * Storage for XOR of High 64 bits and low 64 bits of HashKey mod poly.
+         * This is needed for Karatsuba purposes.
+         */
+        uint8_t shifted_hkey_1_k[GCM_ENC_KEY_LEN]; /* HashKey<<1 mod poly */
+        uint8_t shifted_hkey_2_k[GCM_ENC_KEY_LEN]; /* HashKey^2<<1 mod poly */
+        uint8_t shifted_hkey_3_k[GCM_ENC_KEY_LEN]; /* HashKey^3<<1 mod poly */
+        uint8_t shifted_hkey_4_k[GCM_ENC_KEY_LEN]; /* HashKey^4<<1 mod poly */
+        uint8_t shifted_hkey_5_k[GCM_ENC_KEY_LEN]; /* HashKey^5<<1 mod poly */
+        uint8_t shifted_hkey_6_k[GCM_ENC_KEY_LEN]; /* HashKey^6<<1 mod poly */
+        uint8_t shifted_hkey_7_k[GCM_ENC_KEY_LEN]; /* HashKey^7<<1 mod poly */
+        uint8_t shifted_hkey_8_k[GCM_ENC_KEY_LEN]; /* HashKey^8<<1 mod poly */
+}
+#ifdef LINUX
+__attribute__((aligned(16)));
+#else
+;
+#endif
+
+/**
+ * @brief holds GCM operation context
+ */
+struct gcm_context_data {
+        /* init, update and finalize context data */
+        uint8_t  aad_hash[GCM_BLOCK_LEN];
+        uint64_t aad_length;
+        uint64_t in_length;
+        uint8_t  partial_block_enc_key[GCM_BLOCK_LEN];
+        uint8_t  orig_IV[GCM_BLOCK_LEN];
+        uint8_t  current_counter[GCM_BLOCK_LEN];
+        uint64_t  partial_block_length;
+};
+
+/**
+ * @brief GCM-AES Encryption
+ *
+ * @param key_data GCM expanded key data
+ * @param context_data GCM operation context data
+ * @param out Ciphertext output. Encrypt in-place is allowed.
+ * @param in Plaintext input.
+ * @param len Length of data in Bytes for encryption.
+ * @param iv pointer to 12 byte IV structure. Internally, library
+ *        concates 0x00000001 value to it.
+ * @param aad Additional Authentication Data (AAD).
+ * @param aad_len Length of AAD.
+ * @param auth_tag Authenticated Tag output.
+ * @param auth_tag_len Authenticated Tag Length in bytes (must be
+ *                     a multiple of 4 bytes). Valid values are
+ *                     16 (most likely), 12 or 8.
+ */
+IMB_DLL_EXPORT void
+aes_gcm_enc_128_sse(const struct gcm_key_data *key_data,
+                    struct gcm_context_data *context_data,
+                    uint8_t *out, uint8_t const *in, uint64_t len,
+                    const uint8_t *iv, uint8_t const *aad, uint64_t aad_len,
+                    uint8_t *auth_tag, uint64_t auth_tag_len);
+IMB_DLL_EXPORT void
+aes_gcm_enc_128_avx_gen2(const struct gcm_key_data *key_data,
+                         struct gcm_context_data *context_data,
+                         uint8_t *out, uint8_t const *in, uint64_t len,
+                         const uint8_t *iv,
+                         uint8_t const *aad, uint64_t aad_len,
+                         uint8_t *auth_tag, uint64_t auth_tag_len);
+IMB_DLL_EXPORT void
+aes_gcm_enc_128_avx_gen4(const struct gcm_key_data *key_data,
+                         struct gcm_context_data *context_data,
+                         uint8_t *out, uint8_t const *in, uint64_t len,
+                         const uint8_t *iv,
+                         uint8_t const *aad, uint64_t aad_len,
+                         uint8_t *auth_tag, uint64_t auth_tag_len);
+
+IMB_DLL_EXPORT void
+aes_gcm_enc_192_sse(const struct gcm_key_data *key_data,
+                    struct gcm_context_data *context_data,
+                    uint8_t *out, uint8_t const *in, uint64_t len,
+                    const uint8_t *iv, uint8_t const *aad, uint64_t aad_len,
+                    uint8_t *auth_tag, uint64_t auth_tag_len);
+IMB_DLL_EXPORT void
+aes_gcm_enc_192_avx_gen2(const struct gcm_key_data *key_data,
+                         struct gcm_context_data *context_data,
+                         uint8_t *out, uint8_t const *in, uint64_t len,
+                         const uint8_t *iv,
+                         uint8_t const *aad, uint64_t aad_len,
+                         uint8_t *auth_tag, uint64_t auth_tag_len);
+IMB_DLL_EXPORT void
+aes_gcm_enc_192_avx_gen4(const struct gcm_key_data *key_data,
+                         struct gcm_context_data *context_data,
+                         uint8_t *out, uint8_t const *in, uint64_t len,
+                         const uint8_t *iv,
+                         uint8_t const *aad, uint64_t aad_len,
+                         uint8_t *auth_tag, uint64_t auth_tag_len);
+
+IMB_DLL_EXPORT void
+aes_gcm_enc_256_sse(const struct gcm_key_data *key_data,
+                    struct gcm_context_data *context_data,
+                    uint8_t *out, uint8_t const *in, uint64_t len,
+                    const uint8_t *iv,
+                    uint8_t const *aad, uint64_t aad_len,
+                    uint8_t *auth_tag, uint64_t auth_tag_len);
+IMB_DLL_EXPORT void
+aes_gcm_enc_256_avx_gen2(const struct gcm_key_data *key_data,
+                         struct gcm_context_data *context_data,
+                         uint8_t *out, uint8_t const *in, uint64_t len,
+                         const uint8_t *iv,
+                         uint8_t const *aad, uint64_t aad_len,
+                         uint8_t *auth_tag, uint64_t auth_tag_len);
+IMB_DLL_EXPORT void
+aes_gcm_enc_256_avx_gen4(const struct gcm_key_data *key_data,
+                         struct gcm_context_data *context_data,
+                         uint8_t *out, uint8_t const *in, uint64_t len,
+                         const uint8_t *iv,
+                         uint8_t const *aad, uint64_t aad_len,
+                         uint8_t *auth_tag, uint64_t auth_tag_len);
+
+/**
+ * @brief GCM-AES Decryption
+ *
+ * @param key_data GCM expanded keys data
+ * @param context_data GCM operation context data
+ * @param out Plaintext output. Decrypt in-place is allowed.
+ * @param in Ciphertext input.
+ * @param len Length of data in Bytes for decryption.
+ * @param iv pointer to 12 byte IV structure. Internally, library
+ *        concates 0x00000001 value to it.
+ * @param aad Additional Authentication Data (AAD).
+ * @param aad_len Length of AAD.
+ * @param auth_tag Authenticated Tag output.
+ * @param auth_tag_len Authenticated Tag Length in bytes (must be
+ *                     a multiple of 4 bytes). Valid values are
+ *                     16 (most likely), 12 or 8.
+ */
+IMB_DLL_EXPORT void
+aes_gcm_dec_128_sse(const struct gcm_key_data *key_data,
+                    struct gcm_context_data *context_data,
+                    uint8_t *out, uint8_t const *in, uint64_t len,
+                    const uint8_t *iv, uint8_t const *aad, uint64_t aad_len,
+                    uint8_t *auth_tag, uint64_t auth_tag_len);
+IMB_DLL_EXPORT void
+aes_gcm_dec_128_avx_gen2(const struct gcm_key_data *key_data,
+                         struct gcm_context_data *context_data,
+                         uint8_t *out, uint8_t const *in, uint64_t len,
+                         const uint8_t *iv,
+                         uint8_t const *aad, uint64_t aad_len,
+                         uint8_t *auth_tag, uint64_t auth_tag_len);
+IMB_DLL_EXPORT void
+aes_gcm_dec_128_avx_gen4(const struct gcm_key_data *key_data,
+                         struct gcm_context_data *context_data,
+                         uint8_t *out, uint8_t const *in, uint64_t len,
+                         const uint8_t *iv,
+                         uint8_t const *aad, uint64_t aad_len,
+                         uint8_t *auth_tag, uint64_t auth_tag_len);
+
+IMB_DLL_EXPORT void
+aes_gcm_dec_192_sse(const struct gcm_key_data *key_data,
+                    struct gcm_context_data *context_data,
+                    uint8_t *out, uint8_t const *in, uint64_t len,
+                    const uint8_t *iv, uint8_t const *aad, uint64_t aad_len,
+                    uint8_t *auth_tag, uint64_t auth_tag_len);
+IMB_DLL_EXPORT void
+aes_gcm_dec_192_avx_gen2(const struct gcm_key_data *key_data,
+                         struct gcm_context_data *context_data,
+                         uint8_t *out, uint8_t const *in, uint64_t len,
+                         const uint8_t *iv,
+                         uint8_t const *aad, uint64_t aad_len,
+                         uint8_t *auth_tag, uint64_t auth_tag_len);
+IMB_DLL_EXPORT void
+aes_gcm_dec_192_avx_gen4(const struct gcm_key_data *key_data,
+                         struct gcm_context_data *context_data,
+                         uint8_t *out, uint8_t const *in, uint64_t len,
+                         const uint8_t *iv,
+                         uint8_t const *aad, uint64_t aad_len,
+                         uint8_t *auth_tag, uint64_t auth_tag_len);
+
+IMB_DLL_EXPORT void
+aes_gcm_dec_256_sse(const struct gcm_key_data *key_data,
+                    struct gcm_context_data *context_data,
+                    uint8_t *out, uint8_t const *in, uint64_t len,
+                    const uint8_t *iv, uint8_t const *aad, uint64_t aad_len,
+                    uint8_t *auth_tag, uint64_t auth_tag_len);
+IMB_DLL_EXPORT void
+aes_gcm_dec_256_avx_gen2(const struct gcm_key_data *key_data,
+                         struct gcm_context_data *context_data,
+                         uint8_t *out, uint8_t const *in, uint64_t len,
+                         const uint8_t *iv,
+                         uint8_t const *aad, uint64_t aad_len,
+                         uint8_t *auth_tag, uint64_t auth_tag_len);
+IMB_DLL_EXPORT void
+aes_gcm_dec_256_avx_gen4(const struct gcm_key_data *key_data,
+                         struct gcm_context_data *context_data,
+                         uint8_t *out, uint8_t const *in, uint64_t len,
+                         const uint8_t *iv,
+                         uint8_t const *aad, uint64_t aad_len,
+                         uint8_t *auth_tag, uint64_t auth_tag_len);
+
+/**
+ * @brief Start a AES-GCM Encryption message
+ *
+ * @param key_data GCM expanded key data
+ * @param context_data GCM operation context data
+ * @param iv pointer to 12 byte IV structure. Internally, library
+ *        concates 0x00000001 value to it.
+ * @param aad Additional Authentication Data (AAD).
+ * @param aad_len Length of AAD.
+ *
+ */
+IMB_DLL_EXPORT void
+aes_gcm_init_128_sse(const struct gcm_key_data *key_data,
+                     struct gcm_context_data *context_data,
+                     const uint8_t *iv, uint8_t const *aad, uint64_t aad_len);
+IMB_DLL_EXPORT void
+aes_gcm_init_128_avx_gen2(const struct gcm_key_data *key_data,
+                          struct gcm_context_data *context_data,
+                          const uint8_t *iv,
+                          uint8_t const *aad, uint64_t aad_len);
+IMB_DLL_EXPORT void
+aes_gcm_init_128_avx_gen4(const struct gcm_key_data *key_data,
+                          struct gcm_context_data *context_data,
+                          const uint8_t *iv,
+                          uint8_t const *aad, uint64_t aad_len);
+
+IMB_DLL_EXPORT void
+aes_gcm_init_192_sse(const struct gcm_key_data *key_data,
+                     struct gcm_context_data *context_data,
+                     const uint8_t *iv, uint8_t const *aad, uint64_t aad_len);
+IMB_DLL_EXPORT void
+aes_gcm_init_192_avx_gen2(const struct gcm_key_data *key_data,
+                          struct gcm_context_data *context_data,
+                          const uint8_t *iv,
+                          uint8_t const *aad, uint64_t aad_len);
+IMB_DLL_EXPORT void
+aes_gcm_init_192_avx_gen4(const struct gcm_key_data *key_data,
+                          struct gcm_context_data *context_data,
+                          const uint8_t *iv,
+                          uint8_t const *aad, uint64_t aad_len);
+
+IMB_DLL_EXPORT void
+aes_gcm_init_256_sse(const struct gcm_key_data *key_data,
+                     struct gcm_context_data *context_data,
+                     const uint8_t *iv, uint8_t const *aad, uint64_t aad_len);
+IMB_DLL_EXPORT void
+aes_gcm_init_256_avx_gen2(const struct gcm_key_data *key_data,
+                          struct gcm_context_data *context_data,
+                          const uint8_t *iv,
+                          uint8_t const *aad, uint64_t aad_len);
+IMB_DLL_EXPORT void
+aes_gcm_init_256_avx_gen4(const struct gcm_key_data *key_data,
+                          struct gcm_context_data *context_data,
+                          const uint8_t *iv,
+                          uint8_t const *aad, uint64_t aad_len);
+
+/**
+ * @brief encrypt a block of a AES-GCM Encryption message
+ *
+ * @param key_data GCM expanded key data
+ * @param context_data GCM operation context data
+ * @param out Ciphertext output. Encrypt in-place is allowed.
+ * @param in Plaintext input.
+ * @param len Length of data in Bytes for decryption.
+ */
+IMB_DLL_EXPORT void
+aes_gcm_enc_128_update_sse(const struct gcm_key_data *key_data,
+                           struct gcm_context_data *context_data,
+                           uint8_t *out, const uint8_t *in, uint64_t len);
+IMB_DLL_EXPORT void
+aes_gcm_enc_128_update_avx_gen2(const struct gcm_key_data *key_data,
+                                struct gcm_context_data *context_data,
+                                uint8_t *out, const uint8_t *in, uint64_t len);
+IMB_DLL_EXPORT void
+aes_gcm_enc_128_update_avx_gen4(const struct gcm_key_data *key_data,
+                                struct gcm_context_data *context_data,
+                                uint8_t *out, const uint8_t *in, uint64_t len);
+
+IMB_DLL_EXPORT void
+aes_gcm_enc_192_update_sse(const struct gcm_key_data *key_data,
+                           struct gcm_context_data *context_data,
+                           uint8_t *out, const uint8_t *in, uint64_t len);
+IMB_DLL_EXPORT void
+aes_gcm_enc_192_update_avx_gen2(const struct gcm_key_data *key_data,
+                                struct gcm_context_data *context_data,
+                                uint8_t *out, const uint8_t *in, uint64_t len);
+IMB_DLL_EXPORT void
+aes_gcm_enc_192_update_avx_gen4(const struct gcm_key_data *key_data,
+                                struct gcm_context_data *context_data,
+                                uint8_t *out, const uint8_t *in, uint64_t len);
+
+IMB_DLL_EXPORT void
+aes_gcm_enc_256_update_sse(const struct gcm_key_data *key_data,
+                           struct gcm_context_data *context_data,
+                           uint8_t *out, const uint8_t *in, uint64_t len);
+IMB_DLL_EXPORT void
+aes_gcm_enc_256_update_avx_gen2(const struct gcm_key_data *key_data,
+                                struct gcm_context_data *context_data,
+                                uint8_t *out, const uint8_t *in, uint64_t len);
+IMB_DLL_EXPORT void
+aes_gcm_enc_256_update_avx_gen4(const struct gcm_key_data *key_data,
+                                struct gcm_context_data *context_data,
+                                uint8_t *out, const uint8_t *in, uint64_t len);
+
+/**
+ * @brief decrypt a block of a AES-GCM Encryption message
+ *
+ * @param key_data GCM expanded key data
+ * @param context_data GCM operation context data
+ * @param out Plaintext output. Decrypt in-place is allowed.
+ * @param in Ciphertext input.
+ * @param len Length of data in Bytes for decryption.
+ */
+IMB_DLL_EXPORT void
+aes_gcm_dec_128_update_sse(const struct gcm_key_data *key_data,
+                           struct gcm_context_data *context_data,
+                           uint8_t *out, const uint8_t *in, uint64_t len);
+IMB_DLL_EXPORT void
+aes_gcm_dec_128_update_avx_gen2(const struct gcm_key_data *key_data,
+                                struct gcm_context_data *context_data,
+                                uint8_t *out, const uint8_t *in, uint64_t len);
+IMB_DLL_EXPORT void
+aes_gcm_dec_128_update_avx_gen4(const struct gcm_key_data *key_data,
+                                struct gcm_context_data *context_data,
+                                uint8_t *out, const uint8_t *in, uint64_t len);
+
+IMB_DLL_EXPORT void
+aes_gcm_dec_192_update_sse(const struct gcm_key_data *key_data,
+                           struct gcm_context_data *context_data,
+                           uint8_t *out, const uint8_t *in, uint64_t len);
+IMB_DLL_EXPORT void
+aes_gcm_dec_192_update_avx_gen2(const struct gcm_key_data *key_data,
+                                struct gcm_context_data *context_data,
+                                uint8_t *out, const uint8_t *in, uint64_t len);
+IMB_DLL_EXPORT void
+aes_gcm_dec_192_update_avx_gen4(const struct gcm_key_data *key_data,
+                                struct gcm_context_data *context_data,
+                                uint8_t *out, const uint8_t *in, uint64_t len);
+
+IMB_DLL_EXPORT void
+aes_gcm_dec_256_update_sse(const struct gcm_key_data *key_data,
+                           struct gcm_context_data *context_data,
+                           uint8_t *out, const uint8_t *in, uint64_t len);
+IMB_DLL_EXPORT void
+aes_gcm_dec_256_update_avx_gen2(const struct gcm_key_data *key_data,
+                                struct gcm_context_data *context_data,
+                                uint8_t *out, const uint8_t *in, uint64_t len);
+IMB_DLL_EXPORT void
+aes_gcm_dec_256_update_avx_gen4(const struct gcm_key_data *key_data,
+                                struct gcm_context_data *context_data,
+                                uint8_t *out, const uint8_t *in, uint64_t len);
+
+/**
+ * @brief End encryption of a AES-GCM Encryption message
+ *
+ * @param key_data GCM expanded key data
+ * @param context_data GCM operation context data
+ * @param auth_tag Authenticated Tag output.
+ * @param auth_tag_len Authenticated Tag Length in bytes (must be
+ *                     a multiple of 4 bytes). Valid values are
+ *                     16 (most likely), 12 or 8.
+ */
+IMB_DLL_EXPORT void
+aes_gcm_enc_128_finalize_sse(const struct gcm_key_data *key_data,
+                             struct gcm_context_data *context_data,
+                             uint8_t *auth_tag, uint64_t auth_tag_len);
+IMB_DLL_EXPORT void
+aes_gcm_enc_128_finalize_avx_gen2(const struct gcm_key_data *key_data,
+                                  struct gcm_context_data *context_data,
+                                  uint8_t *auth_tag, uint64_t auth_tag_len);
+IMB_DLL_EXPORT void
+aes_gcm_enc_128_finalize_avx_gen4(const struct gcm_key_data *key_data,
+                                  struct gcm_context_data *context_data,
+                                  uint8_t *auth_tag, uint64_t auth_tag_len);
+
+IMB_DLL_EXPORT void
+aes_gcm_enc_192_finalize_sse(const struct gcm_key_data *key_data,
+                             struct gcm_context_data *context_data,
+                             uint8_t *auth_tag, uint64_t auth_tag_len);
+IMB_DLL_EXPORT void
+aes_gcm_enc_192_finalize_avx_gen2(const struct gcm_key_data *key_data,
+                                  struct gcm_context_data *context_data,
+                                  uint8_t *auth_tag, uint64_t auth_tag_len);
+IMB_DLL_EXPORT void
+aes_gcm_enc_192_finalize_avx_gen4(const struct gcm_key_data *key_data,
+                                  struct gcm_context_data *context_data,
+                                  uint8_t *auth_tag, uint64_t auth_tag_len);
+
+IMB_DLL_EXPORT void
+aes_gcm_enc_256_finalize_sse(const struct gcm_key_data *key_data,
+                             struct gcm_context_data *context_data,
+                             uint8_t *auth_tag, uint64_t auth_tag_len);
+IMB_DLL_EXPORT void
+aes_gcm_enc_256_finalize_avx_gen2(const struct gcm_key_data *key_data,
+                                  struct gcm_context_data *context_data,
+                                  uint8_t *auth_tag, uint64_t auth_tag_len);
+IMB_DLL_EXPORT void
+aes_gcm_enc_256_finalize_avx_gen4(const struct gcm_key_data *key_data,
+                                  struct gcm_context_data *context_data,
+                                  uint8_t *auth_tag, uint64_t auth_tag_len);
+
+/**
+ * @brief End decryption of a AES-GCM Encryption message
+ *
+ * @param key_data GCM expanded key data
+ * @param context_data GCM operation context data
+ * @param auth_tag Authenticated Tag output.
+ * @param auth_tag_len Authenticated Tag Length in bytes (must be
+ *                     a multiple of 4 bytes). Valid values are
+ *                     16 (most likely), 12 or 8.
+ */
+IMB_DLL_EXPORT void
+aes_gcm_dec_128_finalize_sse(const struct gcm_key_data *key_data,
+                             struct gcm_context_data *context_data,
+                             uint8_t *auth_tag, uint64_t auth_tag_len);
+IMB_DLL_EXPORT void
+aes_gcm_dec_128_finalize_avx_gen2(const struct gcm_key_data *key_data,
+                                  struct gcm_context_data *context_data,
+                                  uint8_t *auth_tag, uint64_t auth_tag_len);
+IMB_DLL_EXPORT void
+aes_gcm_dec_128_finalize_avx_gen4(const struct gcm_key_data *key_data,
+                                  struct gcm_context_data *context_data,
+                                  uint8_t *auth_tag, uint64_t auth_tag_len);
+
+IMB_DLL_EXPORT void
+aes_gcm_dec_192_finalize_sse(const struct gcm_key_data *key_data,
+                             struct gcm_context_data *context_data,
+                             uint8_t *auth_tag, uint64_t auth_tag_len);
+IMB_DLL_EXPORT void
+aes_gcm_dec_192_finalize_avx_gen2(const struct gcm_key_data *key_data,
+                                  struct gcm_context_data *context_data,
+                                  uint8_t *auth_tag, uint64_t auth_tag_len);
+IMB_DLL_EXPORT void
+aes_gcm_dec_192_finalize_avx_gen4(const struct gcm_key_data *key_data,
+                                  struct gcm_context_data *context_data,
+                                  uint8_t *auth_tag, uint64_t auth_tag_len);
+
+IMB_DLL_EXPORT void
+aes_gcm_dec_256_finalize_sse(const struct gcm_key_data *key_data,
+                             struct gcm_context_data *context_data,
+                             uint8_t *auth_tag, uint64_t auth_tag_len);
+IMB_DLL_EXPORT void
+aes_gcm_dec_256_finalize_avx_gen2(const struct gcm_key_data *key_data,
+                                  struct gcm_context_data *context_data,
+                                  uint8_t *auth_tag, uint64_t auth_tag_len);
+IMB_DLL_EXPORT void
+aes_gcm_dec_256_finalize_avx_gen4(const struct gcm_key_data *key_data,
+                                  struct gcm_context_data *context_data,
+                                  uint8_t *auth_tag, uint64_t auth_tag_len);
+
+/**
+ * @brief Precomputation of HashKey constants
+ *
+ * Precomputation of HashKey<<1 mod poly constants (shifted_hkey_X and
+ * shifted_hkey_X_k).
+ *
+ * @param gdata GCM context data
+ */
+IMB_DLL_EXPORT void aes_gcm_precomp_128_sse(struct gcm_key_data *key_data);
+IMB_DLL_EXPORT void aes_gcm_precomp_128_avx_gen2(struct gcm_key_data *key_data);
+IMB_DLL_EXPORT void aes_gcm_precomp_128_avx_gen4(struct gcm_key_data *key_data);
+
+IMB_DLL_EXPORT void aes_gcm_precomp_192_sse(struct gcm_key_data *key_data);
+IMB_DLL_EXPORT void aes_gcm_precomp_192_avx_gen2(struct gcm_key_data *key_data);
+IMB_DLL_EXPORT void aes_gcm_precomp_192_avx_gen4(struct gcm_key_data *key_data);
+
+IMB_DLL_EXPORT void aes_gcm_precomp_256_sse(struct gcm_key_data *key_data);
+IMB_DLL_EXPORT void aes_gcm_precomp_256_avx_gen2(struct gcm_key_data *key_data);
+IMB_DLL_EXPORT void aes_gcm_precomp_256_avx_gen4(struct gcm_key_data *key_data);
+
+/**
+ * @brief Pre-processes GCM key data
+ *
+ * Prefills the gcm key data with key values for each round and
+ * the initial sub hash key for tag encoding
+ *
+ * @param key pointer to key data
+ * @param key_data GCM expanded key data
+ *
+ */
+__forceinline
+void aes_gcm_pre_128_sse(const void *key, struct gcm_key_data *key_data)
+{
+        aes_keyexp_128_enc_sse(key, key_data->expanded_keys);
+        aes_gcm_precomp_128_sse(key_data);
+}
+__forceinline
+void aes_gcm_pre_128_avx_gen2(const void *key, struct gcm_key_data *key_data)
+{
+        aes_keyexp_128_enc_avx(key, key_data->expanded_keys);
+        aes_gcm_precomp_128_avx_gen2(key_data);
+}
+__forceinline
+void aes_gcm_pre_128_avx_gen4(const void *key, struct gcm_key_data *key_data)
+{
+        aes_keyexp_128_enc_avx2(key, key_data->expanded_keys);
+        aes_gcm_precomp_128_avx_gen4(key_data);
+}
+
+__forceinline
+void aes_gcm_pre_192_sse(const void *key, struct gcm_key_data *key_data)
+{
+        aes_keyexp_192_enc_sse(key, key_data->expanded_keys);
+        aes_gcm_precomp_192_sse(key_data);
+}
+__forceinline
+void aes_gcm_pre_192_avx_gen2(const void *key, struct gcm_key_data *key_data)
+{
+        aes_keyexp_192_enc_avx(key, key_data->expanded_keys);
+        aes_gcm_precomp_192_avx_gen2(key_data);
+}
+__forceinline
+void aes_gcm_pre_192_avx_gen4(const void *key, struct gcm_key_data *key_data)
+{
+        aes_keyexp_192_enc_avx2(key, key_data->expanded_keys);
+        aes_gcm_precomp_192_avx_gen4(key_data);
+}
+
+__forceinline
+void aes_gcm_pre_256_sse(const void *key, struct gcm_key_data *key_data)
+{
+        struct gcm_key_data tmp;
+
+        aes_keyexp_256_sse(key, key_data->expanded_keys, tmp.expanded_keys);
+        aes_gcm_precomp_256_sse(key_data);
+}
+__forceinline
+void aes_gcm_pre_256_avx_gen2(const void *key, struct gcm_key_data *key_data)
+{
+        aes_keyexp_256_enc_avx(key, key_data->expanded_keys);
+        aes_gcm_precomp_256_avx_gen2(key_data);
+}
+__forceinline
+void aes_gcm_pre_256_avx_gen4(const void *key, struct gcm_key_data *key_data)
+{
+        aes_keyexp_256_enc_avx2(key, key_data->expanded_keys);
+        aes_gcm_precomp_256_avx_gen4(key_data);
+}
+#endif /* !NO_GCM */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* IMB_IPSEC_MB_H */