summaryrefslogtreecommitdiffstats
path: root/src/spdk/lib/util
diff options
context:
space:
mode:
Diffstat (limited to 'src/spdk/lib/util')
-rw-r--r--src/spdk/lib/util/Makefile47
-rw-r--r--src/spdk/lib/util/base64.c262
-rw-r--r--src/spdk/lib/util/base64_neon.c225
-rw-r--r--src/spdk/lib/util/bit_array.c363
-rw-r--r--src/spdk/lib/util/cpuset.c336
-rw-r--r--src/spdk/lib/util/crc16.c668
-rw-r--r--src/spdk/lib/util/crc32.c95
-rw-r--r--src/spdk/lib/util/crc32_ieee.c49
-rw-r--r--src/spdk/lib/util/crc32c.c133
-rw-r--r--src/spdk/lib/util/dif.c1999
-rw-r--r--src/spdk/lib/util/fd.c103
-rw-r--r--src/spdk/lib/util/file.c71
-rw-r--r--src/spdk/lib/util/iov.c111
-rw-r--r--src/spdk/lib/util/math.c69
-rw-r--r--src/spdk/lib/util/pipe.c246
-rw-r--r--src/spdk/lib/util/spdk_util.map128
-rw-r--r--src/spdk/lib/util/strerror_tls.c43
-rw-r--r--src/spdk/lib/util/string.c476
-rw-r--r--src/spdk/lib/util/util_internal.h77
-rw-r--r--src/spdk/lib/util/uuid.c73
20 files changed, 5574 insertions, 0 deletions
diff --git a/src/spdk/lib/util/Makefile b/src/spdk/lib/util/Makefile
new file mode 100644
index 000000000..23f8db6d0
--- /dev/null
+++ b/src/spdk/lib/util/Makefile
@@ -0,0 +1,47 @@
+#
+# BSD LICENSE
+#
+# Copyright (c) Intel Corporation.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Intel Corporation nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+SO_VER := 2
+SO_MINOR := 0
+
+C_SRCS = base64.c bit_array.c cpuset.c crc16.c crc32.c crc32c.c crc32_ieee.c \
+ dif.c fd.c file.c iov.c math.c pipe.c strerror_tls.c string.c uuid.c
+LIBNAME = util
+LOCAL_SYS_LIBS = -luuid
+
+SPDK_MAP_FILE = $(abspath $(CURDIR)/spdk_util.map)
+
+include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk
diff --git a/src/spdk/lib/util/base64.c b/src/spdk/lib/util/base64.c
new file mode 100644
index 000000000..adc5e15da
--- /dev/null
+++ b/src/spdk/lib/util/base64.c
@@ -0,0 +1,262 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+#include "spdk/endian.h"
+#include "spdk/base64.h"
+
+#ifdef __aarch64__
+#include "base64_neon.c"
+#endif
+
+#define BASE64_ENC_BITMASK 0x3FUL
+#define BASE64_PADDING_CHAR '='
+
+static const char base64_enc_table[] =
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+ "abcdefghijklmnopqrstuvwxyz"
+ "0123456789+/";
+
+static const char base64_urfsafe_enc_table[] =
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+ "abcdefghijklmnopqrstuvwxyz"
+ "0123456789-_";
+
+static const uint8_t
+base64_dec_table[] = {
+ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255, 255, 255, 63,
+ 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, 255, 255, 255, 255,
+ 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
+ 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 255, 255, 255, 255, 255,
+ 255, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40,
+ 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 255, 255, 255, 255, 255,
+ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+};
+
+static const uint8_t
+base64_urlsafe_dec_table[] = {
+ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255, 255,
+ 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, 255, 255, 255, 255,
+ 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
+ 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 255, 255, 255, 255, 63,
+ 255, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40,
+ 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 255, 255, 255, 255, 255,
+ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+};
+
+static int
+base64_encode(char *dst, const char *enc_table, const void *src, size_t src_len)
+{
+ uint32_t raw_u32;
+
+ if (!dst || !src || src_len <= 0) {
+ return -EINVAL;
+ }
+
+#ifdef __aarch64__
+ base64_encode_neon64(&dst, enc_table, &src, &src_len);
+#endif
+
+ while (src_len >= 4) {
+ raw_u32 = from_be32(src);
+
+ *dst++ = enc_table[(raw_u32 >> 26) & BASE64_ENC_BITMASK];
+ *dst++ = enc_table[(raw_u32 >> 20) & BASE64_ENC_BITMASK];
+ *dst++ = enc_table[(raw_u32 >> 14) & BASE64_ENC_BITMASK];
+ *dst++ = enc_table[(raw_u32 >> 8) & BASE64_ENC_BITMASK];
+
+ src_len -= 3;
+ src += 3;
+ }
+
+ if (src_len == 0) {
+ goto out;
+ }
+
+ raw_u32 = 0;
+ memcpy(&raw_u32, src, src_len);
+ raw_u32 = from_be32(&raw_u32);
+
+ *dst++ = enc_table[(raw_u32 >> 26) & BASE64_ENC_BITMASK];
+ *dst++ = enc_table[(raw_u32 >> 20) & BASE64_ENC_BITMASK];
+ *dst++ = (src_len >= 2) ? enc_table[(raw_u32 >> 14) & BASE64_ENC_BITMASK] : BASE64_PADDING_CHAR;
+ *dst++ = (src_len == 3) ? enc_table[(raw_u32 >> 8) & BASE64_ENC_BITMASK] : BASE64_PADDING_CHAR;
+
+out:
+ *dst = '\0';
+
+ return 0;
+}
+
+int
+spdk_base64_encode(char *dst, const void *src, size_t src_len)
+{
+ return base64_encode(dst, base64_enc_table, src, src_len);
+}
+
+int
+spdk_base64_urlsafe_encode(char *dst, const void *src, size_t src_len)
+{
+ return base64_encode(dst, base64_urfsafe_enc_table, src, src_len);
+}
+
+#ifdef __aarch64__
+static int
+base64_decode(void *dst, size_t *_dst_len, const uint8_t *dec_table,
+ const uint8_t *dec_table_opt, const char *src)
+#else
+static int
+base64_decode(void *dst, size_t *_dst_len, const uint8_t *dec_table, const char *src)
+#endif
+{
+ size_t src_strlen;
+ size_t tail_len = 0;
+ const uint8_t *src_in;
+ uint32_t tmp[4];
+ int i;
+
+ if (!src) {
+ return -EINVAL;
+ }
+
+ src_strlen = strlen(src);
+
+ /* strlen of src should be 4n */
+ if (src_strlen == 0 || src_strlen % 4 != 0) {
+ return -EINVAL;
+ }
+
+ /* Consider Base64 padding, it at most has 2 padding characters. */
+ for (i = 0; i < 2; i++) {
+ if (src[src_strlen - 1] != BASE64_PADDING_CHAR) {
+ break;
+ }
+ src_strlen--;
+ }
+
+ /* strlen of src without padding shouldn't be 4n+1 */
+ if (src_strlen == 0 || src_strlen % 4 == 1) {
+ return -EINVAL;
+ }
+
+ if (_dst_len) {
+ *_dst_len = spdk_base64_get_decoded_len(src_strlen);
+ }
+
+ /* If dst is NULL, the client is only concerned w/ _dst_len, return */
+ if (!dst) {
+ return 0;
+ }
+
+ src_in = (const uint8_t *) src;
+
+#ifdef __aarch64__
+ base64_decode_neon64(&dst, dec_table_opt, &src_in, &src_strlen);
+
+ if (src_strlen == 0) {
+ return 0;
+ }
+#endif
+
+ /* space of dst can be used by to_be32 */
+ while (src_strlen > 4) {
+ tmp[0] = dec_table[*src_in++];
+ tmp[1] = dec_table[*src_in++];
+ tmp[2] = dec_table[*src_in++];
+ tmp[3] = dec_table[*src_in++];
+
+ if (tmp[0] == 255 || tmp[1] == 255 || tmp[2] == 255 || tmp[3] == 255) {
+ return -EINVAL;
+ }
+
+ to_be32(dst, tmp[3] << 8 | tmp[2] << 14 | tmp[1] << 20 | tmp[0] << 26);
+
+ dst += 3;
+ src_strlen -= 4;
+ }
+
+ /* space of dst is not enough to be used by to_be32 */
+ tmp[0] = dec_table[src_in[0]];
+ tmp[1] = dec_table[src_in[1]];
+ tmp[2] = (src_strlen >= 3) ? dec_table[src_in[2]] : 0;
+ tmp[3] = (src_strlen == 4) ? dec_table[src_in[3]] : 0;
+ tail_len = src_strlen - 1;
+
+ if (tmp[0] == 255 || tmp[1] == 255 || tmp[2] == 255 || tmp[3] == 255) {
+ return -EINVAL;
+ }
+
+ to_be32(&tmp[3], tmp[3] << 8 | tmp[2] << 14 | tmp[1] << 20 | tmp[0] << 26);
+ memcpy(dst, (uint8_t *)&tmp[3], tail_len);
+
+ return 0;
+}
+
+int
+spdk_base64_decode(void *dst, size_t *dst_len, const char *src)
+{
+#ifdef __aarch64__
+ return base64_decode(dst, dst_len, base64_dec_table, base64_dec_table_neon64, src);
+#else
+ return base64_decode(dst, dst_len, base64_dec_table, src);
+#endif
+}
+
+int
+spdk_base64_urlsafe_decode(void *dst, size_t *dst_len, const char *src)
+{
+#ifdef __aarch64__
+ return base64_decode(dst, dst_len, base64_urlsafe_dec_table, base64_urlsafe_dec_table_neon64,
+ src);
+#else
+ return base64_decode(dst, dst_len, base64_urlsafe_dec_table, src);
+#endif
+}
diff --git a/src/spdk/lib/util/base64_neon.c b/src/spdk/lib/util/base64_neon.c
new file mode 100644
index 000000000..971cff06c
--- /dev/null
+++ b/src/spdk/lib/util/base64_neon.c
@@ -0,0 +1,225 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) 2005-2007, Nick Galbreath
+ * Copyright (c) 2013-2017, Alfred Klomp
+ * Copyright (c) 2015-2017, Wojciech Mula
+ * Copyright (c) 2016-2017, Matthieu Darbois
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __aarch64__
+#error Unsupported hardware
+#endif
+
+#include "spdk/stdinc.h"
+/*
+ * Encoding
+ * Use a 64-byte lookup to do the encoding.
+ * Reuse existing base64_dec_table and base64_dec_table.
+
+ * Decoding
+ * The input consists of five valid character sets in the Base64 alphabet,
+ * which we need to map back to the 6-bit values they represent.
+ * There are three ranges, two singles, and then there's the rest.
+ *
+ * LUT1[0-63] = base64_dec_table_neon64[0-63]
+ * LUT2[0-63] = base64_dec_table_neon64[64-127]
+ * # From To LUT Characters
+ * 1 [0..42] [255] #1 invalid input
+ * 2 [43] [62] #1 +
+ * 3 [44..46] [255] #1 invalid input
+ * 4 [47] [63] #1 /
+ * 5 [48..57] [52..61] #1 0..9
+ * 6 [58..63] [255] #1 invalid input
+ * 7 [64] [255] #2 invalid input
+ * 8 [65..90] [0..25] #2 A..Z
+ * 9 [91..96] [255] #2 invalid input
+ * 10 [97..122] [26..51] #2 a..z
+ * 11 [123..126] [255] #2 invalid input
+ * (12) Everything else => invalid input
+ */
+static const uint8_t base64_dec_table_neon64[] = {
+ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255, 255, 255, 63,
+ 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, 255, 255, 255, 255,
+ 0, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13,
+ 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 255, 255, 255, 255,
+ 255, 255, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39,
+ 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 255, 255, 255, 255
+};
+
+/*
+ * LUT1[0-63] = base64_urlsafe_dec_table_neon64[0-63]
+ * LUT2[0-63] = base64_urlsafe_dec_table_neon64[64-127]
+ * # From To LUT Characters
+ * 1 [0..44] [255] #1 invalid input
+ * 2 [45] [62] #1 -
+ * 3 [46..47] [255] #1 invalid input
+ * 5 [48..57] [52..61] #1 0..9
+ * 6 [58..63] [255] #1 invalid input
+ * 7 [64] [255] #2 invalid input
+ * 8 [65..90] [0..25] #2 A..Z
+ * 9 [91..94] [255] #2 invalid input
+ * 10 [95] [63] #2 _
+ * 11 [96] [255] #2 invalid input
+ * 12 [97..122] [26..51] #2 a..z
+ * 13 [123..126] [255] #2 invalid input
+ * (14) Everything else => invalid input
+ */
+static const uint8_t base64_urlsafe_dec_table_neon64[] = {
+ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255, 255,
+ 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, 255, 255, 255, 255,
+ 0, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13,
+ 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 255, 255, 255, 255,
+ 63, 255, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39,
+ 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 255, 255, 255, 255
+};
+
+#include <arm_neon.h>
+#define CMPGT(s,n) vcgtq_u8((s), vdupq_n_u8(n))
+
+static inline uint8x16x4_t
+load_64byte_table(const uint8_t *p)
+{
+ uint8x16x4_t ret;
+ ret.val[0] = vld1q_u8(p + 0);
+ ret.val[1] = vld1q_u8(p + 16);
+ ret.val[2] = vld1q_u8(p + 32);
+ ret.val[3] = vld1q_u8(p + 48);
+ return ret;
+}
+
+static void
+base64_encode_neon64(char **dst, const char *enc_table, const void **src, size_t *src_len)
+{
+ const uint8x16x4_t tbl_enc = load_64byte_table(enc_table);
+
+ while (*src_len >= 48) {
+ uint8x16x3_t str;
+ uint8x16x4_t res;
+
+ /* Load 48 bytes and deinterleave */
+ str = vld3q_u8((uint8_t *)*src);
+
+ /* Divide bits of three input bytes over four output bytes and clear top two bits */
+ res.val[0] = vshrq_n_u8(str.val[0], 2);
+ res.val[1] = vandq_u8(vorrq_u8(vshrq_n_u8(str.val[1], 4), vshlq_n_u8(str.val[0], 4)),
+ vdupq_n_u8(0x3F));
+ res.val[2] = vandq_u8(vorrq_u8(vshrq_n_u8(str.val[2], 6), vshlq_n_u8(str.val[1], 2)),
+ vdupq_n_u8(0x3F));
+ res.val[3] = vandq_u8(str.val[2], vdupq_n_u8(0x3F));
+
+ /*
+ * The bits have now been shifted to the right locations;
+ * translate their values 0..63 to the Base64 alphabet.
+ * Use a 64-byte table lookup:
+ */
+ res.val[0] = vqtbl4q_u8(tbl_enc, res.val[0]);
+ res.val[1] = vqtbl4q_u8(tbl_enc, res.val[1]);
+ res.val[2] = vqtbl4q_u8(tbl_enc, res.val[2]);
+ res.val[3] = vqtbl4q_u8(tbl_enc, res.val[3]);
+
+ /* Interleave and store result */
+ vst4q_u8((uint8_t *)*dst, res);
+
+ *src += 48; /* 3 * 16 bytes of input */
+ *dst += 64; /* 4 * 16 bytes of output */
+ *src_len -= 48;
+ }
+}
+
+static void
+base64_decode_neon64(void **dst, const uint8_t *dec_table_neon64, const uint8_t **src,
+ size_t *src_len)
+{
+ /*
+ * First LUT tbl_dec1 will use VTBL instruction (out of range indices are set to 0 in destination).
+ * Second LUT tbl_dec2 will use VTBX instruction (out of range indices will be unchanged in destination).
+ * Input [64..126] will be mapped to index [1..63] in tb1_dec2. Index 0 means that value comes from tb1_dec1.
+ */
+ const uint8x16x4_t tbl_dec1 = load_64byte_table(dec_table_neon64);
+ const uint8x16x4_t tbl_dec2 = load_64byte_table(dec_table_neon64 + 64);
+ const uint8x16_t offset = vdupq_n_u8(63U);
+
+ while (*src_len >= 64) {
+
+ uint8x16x4_t dec1, dec2;
+ uint8x16x3_t dec;
+
+ /* Load 64 bytes and deinterleave */
+ uint8x16x4_t str = vld4q_u8((uint8_t *)*src);
+
+ /* Get indices for 2nd LUT */
+ dec2.val[0] = vqsubq_u8(str.val[0], offset);
+ dec2.val[1] = vqsubq_u8(str.val[1], offset);
+ dec2.val[2] = vqsubq_u8(str.val[2], offset);
+ dec2.val[3] = vqsubq_u8(str.val[3], offset);
+
+ /* Get values from 1st LUT */
+ dec1.val[0] = vqtbl4q_u8(tbl_dec1, str.val[0]);
+ dec1.val[1] = vqtbl4q_u8(tbl_dec1, str.val[1]);
+ dec1.val[2] = vqtbl4q_u8(tbl_dec1, str.val[2]);
+ dec1.val[3] = vqtbl4q_u8(tbl_dec1, str.val[3]);
+
+ /* Get values from 2nd LUT */
+ dec2.val[0] = vqtbx4q_u8(dec2.val[0], tbl_dec2, dec2.val[0]);
+ dec2.val[1] = vqtbx4q_u8(dec2.val[1], tbl_dec2, dec2.val[1]);
+ dec2.val[2] = vqtbx4q_u8(dec2.val[2], tbl_dec2, dec2.val[2]);
+ dec2.val[3] = vqtbx4q_u8(dec2.val[3], tbl_dec2, dec2.val[3]);
+
+ /* Get final values */
+ str.val[0] = vorrq_u8(dec1.val[0], dec2.val[0]);
+ str.val[1] = vorrq_u8(dec1.val[1], dec2.val[1]);
+ str.val[2] = vorrq_u8(dec1.val[2], dec2.val[2]);
+ str.val[3] = vorrq_u8(dec1.val[3], dec2.val[3]);
+
+ /* Check for invalid input, any value larger than 63 */
+ uint8x16_t classified = CMPGT(str.val[0], 63);
+ classified = vorrq_u8(classified, CMPGT(str.val[1], 63));
+ classified = vorrq_u8(classified, CMPGT(str.val[2], 63));
+ classified = vorrq_u8(classified, CMPGT(str.val[3], 63));
+
+ /* check that all bits are zero */
+ if (vmaxvq_u8(classified) != 0U) {
+ break;
+ }
+
+ /* Compress four bytes into three */
+ dec.val[0] = vorrq_u8(vshlq_n_u8(str.val[0], 2), vshrq_n_u8(str.val[1], 4));
+ dec.val[1] = vorrq_u8(vshlq_n_u8(str.val[1], 4), vshrq_n_u8(str.val[2], 2));
+ dec.val[2] = vorrq_u8(vshlq_n_u8(str.val[2], 6), str.val[3]);
+
+ /* Interleave and store decoded result */
+ vst3q_u8((uint8_t *)*dst, dec);
+
+ *src += 64;
+ *dst += 48;
+ *src_len -= 64;
+ }
+}
diff --git a/src/spdk/lib/util/bit_array.c b/src/spdk/lib/util/bit_array.c
new file mode 100644
index 000000000..43c1a4d9b
--- /dev/null
+++ b/src/spdk/lib/util/bit_array.c
@@ -0,0 +1,363 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+
+#include "spdk/bit_array.h"
+#include "spdk/env.h"
+
+#include "spdk/likely.h"
+#include "spdk/util.h"
+
+typedef uint64_t spdk_bit_array_word;
+#define SPDK_BIT_ARRAY_WORD_TZCNT(x) (__builtin_ctzll(x))
+#define SPDK_BIT_ARRAY_WORD_POPCNT(x) (__builtin_popcountll(x))
+#define SPDK_BIT_ARRAY_WORD_C(x) ((spdk_bit_array_word)(x))
+#define SPDK_BIT_ARRAY_WORD_BYTES sizeof(spdk_bit_array_word)
+#define SPDK_BIT_ARRAY_WORD_BITS (SPDK_BIT_ARRAY_WORD_BYTES * 8)
+#define SPDK_BIT_ARRAY_WORD_INDEX_SHIFT spdk_u32log2(SPDK_BIT_ARRAY_WORD_BITS)
+#define SPDK_BIT_ARRAY_WORD_INDEX_MASK ((1u << SPDK_BIT_ARRAY_WORD_INDEX_SHIFT) - 1)
+
+struct spdk_bit_array {
+ uint32_t bit_count;
+ spdk_bit_array_word words[];
+};
+
+struct spdk_bit_array *
+spdk_bit_array_create(uint32_t num_bits)
+{
+ struct spdk_bit_array *ba = NULL;
+
+ spdk_bit_array_resize(&ba, num_bits);
+
+ return ba;
+}
+
+void
+spdk_bit_array_free(struct spdk_bit_array **bap)
+{
+ struct spdk_bit_array *ba;
+
+ if (!bap) {
+ return;
+ }
+
+ ba = *bap;
+ *bap = NULL;
+ spdk_free(ba);
+}
+
+static inline uint32_t
+bit_array_word_count(uint32_t num_bits)
+{
+ return (num_bits + SPDK_BIT_ARRAY_WORD_BITS - 1) >> SPDK_BIT_ARRAY_WORD_INDEX_SHIFT;
+}
+
+static inline spdk_bit_array_word
+bit_array_word_mask(uint32_t num_bits)
+{
+ assert(num_bits < SPDK_BIT_ARRAY_WORD_BITS);
+ return (SPDK_BIT_ARRAY_WORD_C(1) << num_bits) - 1;
+}
+
+int
+spdk_bit_array_resize(struct spdk_bit_array **bap, uint32_t num_bits)
+{
+ struct spdk_bit_array *new_ba;
+ uint32_t old_word_count, new_word_count;
+ size_t new_size;
+
+ /*
+ * Max number of bits allowed is UINT32_MAX - 1, because we use UINT32_MAX to denote
+ * when a set or cleared bit cannot be found.
+ */
+ if (!bap || num_bits == UINT32_MAX) {
+ return -EINVAL;
+ }
+
+ new_word_count = bit_array_word_count(num_bits);
+ new_size = offsetof(struct spdk_bit_array, words) + new_word_count * SPDK_BIT_ARRAY_WORD_BYTES;
+
+ /*
+ * Always keep one extra word with a 0 and a 1 past the actual required size so that the
+ * find_first functions can just keep going until they match.
+ */
+ new_size += SPDK_BIT_ARRAY_WORD_BYTES;
+
+ new_ba = (struct spdk_bit_array *)spdk_realloc(*bap, new_size, 64);
+ if (!new_ba) {
+ return -ENOMEM;
+ }
+
+ /*
+ * Set up special extra word (see above comment about find_first_clear).
+ *
+ * This is set to 0b10 so that find_first_clear will find a 0 at the very first
+ * bit past the end of the buffer, and find_first_set will find a 1 at the next bit
+ * past that.
+ */
+ new_ba->words[new_word_count] = 0x2;
+
+ if (*bap == NULL) {
+ old_word_count = 0;
+ new_ba->bit_count = 0;
+ } else {
+ old_word_count = bit_array_word_count(new_ba->bit_count);
+ }
+
+ if (new_word_count > old_word_count) {
+ /* Zero out new entries */
+ memset(&new_ba->words[old_word_count], 0,
+ (new_word_count - old_word_count) * SPDK_BIT_ARRAY_WORD_BYTES);
+ } else if (new_word_count == old_word_count && num_bits < new_ba->bit_count) {
+ /* Make sure any existing partial last word is cleared beyond the new num_bits. */
+ uint32_t last_word_bits;
+ spdk_bit_array_word mask;
+
+ last_word_bits = num_bits & SPDK_BIT_ARRAY_WORD_INDEX_MASK;
+ mask = bit_array_word_mask(last_word_bits);
+ new_ba->words[old_word_count - 1] &= mask;
+ }
+
+ new_ba->bit_count = num_bits;
+ *bap = new_ba;
+ return 0;
+}
+
+uint32_t
+spdk_bit_array_capacity(const struct spdk_bit_array *ba)
+{
+ return ba->bit_count;
+}
+
+static inline int
+bit_array_get_word(const struct spdk_bit_array *ba, uint32_t bit_index,
+ uint32_t *word_index, uint32_t *word_bit_index)
+{
+ if (spdk_unlikely(bit_index >= ba->bit_count)) {
+ return -EINVAL;
+ }
+
+ *word_index = bit_index >> SPDK_BIT_ARRAY_WORD_INDEX_SHIFT;
+ *word_bit_index = bit_index & SPDK_BIT_ARRAY_WORD_INDEX_MASK;
+
+ return 0;
+}
+
+bool
+spdk_bit_array_get(const struct spdk_bit_array *ba, uint32_t bit_index)
+{
+ uint32_t word_index, word_bit_index;
+
+ if (bit_array_get_word(ba, bit_index, &word_index, &word_bit_index)) {
+ return false;
+ }
+
+ return (ba->words[word_index] >> word_bit_index) & 1U;
+}
+
+int
+spdk_bit_array_set(struct spdk_bit_array *ba, uint32_t bit_index)
+{
+ uint32_t word_index, word_bit_index;
+
+ if (bit_array_get_word(ba, bit_index, &word_index, &word_bit_index)) {
+ return -EINVAL;
+ }
+
+ ba->words[word_index] |= (SPDK_BIT_ARRAY_WORD_C(1) << word_bit_index);
+ return 0;
+}
+
+void
+spdk_bit_array_clear(struct spdk_bit_array *ba, uint32_t bit_index)
+{
+ uint32_t word_index, word_bit_index;
+
+ if (bit_array_get_word(ba, bit_index, &word_index, &word_bit_index)) {
+ /*
+ * Clearing past the end of the bit array is a no-op, since bit past the end
+ * are implicitly 0.
+ */
+ return;
+ }
+
+ ba->words[word_index] &= ~(SPDK_BIT_ARRAY_WORD_C(1) << word_bit_index);
+}
+
+static inline uint32_t
+bit_array_find_first(const struct spdk_bit_array *ba, uint32_t start_bit_index,
+ spdk_bit_array_word xor_mask)
+{
+ uint32_t word_index, first_word_bit_index;
+ spdk_bit_array_word word, first_word_mask;
+ const spdk_bit_array_word *words, *cur_word;
+
+ if (spdk_unlikely(start_bit_index >= ba->bit_count)) {
+ return ba->bit_count;
+ }
+
+ word_index = start_bit_index >> SPDK_BIT_ARRAY_WORD_INDEX_SHIFT;
+ words = ba->words;
+ cur_word = &words[word_index];
+
+ /*
+ * Special case for first word: skip start_bit_index % SPDK_BIT_ARRAY_WORD_BITS bits
+ * within the first word.
+ */
+ first_word_bit_index = start_bit_index & SPDK_BIT_ARRAY_WORD_INDEX_MASK;
+ first_word_mask = bit_array_word_mask(first_word_bit_index);
+
+ word = (*cur_word ^ xor_mask) & ~first_word_mask;
+
+ /*
+ * spdk_bit_array_resize() guarantees that an extra word with a 1 and a 0 will always be
+ * at the end of the words[] array, so just keep going until a word matches.
+ */
+ while (word == 0) {
+ word = *++cur_word ^ xor_mask;
+ }
+
+ return ((uintptr_t)cur_word - (uintptr_t)words) * 8 + SPDK_BIT_ARRAY_WORD_TZCNT(word);
+}
+
+
+uint32_t
+spdk_bit_array_find_first_set(const struct spdk_bit_array *ba, uint32_t start_bit_index)
+{
+ uint32_t bit_index;
+
+ bit_index = bit_array_find_first(ba, start_bit_index, 0);
+
+ /*
+ * If we ran off the end of the array and found the 1 bit in the extra word,
+ * return UINT32_MAX to indicate no actual 1 bits were found.
+ */
+ if (bit_index >= ba->bit_count) {
+ bit_index = UINT32_MAX;
+ }
+
+ return bit_index;
+}
+
+uint32_t
+spdk_bit_array_find_first_clear(const struct spdk_bit_array *ba, uint32_t start_bit_index)
+{
+ uint32_t bit_index;
+
+ bit_index = bit_array_find_first(ba, start_bit_index, SPDK_BIT_ARRAY_WORD_C(-1));
+
+ /*
+ * If we ran off the end of the array and found the 0 bit in the extra word,
+ * return UINT32_MAX to indicate no actual 0 bits were found.
+ */
+ if (bit_index >= ba->bit_count) {
+ bit_index = UINT32_MAX;
+ }
+
+ return bit_index;
+}
+
+uint32_t
+spdk_bit_array_count_set(const struct spdk_bit_array *ba)
+{
+ const spdk_bit_array_word *cur_word = ba->words;
+ uint32_t word_count = bit_array_word_count(ba->bit_count);
+ uint32_t set_count = 0;
+
+ while (word_count--) {
+ /*
+ * No special treatment is needed for the last (potentially partial) word, since
+ * spdk_bit_array_resize() makes sure the bits past bit_count are cleared.
+ */
+ set_count += SPDK_BIT_ARRAY_WORD_POPCNT(*cur_word++);
+ }
+
+ return set_count;
+}
+
+uint32_t
+spdk_bit_array_count_clear(const struct spdk_bit_array *ba)
+{
+ return ba->bit_count - spdk_bit_array_count_set(ba);
+}
+
+void
+spdk_bit_array_store_mask(const struct spdk_bit_array *ba, void *mask)
+{
+ uint32_t size, i;
+ uint32_t num_bits = spdk_bit_array_capacity(ba);
+
+ size = num_bits / CHAR_BIT;
+ memcpy(mask, ba->words, size);
+
+ for (i = 0; i < num_bits % CHAR_BIT; i++) {
+ if (spdk_bit_array_get(ba, i + size * CHAR_BIT)) {
+ ((uint8_t *)mask)[size] |= (1U << i);
+ } else {
+ ((uint8_t *)mask)[size] &= ~(1U << i);
+ }
+ }
+}
+
+void
+spdk_bit_array_load_mask(struct spdk_bit_array *ba, const void *mask)
+{
+ uint32_t size, i;
+ uint32_t num_bits = spdk_bit_array_capacity(ba);
+
+ size = num_bits / CHAR_BIT;
+ memcpy(ba->words, mask, size);
+
+ for (i = 0; i < num_bits % CHAR_BIT; i++) {
+ if (((uint8_t *)mask)[size] & (1U << i)) {
+ spdk_bit_array_set(ba, i + size * CHAR_BIT);
+ } else {
+ spdk_bit_array_clear(ba, i + size * CHAR_BIT);
+ }
+ }
+}
+
+void
+spdk_bit_array_clear_mask(struct spdk_bit_array *ba)
+{
+ uint32_t size, i;
+ uint32_t num_bits = spdk_bit_array_capacity(ba);
+
+ size = num_bits / CHAR_BIT;
+ memset(ba->words, 0, size);
+
+ for (i = 0; i < num_bits % CHAR_BIT; i++) {
+ spdk_bit_array_clear(ba, i + size * CHAR_BIT);
+ }
+}
diff --git a/src/spdk/lib/util/cpuset.c b/src/spdk/lib/util/cpuset.c
new file mode 100644
index 000000000..8d7c8dc89
--- /dev/null
+++ b/src/spdk/lib/util/cpuset.c
@@ -0,0 +1,336 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/cpuset.h"
+#include "spdk/log.h"
+
+struct spdk_cpuset *
+spdk_cpuset_alloc(void)
+{
+ return (struct spdk_cpuset *)calloc(sizeof(struct spdk_cpuset), 1);
+}
+
+void
+spdk_cpuset_free(struct spdk_cpuset *set)
+{
+ free(set);
+}
+
+bool
+spdk_cpuset_equal(const struct spdk_cpuset *set1, const struct spdk_cpuset *set2)
+{
+ assert(set1 != NULL);
+ assert(set2 != NULL);
+ return memcmp(set1->cpus, set2->cpus, sizeof(set2->cpus)) == 0;
+}
+
+void
+spdk_cpuset_copy(struct spdk_cpuset *dst, const struct spdk_cpuset *src)
+{
+ assert(dst != NULL);
+ assert(src != NULL);
+ memcpy(&dst->cpus, &src->cpus, sizeof(src->cpus));
+}
+
+void
+spdk_cpuset_negate(struct spdk_cpuset *set)
+{
+ unsigned int i;
+ assert(set != NULL);
+ for (i = 0; i < sizeof(set->cpus); i++) {
+ set->cpus[i] = ~set->cpus[i];
+ }
+}
+
+void
+spdk_cpuset_and(struct spdk_cpuset *dst, const struct spdk_cpuset *src)
+{
+ unsigned int i;
+ assert(dst != NULL);
+ assert(src != NULL);
+ for (i = 0; i < sizeof(src->cpus); i++) {
+ dst->cpus[i] &= src->cpus[i];
+ }
+}
+
+void
+spdk_cpuset_or(struct spdk_cpuset *dst, const struct spdk_cpuset *src)
+{
+ unsigned int i;
+ assert(dst != NULL);
+ assert(src != NULL);
+ for (i = 0; i < sizeof(src->cpus); i++) {
+ dst->cpus[i] |= src->cpus[i];
+ }
+}
+
+void
+spdk_cpuset_xor(struct spdk_cpuset *dst, const struct spdk_cpuset *src)
+{
+ unsigned int i;
+ assert(dst != NULL);
+ assert(src != NULL);
+ for (i = 0; i < sizeof(src->cpus); i++) {
+ dst->cpus[i] ^= src->cpus[i];
+ }
+}
+
+void
+spdk_cpuset_zero(struct spdk_cpuset *set)
+{
+ assert(set != NULL);
+ memset(set->cpus, 0, sizeof(set->cpus));
+}
+
+void
+spdk_cpuset_set_cpu(struct spdk_cpuset *set, uint32_t cpu, bool state)
+{
+ assert(set != NULL);
+ assert(cpu < sizeof(set->cpus) * 8);
+ if (state) {
+ set->cpus[cpu / 8] |= (1U << (cpu % 8));
+ } else {
+ set->cpus[cpu / 8] &= ~(1U << (cpu % 8));
+ }
+}
+
+bool
+spdk_cpuset_get_cpu(const struct spdk_cpuset *set, uint32_t cpu)
+{
+ assert(set != NULL);
+ assert(cpu < sizeof(set->cpus) * 8);
+ return (set->cpus[cpu / 8] >> (cpu % 8)) & 1U;
+}
+
+uint32_t
+spdk_cpuset_count(const struct spdk_cpuset *set)
+{
+ uint32_t count = 0;
+ uint8_t n;
+ unsigned int i;
+ for (i = 0; i < sizeof(set->cpus); i++) {
+ n = set->cpus[i];
+ while (n) {
+ n &= (n - 1);
+ count++;
+ }
+ }
+ return count;
+}
+
+const char *
+spdk_cpuset_fmt(struct spdk_cpuset *set)
+{
+ uint32_t lcore, lcore_max = 0;
+ int val, i, n;
+ char *ptr;
+ static const char *hex = "0123456789abcdef";
+
+ assert(set != NULL);
+
+ for (lcore = 0; lcore < sizeof(set->cpus) * 8; lcore++) {
+ if (spdk_cpuset_get_cpu(set, lcore)) {
+ lcore_max = lcore;
+ }
+ }
+
+ ptr = set->str;
+ n = lcore_max / 8;
+ val = set->cpus[n];
+
+ /* Store first number only if it is not leading zero */
+ if ((val & 0xf0) != 0) {
+ *(ptr++) = hex[(val & 0xf0) >> 4];
+ }
+ *(ptr++) = hex[val & 0x0f];
+
+ for (i = n - 1; i >= 0; i--) {
+ val = set->cpus[i];
+ *(ptr++) = hex[(val & 0xf0) >> 4];
+ *(ptr++) = hex[val & 0x0f];
+ }
+ *ptr = '\0';
+
+ return set->str;
+}
+
+static int
+hex_value(uint8_t c)
+{
+#define V(x, y) [x] = y + 1
+ static const int8_t val[256] = {
+ V('0', 0), V('1', 1), V('2', 2), V('3', 3), V('4', 4),
+ V('5', 5), V('6', 6), V('7', 7), V('8', 8), V('9', 9),
+ V('A', 0xA), V('B', 0xB), V('C', 0xC), V('D', 0xD), V('E', 0xE), V('F', 0xF),
+ V('a', 0xA), V('b', 0xB), V('c', 0xC), V('d', 0xD), V('e', 0xE), V('f', 0xF),
+ };
+#undef V
+
+ return val[c] - 1;
+}
+
+static int
+parse_list(const char *mask, struct spdk_cpuset *set)
+{
+ char *end;
+ const char *ptr = mask;
+ uint32_t lcore;
+ uint32_t lcore_min, lcore_max;
+
+ spdk_cpuset_zero(set);
+ lcore_min = UINT32_MAX;
+
+ ptr++;
+ end = (char *)ptr;
+ do {
+ while (isblank(*ptr)) {
+ ptr++;
+ }
+ if (*ptr == '\0' || *ptr == ']' || *ptr == '-' || *ptr == ',') {
+ goto invalid_character;
+ }
+
+ errno = 0;
+ lcore = strtoul(ptr, &end, 10);
+ if (errno) {
+ SPDK_ERRLOG("Conversion of core mask in '%s' failed\n", mask);
+ return -1;
+ }
+
+ if (lcore >= sizeof(set->cpus) * 8) {
+ SPDK_ERRLOG("Core number %" PRIu32 " is out of range in '%s'\n", lcore, mask);
+ return -1;
+ }
+
+ while (isblank(*end)) {
+ end++;
+ }
+
+ if (*end == '-') {
+ lcore_min = lcore;
+ } else if (*end == ',' || *end == ']') {
+ lcore_max = lcore;
+ if (lcore_min == UINT32_MAX) {
+ lcore_min = lcore;
+ }
+ if (lcore_min > lcore_max) {
+ SPDK_ERRLOG("Invalid range of CPUs (%" PRIu32 " > %" PRIu32 ")\n",
+ lcore_min, lcore_max);
+ return -1;
+ }
+ for (lcore = lcore_min; lcore <= lcore_max; lcore++) {
+ spdk_cpuset_set_cpu(set, lcore, true);
+ }
+ lcore_min = UINT32_MAX;
+ } else {
+ goto invalid_character;
+ }
+
+ ptr = end + 1;
+
+ } while (*end != ']');
+
+ return 0;
+
+invalid_character:
+ if (*end == '\0') {
+ SPDK_ERRLOG("Unexpected end of core list '%s'\n", mask);
+ } else {
+ SPDK_ERRLOG("Parsing of core list '%s' failed on character '%c'\n", mask, *end);
+ }
+ return -1;
+}
+
+static int
+parse_mask(const char *mask, struct spdk_cpuset *set, size_t len)
+{
+ int i, j;
+ char c;
+ int val;
+ uint32_t lcore = 0;
+
+ if (mask[0] == '0' && (mask[1] == 'x' || mask[1] == 'X')) {
+ mask += 2;
+ len -= 2;
+ }
+
+ spdk_cpuset_zero(set);
+ for (i = len - 1; i >= 0; i--) {
+ c = mask[i];
+ val = hex_value(c);
+ if (val < 0) {
+ /* Invalid character */
+ SPDK_ERRLOG("Invalid character in core mask '%s' (%c)\n", mask, c);
+ return -1;
+ }
+ for (j = 0; j < 4 && lcore < sizeof(set->cpus); j++, lcore++) {
+ if ((1 << j) & val) {
+ spdk_cpuset_set_cpu(set, lcore, true);
+ }
+ }
+ }
+
+ return 0;
+}
+
+int
+spdk_cpuset_parse(struct spdk_cpuset *set, const char *mask)
+{
+ int ret;
+ size_t len;
+
+ if (mask == NULL || set == NULL) {
+ return -1;
+ }
+
+ while (isblank(*mask)) {
+ mask++;
+ }
+
+ len = strlen(mask);
+ while (len > 0 && isblank(mask[len - 1])) {
+ len--;
+ }
+
+ if (len == 0) {
+ return -1;
+ }
+
+ if (mask[0] == '[') {
+ ret = parse_list(mask, set);
+ } else {
+ ret = parse_mask(mask, set, len);
+ }
+
+ return ret;
+}
diff --git a/src/spdk/lib/util/crc16.c b/src/spdk/lib/util/crc16.c
new file mode 100644
index 000000000..2ba168c4b
--- /dev/null
+++ b/src/spdk/lib/util/crc16.c
@@ -0,0 +1,668 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/crc16.h"
+#include "spdk/config.h"
+
+/*
+ * Use Intelligent Storage Acceleration Library for line speed CRC
+ */
+
+#ifdef SPDK_CONFIG_ISAL
+#include "isa-l/include/crc.h"
+
+uint16_t
+spdk_crc16_t10dif(uint16_t init_crc, const void *buf, size_t len)
+{
+ return (crc16_t10dif(init_crc, buf, len));
+}
+
+uint16_t
+spdk_crc16_t10dif_copy(uint16_t init_crc, uint8_t *dst, uint8_t *src,
+ size_t len)
+{
+ return (crc16_t10dif_copy(init_crc, dst, src, len));
+}
+
+#else
+/*
+ * Use table-driven (somewhat faster) CRC
+ */
+
+/*
+ * Static tables used for the table_driven implementation.
+ */
+
+static const uint16_t crc_table_fast[16][256] = {
+ {
+ 0x0000u, 0x8BB7u, 0x9CD9u, 0x176Eu, 0xB205u, 0x39B2u, 0x2EDCu, 0xA56Bu,
+ 0xEFBDu, 0x640Au, 0x7364u, 0xF8D3u, 0x5DB8u, 0xD60Fu, 0xC161u, 0x4AD6u,
+ 0x54CDu, 0xDF7Au, 0xC814u, 0x43A3u, 0xE6C8u, 0x6D7Fu, 0x7A11u, 0xF1A6u,
+ 0xBB70u, 0x30C7u, 0x27A9u, 0xAC1Eu, 0x0975u, 0x82C2u, 0x95ACu, 0x1E1Bu,
+ 0xA99Au, 0x222Du, 0x3543u, 0xBEF4u, 0x1B9Fu, 0x9028u, 0x8746u, 0x0CF1u,
+ 0x4627u, 0xCD90u, 0xDAFEu, 0x5149u, 0xF422u, 0x7F95u, 0x68FBu, 0xE34Cu,
+ 0xFD57u, 0x76E0u, 0x618Eu, 0xEA39u, 0x4F52u, 0xC4E5u, 0xD38Bu, 0x583Cu,
+ 0x12EAu, 0x995Du, 0x8E33u, 0x0584u, 0xA0EFu, 0x2B58u, 0x3C36u, 0xB781u,
+ 0xD883u, 0x5334u, 0x445Au, 0xCFEDu, 0x6A86u, 0xE131u, 0xF65Fu, 0x7DE8u,
+ 0x373Eu, 0xBC89u, 0xABE7u, 0x2050u, 0x853Bu, 0x0E8Cu, 0x19E2u, 0x9255u,
+ 0x8C4Eu, 0x07F9u, 0x1097u, 0x9B20u, 0x3E4Bu, 0xB5FCu, 0xA292u, 0x2925u,
+ 0x63F3u, 0xE844u, 0xFF2Au, 0x749Du, 0xD1F6u, 0x5A41u, 0x4D2Fu, 0xC698u,
+ 0x7119u, 0xFAAEu, 0xEDC0u, 0x6677u, 0xC31Cu, 0x48ABu, 0x5FC5u, 0xD472u,
+ 0x9EA4u, 0x1513u, 0x027Du, 0x89CAu, 0x2CA1u, 0xA716u, 0xB078u, 0x3BCFu,
+ 0x25D4u, 0xAE63u, 0xB90Du, 0x32BAu, 0x97D1u, 0x1C66u, 0x0B08u, 0x80BFu,
+ 0xCA69u, 0x41DEu, 0x56B0u, 0xDD07u, 0x786Cu, 0xF3DBu, 0xE4B5u, 0x6F02u,
+ 0x3AB1u, 0xB106u, 0xA668u, 0x2DDFu, 0x88B4u, 0x0303u, 0x146Du, 0x9FDAu,
+ 0xD50Cu, 0x5EBBu, 0x49D5u, 0xC262u, 0x6709u, 0xECBEu, 0xFBD0u, 0x7067u,
+ 0x6E7Cu, 0xE5CBu, 0xF2A5u, 0x7912u, 0xDC79u, 0x57CEu, 0x40A0u, 0xCB17u,
+ 0x81C1u, 0x0A76u, 0x1D18u, 0x96AFu, 0x33C4u, 0xB873u, 0xAF1Du, 0x24AAu,
+ 0x932Bu, 0x189Cu, 0x0FF2u, 0x8445u, 0x212Eu, 0xAA99u, 0xBDF7u, 0x3640u,
+ 0x7C96u, 0xF721u, 0xE04Fu, 0x6BF8u, 0xCE93u, 0x4524u, 0x524Au, 0xD9FDu,
+ 0xC7E6u, 0x4C51u, 0x5B3Fu, 0xD088u, 0x75E3u, 0xFE54u, 0xE93Au, 0x628Du,
+ 0x285Bu, 0xA3ECu, 0xB482u, 0x3F35u, 0x9A5Eu, 0x11E9u, 0x0687u, 0x8D30u,
+ 0xE232u, 0x6985u, 0x7EEBu, 0xF55Cu, 0x5037u, 0xDB80u, 0xCCEEu, 0x4759u,
+ 0x0D8Fu, 0x8638u, 0x9156u, 0x1AE1u, 0xBF8Au, 0x343Du, 0x2353u, 0xA8E4u,
+ 0xB6FFu, 0x3D48u, 0x2A26u, 0xA191u, 0x04FAu, 0x8F4Du, 0x9823u, 0x1394u,
+ 0x5942u, 0xD2F5u, 0xC59Bu, 0x4E2Cu, 0xEB47u, 0x60F0u, 0x779Eu, 0xFC29u,
+ 0x4BA8u, 0xC01Fu, 0xD771u, 0x5CC6u, 0xF9ADu, 0x721Au, 0x6574u, 0xEEC3u,
+ 0xA415u, 0x2FA2u, 0x38CCu, 0xB37Bu, 0x1610u, 0x9DA7u, 0x8AC9u, 0x017Eu,
+ 0x1F65u, 0x94D2u, 0x83BCu, 0x080Bu, 0xAD60u, 0x26D7u, 0x31B9u, 0xBA0Eu,
+ 0xF0D8u, 0x7B6Fu, 0x6C01u, 0xE7B6u, 0x42DDu, 0xC96Au, 0xDE04u, 0x55B3u
+ },
+ {
+ 0x0000u, 0x7562u, 0xEAC4u, 0x9FA6u, 0x5E3Fu, 0x2B5Du, 0xB4FBu, 0xC199u,
+ 0xBC7Eu, 0xC91Cu, 0x56BAu, 0x23D8u, 0xE241u, 0x9723u, 0x0885u, 0x7DE7u,
+ 0xF34Bu, 0x8629u, 0x198Fu, 0x6CEDu, 0xAD74u, 0xD816u, 0x47B0u, 0x32D2u,
+ 0x4F35u, 0x3A57u, 0xA5F1u, 0xD093u, 0x110Au, 0x6468u, 0xFBCEu, 0x8EACu,
+ 0x6D21u, 0x1843u, 0x87E5u, 0xF287u, 0x331Eu, 0x467Cu, 0xD9DAu, 0xACB8u,
+ 0xD15Fu, 0xA43Du, 0x3B9Bu, 0x4EF9u, 0x8F60u, 0xFA02u, 0x65A4u, 0x10C6u,
+ 0x9E6Au, 0xEB08u, 0x74AEu, 0x01CCu, 0xC055u, 0xB537u, 0x2A91u, 0x5FF3u,
+ 0x2214u, 0x5776u, 0xC8D0u, 0xBDB2u, 0x7C2Bu, 0x0949u, 0x96EFu, 0xE38Du,
+ 0xDA42u, 0xAF20u, 0x3086u, 0x45E4u, 0x847Du, 0xF11Fu, 0x6EB9u, 0x1BDBu,
+ 0x663Cu, 0x135Eu, 0x8CF8u, 0xF99Au, 0x3803u, 0x4D61u, 0xD2C7u, 0xA7A5u,
+ 0x2909u, 0x5C6Bu, 0xC3CDu, 0xB6AFu, 0x7736u, 0x0254u, 0x9DF2u, 0xE890u,
+ 0x9577u, 0xE015u, 0x7FB3u, 0x0AD1u, 0xCB48u, 0xBE2Au, 0x218Cu, 0x54EEu,
+ 0xB763u, 0xC201u, 0x5DA7u, 0x28C5u, 0xE95Cu, 0x9C3Eu, 0x0398u, 0x76FAu,
+ 0x0B1Du, 0x7E7Fu, 0xE1D9u, 0x94BBu, 0x5522u, 0x2040u, 0xBFE6u, 0xCA84u,
+ 0x4428u, 0x314Au, 0xAEECu, 0xDB8Eu, 0x1A17u, 0x6F75u, 0xF0D3u, 0x85B1u,
+ 0xF856u, 0x8D34u, 0x1292u, 0x67F0u, 0xA669u, 0xD30Bu, 0x4CADu, 0x39CFu,
+ 0x3F33u, 0x4A51u, 0xD5F7u, 0xA095u, 0x610Cu, 0x146Eu, 0x8BC8u, 0xFEAAu,
+ 0x834Du, 0xF62Fu, 0x6989u, 0x1CEBu, 0xDD72u, 0xA810u, 0x37B6u, 0x42D4u,
+ 0xCC78u, 0xB91Au, 0x26BCu, 0x53DEu, 0x9247u, 0xE725u, 0x7883u, 0x0DE1u,
+ 0x7006u, 0x0564u, 0x9AC2u, 0xEFA0u, 0x2E39u, 0x5B5Bu, 0xC4FDu, 0xB19Fu,
+ 0x5212u, 0x2770u, 0xB8D6u, 0xCDB4u, 0x0C2Du, 0x794Fu, 0xE6E9u, 0x938Bu,
+ 0xEE6Cu, 0x9B0Eu, 0x04A8u, 0x71CAu, 0xB053u, 0xC531u, 0x5A97u, 0x2FF5u,
+ 0xA159u, 0xD43Bu, 0x4B9Du, 0x3EFFu, 0xFF66u, 0x8A04u, 0x15A2u, 0x60C0u,
+ 0x1D27u, 0x6845u, 0xF7E3u, 0x8281u, 0x4318u, 0x367Au, 0xA9DCu, 0xDCBEu,
+ 0xE571u, 0x9013u, 0x0FB5u, 0x7AD7u, 0xBB4Eu, 0xCE2Cu, 0x518Au, 0x24E8u,
+ 0x590Fu, 0x2C6Du, 0xB3CBu, 0xC6A9u, 0x0730u, 0x7252u, 0xEDF4u, 0x9896u,
+ 0x163Au, 0x6358u, 0xFCFEu, 0x899Cu, 0x4805u, 0x3D67u, 0xA2C1u, 0xD7A3u,
+ 0xAA44u, 0xDF26u, 0x4080u, 0x35E2u, 0xF47Bu, 0x8119u, 0x1EBFu, 0x6BDDu,
+ 0x8850u, 0xFD32u, 0x6294u, 0x17F6u, 0xD66Fu, 0xA30Du, 0x3CABu, 0x49C9u,
+ 0x342Eu, 0x414Cu, 0xDEEAu, 0xAB88u, 0x6A11u, 0x1F73u, 0x80D5u, 0xF5B7u,
+ 0x7B1Bu, 0x0E79u, 0x91DFu, 0xE4BDu, 0x2524u, 0x5046u, 0xCFE0u, 0xBA82u,
+ 0xC765u, 0xB207u, 0x2DA1u, 0x58C3u, 0x995Au, 0xEC38u, 0x739Eu, 0x06FCu
+ },
+ {
+ 0x0000u, 0x7E66u, 0xFCCCu, 0x82AAu, 0x722Fu, 0x0C49u, 0x8EE3u, 0xF085u,
+ 0xE45Eu, 0x9A38u, 0x1892u, 0x66F4u, 0x9671u, 0xE817u, 0x6ABDu, 0x14DBu,
+ 0x430Bu, 0x3D6Du, 0xBFC7u, 0xC1A1u, 0x3124u, 0x4F42u, 0xCDE8u, 0xB38Eu,
+ 0xA755u, 0xD933u, 0x5B99u, 0x25FFu, 0xD57Au, 0xAB1Cu, 0x29B6u, 0x57D0u,
+ 0x8616u, 0xF870u, 0x7ADAu, 0x04BCu, 0xF439u, 0x8A5Fu, 0x08F5u, 0x7693u,
+ 0x6248u, 0x1C2Eu, 0x9E84u, 0xE0E2u, 0x1067u, 0x6E01u, 0xECABu, 0x92CDu,
+ 0xC51Du, 0xBB7Bu, 0x39D1u, 0x47B7u, 0xB732u, 0xC954u, 0x4BFEu, 0x3598u,
+ 0x2143u, 0x5F25u, 0xDD8Fu, 0xA3E9u, 0x536Cu, 0x2D0Au, 0xAFA0u, 0xD1C6u,
+ 0x879Bu, 0xF9FDu, 0x7B57u, 0x0531u, 0xF5B4u, 0x8BD2u, 0x0978u, 0x771Eu,
+ 0x63C5u, 0x1DA3u, 0x9F09u, 0xE16Fu, 0x11EAu, 0x6F8Cu, 0xED26u, 0x9340u,
+ 0xC490u, 0xBAF6u, 0x385Cu, 0x463Au, 0xB6BFu, 0xC8D9u, 0x4A73u, 0x3415u,
+ 0x20CEu, 0x5EA8u, 0xDC02u, 0xA264u, 0x52E1u, 0x2C87u, 0xAE2Du, 0xD04Bu,
+ 0x018Du, 0x7FEBu, 0xFD41u, 0x8327u, 0x73A2u, 0x0DC4u, 0x8F6Eu, 0xF108u,
+ 0xE5D3u, 0x9BB5u, 0x191Fu, 0x6779u, 0x97FCu, 0xE99Au, 0x6B30u, 0x1556u,
+ 0x4286u, 0x3CE0u, 0xBE4Au, 0xC02Cu, 0x30A9u, 0x4ECFu, 0xCC65u, 0xB203u,
+ 0xA6D8u, 0xD8BEu, 0x5A14u, 0x2472u, 0xD4F7u, 0xAA91u, 0x283Bu, 0x565Du,
+ 0x8481u, 0xFAE7u, 0x784Du, 0x062Bu, 0xF6AEu, 0x88C8u, 0x0A62u, 0x7404u,
+ 0x60DFu, 0x1EB9u, 0x9C13u, 0xE275u, 0x12F0u, 0x6C96u, 0xEE3Cu, 0x905Au,
+ 0xC78Au, 0xB9ECu, 0x3B46u, 0x4520u, 0xB5A5u, 0xCBC3u, 0x4969u, 0x370Fu,
+ 0x23D4u, 0x5DB2u, 0xDF18u, 0xA17Eu, 0x51FBu, 0x2F9Du, 0xAD37u, 0xD351u,
+ 0x0297u, 0x7CF1u, 0xFE5Bu, 0x803Du, 0x70B8u, 0x0EDEu, 0x8C74u, 0xF212u,
+ 0xE6C9u, 0x98AFu, 0x1A05u, 0x6463u, 0x94E6u, 0xEA80u, 0x682Au, 0x164Cu,
+ 0x419Cu, 0x3FFAu, 0xBD50u, 0xC336u, 0x33B3u, 0x4DD5u, 0xCF7Fu, 0xB119u,
+ 0xA5C2u, 0xDBA4u, 0x590Eu, 0x2768u, 0xD7EDu, 0xA98Bu, 0x2B21u, 0x5547u,
+ 0x031Au, 0x7D7Cu, 0xFFD6u, 0x81B0u, 0x7135u, 0x0F53u, 0x8DF9u, 0xF39Fu,
+ 0xE744u, 0x9922u, 0x1B88u, 0x65EEu, 0x956Bu, 0xEB0Du, 0x69A7u, 0x17C1u,
+ 0x4011u, 0x3E77u, 0xBCDDu, 0xC2BBu, 0x323Eu, 0x4C58u, 0xCEF2u, 0xB094u,
+ 0xA44Fu, 0xDA29u, 0x5883u, 0x26E5u, 0xD660u, 0xA806u, 0x2AACu, 0x54CAu,
+ 0x850Cu, 0xFB6Au, 0x79C0u, 0x07A6u, 0xF723u, 0x8945u, 0x0BEFu, 0x7589u,
+ 0x6152u, 0x1F34u, 0x9D9Eu, 0xE3F8u, 0x137Du, 0x6D1Bu, 0xEFB1u, 0x91D7u,
+ 0xC607u, 0xB861u, 0x3ACBu, 0x44ADu, 0xB428u, 0xCA4Eu, 0x48E4u, 0x3682u,
+ 0x2259u, 0x5C3Fu, 0xDE95u, 0xA0F3u, 0x5076u, 0x2E10u, 0xACBAu, 0xD2DCu
+ },
+ {
+ 0x0000u, 0x82B5u, 0x8EDDu, 0x0C68u, 0x960Du, 0x14B8u, 0x18D0u, 0x9A65u,
+ 0xA7ADu, 0x2518u, 0x2970u, 0xABC5u, 0x31A0u, 0xB315u, 0xBF7Du, 0x3DC8u,
+ 0xC4EDu, 0x4658u, 0x4A30u, 0xC885u, 0x52E0u, 0xD055u, 0xDC3Du, 0x5E88u,
+ 0x6340u, 0xE1F5u, 0xED9Du, 0x6F28u, 0xF54Du, 0x77F8u, 0x7B90u, 0xF925u,
+ 0x026Du, 0x80D8u, 0x8CB0u, 0x0E05u, 0x9460u, 0x16D5u, 0x1ABDu, 0x9808u,
+ 0xA5C0u, 0x2775u, 0x2B1Du, 0xA9A8u, 0x33CDu, 0xB178u, 0xBD10u, 0x3FA5u,
+ 0xC680u, 0x4435u, 0x485Du, 0xCAE8u, 0x508Du, 0xD238u, 0xDE50u, 0x5CE5u,
+ 0x612Du, 0xE398u, 0xEFF0u, 0x6D45u, 0xF720u, 0x7595u, 0x79FDu, 0xFB48u,
+ 0x04DAu, 0x866Fu, 0x8A07u, 0x08B2u, 0x92D7u, 0x1062u, 0x1C0Au, 0x9EBFu,
+ 0xA377u, 0x21C2u, 0x2DAAu, 0xAF1Fu, 0x357Au, 0xB7CFu, 0xBBA7u, 0x3912u,
+ 0xC037u, 0x4282u, 0x4EEAu, 0xCC5Fu, 0x563Au, 0xD48Fu, 0xD8E7u, 0x5A52u,
+ 0x679Au, 0xE52Fu, 0xE947u, 0x6BF2u, 0xF197u, 0x7322u, 0x7F4Au, 0xFDFFu,
+ 0x06B7u, 0x8402u, 0x886Au, 0x0ADFu, 0x90BAu, 0x120Fu, 0x1E67u, 0x9CD2u,
+ 0xA11Au, 0x23AFu, 0x2FC7u, 0xAD72u, 0x3717u, 0xB5A2u, 0xB9CAu, 0x3B7Fu,
+ 0xC25Au, 0x40EFu, 0x4C87u, 0xCE32u, 0x5457u, 0xD6E2u, 0xDA8Au, 0x583Fu,
+ 0x65F7u, 0xE742u, 0xEB2Au, 0x699Fu, 0xF3FAu, 0x714Fu, 0x7D27u, 0xFF92u,
+ 0x09B4u, 0x8B01u, 0x8769u, 0x05DCu, 0x9FB9u, 0x1D0Cu, 0x1164u, 0x93D1u,
+ 0xAE19u, 0x2CACu, 0x20C4u, 0xA271u, 0x3814u, 0xBAA1u, 0xB6C9u, 0x347Cu,
+ 0xCD59u, 0x4FECu, 0x4384u, 0xC131u, 0x5B54u, 0xD9E1u, 0xD589u, 0x573Cu,
+ 0x6AF4u, 0xE841u, 0xE429u, 0x669Cu, 0xFCF9u, 0x7E4Cu, 0x7224u, 0xF091u,
+ 0x0BD9u, 0x896Cu, 0x8504u, 0x07B1u, 0x9DD4u, 0x1F61u, 0x1309u, 0x91BCu,
+ 0xAC74u, 0x2EC1u, 0x22A9u, 0xA01Cu, 0x3A79u, 0xB8CCu, 0xB4A4u, 0x3611u,
+ 0xCF34u, 0x4D81u, 0x41E9u, 0xC35Cu, 0x5939u, 0xDB8Cu, 0xD7E4u, 0x5551u,
+ 0x6899u, 0xEA2Cu, 0xE644u, 0x64F1u, 0xFE94u, 0x7C21u, 0x7049u, 0xF2FCu,
+ 0x0D6Eu, 0x8FDBu, 0x83B3u, 0x0106u, 0x9B63u, 0x19D6u, 0x15BEu, 0x970Bu,
+ 0xAAC3u, 0x2876u, 0x241Eu, 0xA6ABu, 0x3CCEu, 0xBE7Bu, 0xB213u, 0x30A6u,
+ 0xC983u, 0x4B36u, 0x475Eu, 0xC5EBu, 0x5F8Eu, 0xDD3Bu, 0xD153u, 0x53E6u,
+ 0x6E2Eu, 0xEC9Bu, 0xE0F3u, 0x6246u, 0xF823u, 0x7A96u, 0x76FEu, 0xF44Bu,
+ 0x0F03u, 0x8DB6u, 0x81DEu, 0x036Bu, 0x990Eu, 0x1BBBu, 0x17D3u, 0x9566u,
+ 0xA8AEu, 0x2A1Bu, 0x2673u, 0xA4C6u, 0x3EA3u, 0xBC16u, 0xB07Eu, 0x32CBu,
+ 0xCBEEu, 0x495Bu, 0x4533u, 0xC786u, 0x5DE3u, 0xDF56u, 0xD33Eu, 0x518Bu,
+ 0x6C43u, 0xEEF6u, 0xE29Eu, 0x602Bu, 0xFA4Eu, 0x78FBu, 0x7493u, 0xF626u
+ },
+ {
+ 0x0000u, 0x1368u, 0x26D0u, 0x35B8u, 0x4DA0u, 0x5EC8u, 0x6B70u, 0x7818u,
+ 0x9B40u, 0x8828u, 0xBD90u, 0xAEF8u, 0xD6E0u, 0xC588u, 0xF030u, 0xE358u,
+ 0xBD37u, 0xAE5Fu, 0x9BE7u, 0x888Fu, 0xF097u, 0xE3FFu, 0xD647u, 0xC52Fu,
+ 0x2677u, 0x351Fu, 0x00A7u, 0x13CFu, 0x6BD7u, 0x78BFu, 0x4D07u, 0x5E6Fu,
+ 0xF1D9u, 0xE2B1u, 0xD709u, 0xC461u, 0xBC79u, 0xAF11u, 0x9AA9u, 0x89C1u,
+ 0x6A99u, 0x79F1u, 0x4C49u, 0x5F21u, 0x2739u, 0x3451u, 0x01E9u, 0x1281u,
+ 0x4CEEu, 0x5F86u, 0x6A3Eu, 0x7956u, 0x014Eu, 0x1226u, 0x279Eu, 0x34F6u,
+ 0xD7AEu, 0xC4C6u, 0xF17Eu, 0xE216u, 0x9A0Eu, 0x8966u, 0xBCDEu, 0xAFB6u,
+ 0x6805u, 0x7B6Du, 0x4ED5u, 0x5DBDu, 0x25A5u, 0x36CDu, 0x0375u, 0x101Du,
+ 0xF345u, 0xE02Du, 0xD595u, 0xC6FDu, 0xBEE5u, 0xAD8Du, 0x9835u, 0x8B5Du,
+ 0xD532u, 0xC65Au, 0xF3E2u, 0xE08Au, 0x9892u, 0x8BFAu, 0xBE42u, 0xAD2Au,
+ 0x4E72u, 0x5D1Au, 0x68A2u, 0x7BCAu, 0x03D2u, 0x10BAu, 0x2502u, 0x366Au,
+ 0x99DCu, 0x8AB4u, 0xBF0Cu, 0xAC64u, 0xD47Cu, 0xC714u, 0xF2ACu, 0xE1C4u,
+ 0x029Cu, 0x11F4u, 0x244Cu, 0x3724u, 0x4F3Cu, 0x5C54u, 0x69ECu, 0x7A84u,
+ 0x24EBu, 0x3783u, 0x023Bu, 0x1153u, 0x694Bu, 0x7A23u, 0x4F9Bu, 0x5CF3u,
+ 0xBFABu, 0xACC3u, 0x997Bu, 0x8A13u, 0xF20Bu, 0xE163u, 0xD4DBu, 0xC7B3u,
+ 0xD00Au, 0xC362u, 0xF6DAu, 0xE5B2u, 0x9DAAu, 0x8EC2u, 0xBB7Au, 0xA812u,
+ 0x4B4Au, 0x5822u, 0x6D9Au, 0x7EF2u, 0x06EAu, 0x1582u, 0x203Au, 0x3352u,
+ 0x6D3Du, 0x7E55u, 0x4BEDu, 0x5885u, 0x209Du, 0x33F5u, 0x064Du, 0x1525u,
+ 0xF67Du, 0xE515u, 0xD0ADu, 0xC3C5u, 0xBBDDu, 0xA8B5u, 0x9D0Du, 0x8E65u,
+ 0x21D3u, 0x32BBu, 0x0703u, 0x146Bu, 0x6C73u, 0x7F1Bu, 0x4AA3u, 0x59CBu,
+ 0xBA93u, 0xA9FBu, 0x9C43u, 0x8F2Bu, 0xF733u, 0xE45Bu, 0xD1E3u, 0xC28Bu,
+ 0x9CE4u, 0x8F8Cu, 0xBA34u, 0xA95Cu, 0xD144u, 0xC22Cu, 0xF794u, 0xE4FCu,
+ 0x07A4u, 0x14CCu, 0x2174u, 0x321Cu, 0x4A04u, 0x596Cu, 0x6CD4u, 0x7FBCu,
+ 0xB80Fu, 0xAB67u, 0x9EDFu, 0x8DB7u, 0xF5AFu, 0xE6C7u, 0xD37Fu, 0xC017u,
+ 0x234Fu, 0x3027u, 0x059Fu, 0x16F7u, 0x6EEFu, 0x7D87u, 0x483Fu, 0x5B57u,
+ 0x0538u, 0x1650u, 0x23E8u, 0x3080u, 0x4898u, 0x5BF0u, 0x6E48u, 0x7D20u,
+ 0x9E78u, 0x8D10u, 0xB8A8u, 0xABC0u, 0xD3D8u, 0xC0B0u, 0xF508u, 0xE660u,
+ 0x49D6u, 0x5ABEu, 0x6F06u, 0x7C6Eu, 0x0476u, 0x171Eu, 0x22A6u, 0x31CEu,
+ 0xD296u, 0xC1FEu, 0xF446u, 0xE72Eu, 0x9F36u, 0x8C5Eu, 0xB9E6u, 0xAA8Eu,
+ 0xF4E1u, 0xE789u, 0xD231u, 0xC159u, 0xB941u, 0xAA29u, 0x9F91u, 0x8CF9u,
+ 0x6FA1u, 0x7CC9u, 0x4971u, 0x5A19u, 0x2201u, 0x3169u, 0x04D1u, 0x17B9u
+ },
+ {
+ 0x0000u, 0x2BA3u, 0x5746u, 0x7CE5u, 0xAE8Cu, 0x852Fu, 0xF9CAu, 0xD269u,
+ 0xD6AFu, 0xFD0Cu, 0x81E9u, 0xAA4Au, 0x7823u, 0x5380u, 0x2F65u, 0x04C6u,
+ 0x26E9u, 0x0D4Au, 0x71AFu, 0x5A0Cu, 0x8865u, 0xA3C6u, 0xDF23u, 0xF480u,
+ 0xF046u, 0xDBE5u, 0xA700u, 0x8CA3u, 0x5ECAu, 0x7569u, 0x098Cu, 0x222Fu,
+ 0x4DD2u, 0x6671u, 0x1A94u, 0x3137u, 0xE35Eu, 0xC8FDu, 0xB418u, 0x9FBBu,
+ 0x9B7Du, 0xB0DEu, 0xCC3Bu, 0xE798u, 0x35F1u, 0x1E52u, 0x62B7u, 0x4914u,
+ 0x6B3Bu, 0x4098u, 0x3C7Du, 0x17DEu, 0xC5B7u, 0xEE14u, 0x92F1u, 0xB952u,
+ 0xBD94u, 0x9637u, 0xEAD2u, 0xC171u, 0x1318u, 0x38BBu, 0x445Eu, 0x6FFDu,
+ 0x9BA4u, 0xB007u, 0xCCE2u, 0xE741u, 0x3528u, 0x1E8Bu, 0x626Eu, 0x49CDu,
+ 0x4D0Bu, 0x66A8u, 0x1A4Du, 0x31EEu, 0xE387u, 0xC824u, 0xB4C1u, 0x9F62u,
+ 0xBD4Du, 0x96EEu, 0xEA0Bu, 0xC1A8u, 0x13C1u, 0x3862u, 0x4487u, 0x6F24u,
+ 0x6BE2u, 0x4041u, 0x3CA4u, 0x1707u, 0xC56Eu, 0xEECDu, 0x9228u, 0xB98Bu,
+ 0xD676u, 0xFDD5u, 0x8130u, 0xAA93u, 0x78FAu, 0x5359u, 0x2FBCu, 0x041Fu,
+ 0x00D9u, 0x2B7Au, 0x579Fu, 0x7C3Cu, 0xAE55u, 0x85F6u, 0xF913u, 0xD2B0u,
+ 0xF09Fu, 0xDB3Cu, 0xA7D9u, 0x8C7Au, 0x5E13u, 0x75B0u, 0x0955u, 0x22F6u,
+ 0x2630u, 0x0D93u, 0x7176u, 0x5AD5u, 0x88BCu, 0xA31Fu, 0xDFFAu, 0xF459u,
+ 0xBCFFu, 0x975Cu, 0xEBB9u, 0xC01Au, 0x1273u, 0x39D0u, 0x4535u, 0x6E96u,
+ 0x6A50u, 0x41F3u, 0x3D16u, 0x16B5u, 0xC4DCu, 0xEF7Fu, 0x939Au, 0xB839u,
+ 0x9A16u, 0xB1B5u, 0xCD50u, 0xE6F3u, 0x349Au, 0x1F39u, 0x63DCu, 0x487Fu,
+ 0x4CB9u, 0x671Au, 0x1BFFu, 0x305Cu, 0xE235u, 0xC996u, 0xB573u, 0x9ED0u,
+ 0xF12Du, 0xDA8Eu, 0xA66Bu, 0x8DC8u, 0x5FA1u, 0x7402u, 0x08E7u, 0x2344u,
+ 0x2782u, 0x0C21u, 0x70C4u, 0x5B67u, 0x890Eu, 0xA2ADu, 0xDE48u, 0xF5EBu,
+ 0xD7C4u, 0xFC67u, 0x8082u, 0xAB21u, 0x7948u, 0x52EBu, 0x2E0Eu, 0x05ADu,
+ 0x016Bu, 0x2AC8u, 0x562Du, 0x7D8Eu, 0xAFE7u, 0x8444u, 0xF8A1u, 0xD302u,
+ 0x275Bu, 0x0CF8u, 0x701Du, 0x5BBEu, 0x89D7u, 0xA274u, 0xDE91u, 0xF532u,
+ 0xF1F4u, 0xDA57u, 0xA6B2u, 0x8D11u, 0x5F78u, 0x74DBu, 0x083Eu, 0x239Du,
+ 0x01B2u, 0x2A11u, 0x56F4u, 0x7D57u, 0xAF3Eu, 0x849Du, 0xF878u, 0xD3DBu,
+ 0xD71Du, 0xFCBEu, 0x805Bu, 0xABF8u, 0x7991u, 0x5232u, 0x2ED7u, 0x0574u,
+ 0x6A89u, 0x412Au, 0x3DCFu, 0x166Cu, 0xC405u, 0xEFA6u, 0x9343u, 0xB8E0u,
+ 0xBC26u, 0x9785u, 0xEB60u, 0xC0C3u, 0x12AAu, 0x3909u, 0x45ECu, 0x6E4Fu,
+ 0x4C60u, 0x67C3u, 0x1B26u, 0x3085u, 0xE2ECu, 0xC94Fu, 0xB5AAu, 0x9E09u,
+ 0x9ACFu, 0xB16Cu, 0xCD89u, 0xE62Au, 0x3443u, 0x1FE0u, 0x6305u, 0x48A6u
+ },
+ {
+ 0x0000u, 0xF249u, 0x6F25u, 0x9D6Cu, 0xDE4Au, 0x2C03u, 0xB16Fu, 0x4326u,
+ 0x3723u, 0xC56Au, 0x5806u, 0xAA4Fu, 0xE969u, 0x1B20u, 0x864Cu, 0x7405u,
+ 0x6E46u, 0x9C0Fu, 0x0163u, 0xF32Au, 0xB00Cu, 0x4245u, 0xDF29u, 0x2D60u,
+ 0x5965u, 0xAB2Cu, 0x3640u, 0xC409u, 0x872Fu, 0x7566u, 0xE80Au, 0x1A43u,
+ 0xDC8Cu, 0x2EC5u, 0xB3A9u, 0x41E0u, 0x02C6u, 0xF08Fu, 0x6DE3u, 0x9FAAu,
+ 0xEBAFu, 0x19E6u, 0x848Au, 0x76C3u, 0x35E5u, 0xC7ACu, 0x5AC0u, 0xA889u,
+ 0xB2CAu, 0x4083u, 0xDDEFu, 0x2FA6u, 0x6C80u, 0x9EC9u, 0x03A5u, 0xF1ECu,
+ 0x85E9u, 0x77A0u, 0xEACCu, 0x1885u, 0x5BA3u, 0xA9EAu, 0x3486u, 0xC6CFu,
+ 0x32AFu, 0xC0E6u, 0x5D8Au, 0xAFC3u, 0xECE5u, 0x1EACu, 0x83C0u, 0x7189u,
+ 0x058Cu, 0xF7C5u, 0x6AA9u, 0x98E0u, 0xDBC6u, 0x298Fu, 0xB4E3u, 0x46AAu,
+ 0x5CE9u, 0xAEA0u, 0x33CCu, 0xC185u, 0x82A3u, 0x70EAu, 0xED86u, 0x1FCFu,
+ 0x6BCAu, 0x9983u, 0x04EFu, 0xF6A6u, 0xB580u, 0x47C9u, 0xDAA5u, 0x28ECu,
+ 0xEE23u, 0x1C6Au, 0x8106u, 0x734Fu, 0x3069u, 0xC220u, 0x5F4Cu, 0xAD05u,
+ 0xD900u, 0x2B49u, 0xB625u, 0x446Cu, 0x074Au, 0xF503u, 0x686Fu, 0x9A26u,
+ 0x8065u, 0x722Cu, 0xEF40u, 0x1D09u, 0x5E2Fu, 0xAC66u, 0x310Au, 0xC343u,
+ 0xB746u, 0x450Fu, 0xD863u, 0x2A2Au, 0x690Cu, 0x9B45u, 0x0629u, 0xF460u,
+ 0x655Eu, 0x9717u, 0x0A7Bu, 0xF832u, 0xBB14u, 0x495Du, 0xD431u, 0x2678u,
+ 0x527Du, 0xA034u, 0x3D58u, 0xCF11u, 0x8C37u, 0x7E7Eu, 0xE312u, 0x115Bu,
+ 0x0B18u, 0xF951u, 0x643Du, 0x9674u, 0xD552u, 0x271Bu, 0xBA77u, 0x483Eu,
+ 0x3C3Bu, 0xCE72u, 0x531Eu, 0xA157u, 0xE271u, 0x1038u, 0x8D54u, 0x7F1Du,
+ 0xB9D2u, 0x4B9Bu, 0xD6F7u, 0x24BEu, 0x6798u, 0x95D1u, 0x08BDu, 0xFAF4u,
+ 0x8EF1u, 0x7CB8u, 0xE1D4u, 0x139Du, 0x50BBu, 0xA2F2u, 0x3F9Eu, 0xCDD7u,
+ 0xD794u, 0x25DDu, 0xB8B1u, 0x4AF8u, 0x09DEu, 0xFB97u, 0x66FBu, 0x94B2u,
+ 0xE0B7u, 0x12FEu, 0x8F92u, 0x7DDBu, 0x3EFDu, 0xCCB4u, 0x51D8u, 0xA391u,
+ 0x57F1u, 0xA5B8u, 0x38D4u, 0xCA9Du, 0x89BBu, 0x7BF2u, 0xE69Eu, 0x14D7u,
+ 0x60D2u, 0x929Bu, 0x0FF7u, 0xFDBEu, 0xBE98u, 0x4CD1u, 0xD1BDu, 0x23F4u,
+ 0x39B7u, 0xCBFEu, 0x5692u, 0xA4DBu, 0xE7FDu, 0x15B4u, 0x88D8u, 0x7A91u,
+ 0x0E94u, 0xFCDDu, 0x61B1u, 0x93F8u, 0xD0DEu, 0x2297u, 0xBFFBu, 0x4DB2u,
+ 0x8B7Du, 0x7934u, 0xE458u, 0x1611u, 0x5537u, 0xA77Eu, 0x3A12u, 0xC85Bu,
+ 0xBC5Eu, 0x4E17u, 0xD37Bu, 0x2132u, 0x6214u, 0x905Du, 0x0D31u, 0xFF78u,
+ 0xE53Bu, 0x1772u, 0x8A1Eu, 0x7857u, 0x3B71u, 0xC938u, 0x5454u, 0xA61Du,
+ 0xD218u, 0x2051u, 0xBD3Du, 0x4F74u, 0x0C52u, 0xFE1Bu, 0x6377u, 0x913Eu
+ },
+ {
+ 0x0000u, 0xCABCu, 0x1ECFu, 0xD473u, 0x3D9Eu, 0xF722u, 0x2351u, 0xE9EDu,
+ 0x7B3Cu, 0xB180u, 0x65F3u, 0xAF4Fu, 0x46A2u, 0x8C1Eu, 0x586Du, 0x92D1u,
+ 0xF678u, 0x3CC4u, 0xE8B7u, 0x220Bu, 0xCBE6u, 0x015Au, 0xD529u, 0x1F95u,
+ 0x8D44u, 0x47F8u, 0x938Bu, 0x5937u, 0xB0DAu, 0x7A66u, 0xAE15u, 0x64A9u,
+ 0x6747u, 0xADFBu, 0x7988u, 0xB334u, 0x5AD9u, 0x9065u, 0x4416u, 0x8EAAu,
+ 0x1C7Bu, 0xD6C7u, 0x02B4u, 0xC808u, 0x21E5u, 0xEB59u, 0x3F2Au, 0xF596u,
+ 0x913Fu, 0x5B83u, 0x8FF0u, 0x454Cu, 0xACA1u, 0x661Du, 0xB26Eu, 0x78D2u,
+ 0xEA03u, 0x20BFu, 0xF4CCu, 0x3E70u, 0xD79Du, 0x1D21u, 0xC952u, 0x03EEu,
+ 0xCE8Eu, 0x0432u, 0xD041u, 0x1AFDu, 0xF310u, 0x39ACu, 0xEDDFu, 0x2763u,
+ 0xB5B2u, 0x7F0Eu, 0xAB7Du, 0x61C1u, 0x882Cu, 0x4290u, 0x96E3u, 0x5C5Fu,
+ 0x38F6u, 0xF24Au, 0x2639u, 0xEC85u, 0x0568u, 0xCFD4u, 0x1BA7u, 0xD11Bu,
+ 0x43CAu, 0x8976u, 0x5D05u, 0x97B9u, 0x7E54u, 0xB4E8u, 0x609Bu, 0xAA27u,
+ 0xA9C9u, 0x6375u, 0xB706u, 0x7DBAu, 0x9457u, 0x5EEBu, 0x8A98u, 0x4024u,
+ 0xD2F5u, 0x1849u, 0xCC3Au, 0x0686u, 0xEF6Bu, 0x25D7u, 0xF1A4u, 0x3B18u,
+ 0x5FB1u, 0x950Du, 0x417Eu, 0x8BC2u, 0x622Fu, 0xA893u, 0x7CE0u, 0xB65Cu,
+ 0x248Du, 0xEE31u, 0x3A42u, 0xF0FEu, 0x1913u, 0xD3AFu, 0x07DCu, 0xCD60u,
+ 0x16ABu, 0xDC17u, 0x0864u, 0xC2D8u, 0x2B35u, 0xE189u, 0x35FAu, 0xFF46u,
+ 0x6D97u, 0xA72Bu, 0x7358u, 0xB9E4u, 0x5009u, 0x9AB5u, 0x4EC6u, 0x847Au,
+ 0xE0D3u, 0x2A6Fu, 0xFE1Cu, 0x34A0u, 0xDD4Du, 0x17F1u, 0xC382u, 0x093Eu,
+ 0x9BEFu, 0x5153u, 0x8520u, 0x4F9Cu, 0xA671u, 0x6CCDu, 0xB8BEu, 0x7202u,
+ 0x71ECu, 0xBB50u, 0x6F23u, 0xA59Fu, 0x4C72u, 0x86CEu, 0x52BDu, 0x9801u,
+ 0x0AD0u, 0xC06Cu, 0x141Fu, 0xDEA3u, 0x374Eu, 0xFDF2u, 0x2981u, 0xE33Du,
+ 0x8794u, 0x4D28u, 0x995Bu, 0x53E7u, 0xBA0Au, 0x70B6u, 0xA4C5u, 0x6E79u,
+ 0xFCA8u, 0x3614u, 0xE267u, 0x28DBu, 0xC136u, 0x0B8Au, 0xDFF9u, 0x1545u,
+ 0xD825u, 0x1299u, 0xC6EAu, 0x0C56u, 0xE5BBu, 0x2F07u, 0xFB74u, 0x31C8u,
+ 0xA319u, 0x69A5u, 0xBDD6u, 0x776Au, 0x9E87u, 0x543Bu, 0x8048u, 0x4AF4u,
+ 0x2E5Du, 0xE4E1u, 0x3092u, 0xFA2Eu, 0x13C3u, 0xD97Fu, 0x0D0Cu, 0xC7B0u,
+ 0x5561u, 0x9FDDu, 0x4BAEu, 0x8112u, 0x68FFu, 0xA243u, 0x7630u, 0xBC8Cu,
+ 0xBF62u, 0x75DEu, 0xA1ADu, 0x6B11u, 0x82FCu, 0x4840u, 0x9C33u, 0x568Fu,
+ 0xC45Eu, 0x0EE2u, 0xDA91u, 0x102Du, 0xF9C0u, 0x337Cu, 0xE70Fu, 0x2DB3u,
+ 0x491Au, 0x83A6u, 0x57D5u, 0x9D69u, 0x7484u, 0xBE38u, 0x6A4Bu, 0xA0F7u,
+ 0x3226u, 0xF89Au, 0x2CE9u, 0xE655u, 0x0FB8u, 0xC504u, 0x1177u, 0xDBCBu
+ },
+ {
+ 0x0000u, 0x2D56u, 0x5AACu, 0x77FAu, 0xB558u, 0x980Eu, 0xEFF4u, 0xC2A2u,
+ 0xE107u, 0xCC51u, 0xBBABu, 0x96FDu, 0x545Fu, 0x7909u, 0x0EF3u, 0x23A5u,
+ 0x49B9u, 0x64EFu, 0x1315u, 0x3E43u, 0xFCE1u, 0xD1B7u, 0xA64Du, 0x8B1Bu,
+ 0xA8BEu, 0x85E8u, 0xF212u, 0xDF44u, 0x1DE6u, 0x30B0u, 0x474Au, 0x6A1Cu,
+ 0x9372u, 0xBE24u, 0xC9DEu, 0xE488u, 0x262Au, 0x0B7Cu, 0x7C86u, 0x51D0u,
+ 0x7275u, 0x5F23u, 0x28D9u, 0x058Fu, 0xC72Du, 0xEA7Bu, 0x9D81u, 0xB0D7u,
+ 0xDACBu, 0xF79Du, 0x8067u, 0xAD31u, 0x6F93u, 0x42C5u, 0x353Fu, 0x1869u,
+ 0x3BCCu, 0x169Au, 0x6160u, 0x4C36u, 0x8E94u, 0xA3C2u, 0xD438u, 0xF96Eu,
+ 0xAD53u, 0x8005u, 0xF7FFu, 0xDAA9u, 0x180Bu, 0x355Du, 0x42A7u, 0x6FF1u,
+ 0x4C54u, 0x6102u, 0x16F8u, 0x3BAEu, 0xF90Cu, 0xD45Au, 0xA3A0u, 0x8EF6u,
+ 0xE4EAu, 0xC9BCu, 0xBE46u, 0x9310u, 0x51B2u, 0x7CE4u, 0x0B1Eu, 0x2648u,
+ 0x05EDu, 0x28BBu, 0x5F41u, 0x7217u, 0xB0B5u, 0x9DE3u, 0xEA19u, 0xC74Fu,
+ 0x3E21u, 0x1377u, 0x648Du, 0x49DBu, 0x8B79u, 0xA62Fu, 0xD1D5u, 0xFC83u,
+ 0xDF26u, 0xF270u, 0x858Au, 0xA8DCu, 0x6A7Eu, 0x4728u, 0x30D2u, 0x1D84u,
+ 0x7798u, 0x5ACEu, 0x2D34u, 0x0062u, 0xC2C0u, 0xEF96u, 0x986Cu, 0xB53Au,
+ 0x969Fu, 0xBBC9u, 0xCC33u, 0xE165u, 0x23C7u, 0x0E91u, 0x796Bu, 0x543Du,
+ 0xD111u, 0xFC47u, 0x8BBDu, 0xA6EBu, 0x6449u, 0x491Fu, 0x3EE5u, 0x13B3u,
+ 0x3016u, 0x1D40u, 0x6ABAu, 0x47ECu, 0x854Eu, 0xA818u, 0xDFE2u, 0xF2B4u,
+ 0x98A8u, 0xB5FEu, 0xC204u, 0xEF52u, 0x2DF0u, 0x00A6u, 0x775Cu, 0x5A0Au,
+ 0x79AFu, 0x54F9u, 0x2303u, 0x0E55u, 0xCCF7u, 0xE1A1u, 0x965Bu, 0xBB0Du,
+ 0x4263u, 0x6F35u, 0x18CFu, 0x3599u, 0xF73Bu, 0xDA6Du, 0xAD97u, 0x80C1u,
+ 0xA364u, 0x8E32u, 0xF9C8u, 0xD49Eu, 0x163Cu, 0x3B6Au, 0x4C90u, 0x61C6u,
+ 0x0BDAu, 0x268Cu, 0x5176u, 0x7C20u, 0xBE82u, 0x93D4u, 0xE42Eu, 0xC978u,
+ 0xEADDu, 0xC78Bu, 0xB071u, 0x9D27u, 0x5F85u, 0x72D3u, 0x0529u, 0x287Fu,
+ 0x7C42u, 0x5114u, 0x26EEu, 0x0BB8u, 0xC91Au, 0xE44Cu, 0x93B6u, 0xBEE0u,
+ 0x9D45u, 0xB013u, 0xC7E9u, 0xEABFu, 0x281Du, 0x054Bu, 0x72B1u, 0x5FE7u,
+ 0x35FBu, 0x18ADu, 0x6F57u, 0x4201u, 0x80A3u, 0xADF5u, 0xDA0Fu, 0xF759u,
+ 0xD4FCu, 0xF9AAu, 0x8E50u, 0xA306u, 0x61A4u, 0x4CF2u, 0x3B08u, 0x165Eu,
+ 0xEF30u, 0xC266u, 0xB59Cu, 0x98CAu, 0x5A68u, 0x773Eu, 0x00C4u, 0x2D92u,
+ 0x0E37u, 0x2361u, 0x549Bu, 0x79CDu, 0xBB6Fu, 0x9639u, 0xE1C3u, 0xCC95u,
+ 0xA689u, 0x8BDFu, 0xFC25u, 0xD173u, 0x13D1u, 0x3E87u, 0x497Du, 0x642Bu,
+ 0x478Eu, 0x6AD8u, 0x1D22u, 0x3074u, 0xF2D6u, 0xDF80u, 0xA87Au, 0x852Cu
+ },
+ {
+ 0x0000u, 0x2995u, 0x532Au, 0x7ABFu, 0xA654u, 0x8FC1u, 0xF57Eu, 0xDCEBu,
+ 0xC71Fu, 0xEE8Au, 0x9435u, 0xBDA0u, 0x614Bu, 0x48DEu, 0x3261u, 0x1BF4u,
+ 0x0589u, 0x2C1Cu, 0x56A3u, 0x7F36u, 0xA3DDu, 0x8A48u, 0xF0F7u, 0xD962u,
+ 0xC296u, 0xEB03u, 0x91BCu, 0xB829u, 0x64C2u, 0x4D57u, 0x37E8u, 0x1E7Du,
+ 0x0B12u, 0x2287u, 0x5838u, 0x71ADu, 0xAD46u, 0x84D3u, 0xFE6Cu, 0xD7F9u,
+ 0xCC0Du, 0xE598u, 0x9F27u, 0xB6B2u, 0x6A59u, 0x43CCu, 0x3973u, 0x10E6u,
+ 0x0E9Bu, 0x270Eu, 0x5DB1u, 0x7424u, 0xA8CFu, 0x815Au, 0xFBE5u, 0xD270u,
+ 0xC984u, 0xE011u, 0x9AAEu, 0xB33Bu, 0x6FD0u, 0x4645u, 0x3CFAu, 0x156Fu,
+ 0x1624u, 0x3FB1u, 0x450Eu, 0x6C9Bu, 0xB070u, 0x99E5u, 0xE35Au, 0xCACFu,
+ 0xD13Bu, 0xF8AEu, 0x8211u, 0xAB84u, 0x776Fu, 0x5EFAu, 0x2445u, 0x0DD0u,
+ 0x13ADu, 0x3A38u, 0x4087u, 0x6912u, 0xB5F9u, 0x9C6Cu, 0xE6D3u, 0xCF46u,
+ 0xD4B2u, 0xFD27u, 0x8798u, 0xAE0Du, 0x72E6u, 0x5B73u, 0x21CCu, 0x0859u,
+ 0x1D36u, 0x34A3u, 0x4E1Cu, 0x6789u, 0xBB62u, 0x92F7u, 0xE848u, 0xC1DDu,
+ 0xDA29u, 0xF3BCu, 0x8903u, 0xA096u, 0x7C7Du, 0x55E8u, 0x2F57u, 0x06C2u,
+ 0x18BFu, 0x312Au, 0x4B95u, 0x6200u, 0xBEEBu, 0x977Eu, 0xEDC1u, 0xC454u,
+ 0xDFA0u, 0xF635u, 0x8C8Au, 0xA51Fu, 0x79F4u, 0x5061u, 0x2ADEu, 0x034Bu,
+ 0x2C48u, 0x05DDu, 0x7F62u, 0x56F7u, 0x8A1Cu, 0xA389u, 0xD936u, 0xF0A3u,
+ 0xEB57u, 0xC2C2u, 0xB87Du, 0x91E8u, 0x4D03u, 0x6496u, 0x1E29u, 0x37BCu,
+ 0x29C1u, 0x0054u, 0x7AEBu, 0x537Eu, 0x8F95u, 0xA600u, 0xDCBFu, 0xF52Au,
+ 0xEEDEu, 0xC74Bu, 0xBDF4u, 0x9461u, 0x488Au, 0x611Fu, 0x1BA0u, 0x3235u,
+ 0x275Au, 0x0ECFu, 0x7470u, 0x5DE5u, 0x810Eu, 0xA89Bu, 0xD224u, 0xFBB1u,
+ 0xE045u, 0xC9D0u, 0xB36Fu, 0x9AFAu, 0x4611u, 0x6F84u, 0x153Bu, 0x3CAEu,
+ 0x22D3u, 0x0B46u, 0x71F9u, 0x586Cu, 0x8487u, 0xAD12u, 0xD7ADu, 0xFE38u,
+ 0xE5CCu, 0xCC59u, 0xB6E6u, 0x9F73u, 0x4398u, 0x6A0Du, 0x10B2u, 0x3927u,
+ 0x3A6Cu, 0x13F9u, 0x6946u, 0x40D3u, 0x9C38u, 0xB5ADu, 0xCF12u, 0xE687u,
+ 0xFD73u, 0xD4E6u, 0xAE59u, 0x87CCu, 0x5B27u, 0x72B2u, 0x080Du, 0x2198u,
+ 0x3FE5u, 0x1670u, 0x6CCFu, 0x455Au, 0x99B1u, 0xB024u, 0xCA9Bu, 0xE30Eu,
+ 0xF8FAu, 0xD16Fu, 0xABD0u, 0x8245u, 0x5EAEu, 0x773Bu, 0x0D84u, 0x2411u,
+ 0x317Eu, 0x18EBu, 0x6254u, 0x4BC1u, 0x972Au, 0xBEBFu, 0xC400u, 0xED95u,
+ 0xF661u, 0xDFF4u, 0xA54Bu, 0x8CDEu, 0x5035u, 0x79A0u, 0x031Fu, 0x2A8Au,
+ 0x34F7u, 0x1D62u, 0x67DDu, 0x4E48u, 0x92A3u, 0xBB36u, 0xC189u, 0xE81Cu,
+ 0xF3E8u, 0xDA7Du, 0xA0C2u, 0x8957u, 0x55BCu, 0x7C29u, 0x0696u, 0x2F03u
+ },
+ {
+ 0x0000u, 0x5890u, 0xB120u, 0xE9B0u, 0xE9F7u, 0xB167u, 0x58D7u, 0x0047u,
+ 0x5859u, 0x00C9u, 0xE979u, 0xB1E9u, 0xB1AEu, 0xE93Eu, 0x008Eu, 0x581Eu,
+ 0xB0B2u, 0xE822u, 0x0192u, 0x5902u, 0x5945u, 0x01D5u, 0xE865u, 0xB0F5u,
+ 0xE8EBu, 0xB07Bu, 0x59CBu, 0x015Bu, 0x011Cu, 0x598Cu, 0xB03Cu, 0xE8ACu,
+ 0xEAD3u, 0xB243u, 0x5BF3u, 0x0363u, 0x0324u, 0x5BB4u, 0xB204u, 0xEA94u,
+ 0xB28Au, 0xEA1Au, 0x03AAu, 0x5B3Au, 0x5B7Du, 0x03EDu, 0xEA5Du, 0xB2CDu,
+ 0x5A61u, 0x02F1u, 0xEB41u, 0xB3D1u, 0xB396u, 0xEB06u, 0x02B6u, 0x5A26u,
+ 0x0238u, 0x5AA8u, 0xB318u, 0xEB88u, 0xEBCFu, 0xB35Fu, 0x5AEFu, 0x027Fu,
+ 0x5E11u, 0x0681u, 0xEF31u, 0xB7A1u, 0xB7E6u, 0xEF76u, 0x06C6u, 0x5E56u,
+ 0x0648u, 0x5ED8u, 0xB768u, 0xEFF8u, 0xEFBFu, 0xB72Fu, 0x5E9Fu, 0x060Fu,
+ 0xEEA3u, 0xB633u, 0x5F83u, 0x0713u, 0x0754u, 0x5FC4u, 0xB674u, 0xEEE4u,
+ 0xB6FAu, 0xEE6Au, 0x07DAu, 0x5F4Au, 0x5F0Du, 0x079Du, 0xEE2Du, 0xB6BDu,
+ 0xB4C2u, 0xEC52u, 0x05E2u, 0x5D72u, 0x5D35u, 0x05A5u, 0xEC15u, 0xB485u,
+ 0xEC9Bu, 0xB40Bu, 0x5DBBu, 0x052Bu, 0x056Cu, 0x5DFCu, 0xB44Cu, 0xECDCu,
+ 0x0470u, 0x5CE0u, 0xB550u, 0xEDC0u, 0xED87u, 0xB517u, 0x5CA7u, 0x0437u,
+ 0x5C29u, 0x04B9u, 0xED09u, 0xB599u, 0xB5DEu, 0xED4Eu, 0x04FEu, 0x5C6Eu,
+ 0xBC22u, 0xE4B2u, 0x0D02u, 0x5592u, 0x55D5u, 0x0D45u, 0xE4F5u, 0xBC65u,
+ 0xE47Bu, 0xBCEBu, 0x555Bu, 0x0DCBu, 0x0D8Cu, 0x551Cu, 0xBCACu, 0xE43Cu,
+ 0x0C90u, 0x5400u, 0xBDB0u, 0xE520u, 0xE567u, 0xBDF7u, 0x5447u, 0x0CD7u,
+ 0x54C9u, 0x0C59u, 0xE5E9u, 0xBD79u, 0xBD3Eu, 0xE5AEu, 0x0C1Eu, 0x548Eu,
+ 0x56F1u, 0x0E61u, 0xE7D1u, 0xBF41u, 0xBF06u, 0xE796u, 0x0E26u, 0x56B6u,
+ 0x0EA8u, 0x5638u, 0xBF88u, 0xE718u, 0xE75Fu, 0xBFCFu, 0x567Fu, 0x0EEFu,
+ 0xE643u, 0xBED3u, 0x5763u, 0x0FF3u, 0x0FB4u, 0x5724u, 0xBE94u, 0xE604u,
+ 0xBE1Au, 0xE68Au, 0x0F3Au, 0x57AAu, 0x57EDu, 0x0F7Du, 0xE6CDu, 0xBE5Du,
+ 0xE233u, 0xBAA3u, 0x5313u, 0x0B83u, 0x0BC4u, 0x5354u, 0xBAE4u, 0xE274u,
+ 0xBA6Au, 0xE2FAu, 0x0B4Au, 0x53DAu, 0x539Du, 0x0B0Du, 0xE2BDu, 0xBA2Du,
+ 0x5281u, 0x0A11u, 0xE3A1u, 0xBB31u, 0xBB76u, 0xE3E6u, 0x0A56u, 0x52C6u,
+ 0x0AD8u, 0x5248u, 0xBBF8u, 0xE368u, 0xE32Fu, 0xBBBFu, 0x520Fu, 0x0A9Fu,
+ 0x08E0u, 0x5070u, 0xB9C0u, 0xE150u, 0xE117u, 0xB987u, 0x5037u, 0x08A7u,
+ 0x50B9u, 0x0829u, 0xE199u, 0xB909u, 0xB94Eu, 0xE1DEu, 0x086Eu, 0x50FEu,
+ 0xB852u, 0xE0C2u, 0x0972u, 0x51E2u, 0x51A5u, 0x0935u, 0xE085u, 0xB815u,
+ 0xE00Bu, 0xB89Bu, 0x512Bu, 0x09BBu, 0x09FCu, 0x516Cu, 0xB8DCu, 0xE04Cu
+ },
+ {
+ 0x0000u, 0xF3F3u, 0x6C51u, 0x9FA2u, 0xD8A2u, 0x2B51u, 0xB4F3u, 0x4700u,
+ 0x3AF3u, 0xC900u, 0x56A2u, 0xA551u, 0xE251u, 0x11A2u, 0x8E00u, 0x7DF3u,
+ 0x75E6u, 0x8615u, 0x19B7u, 0xEA44u, 0xAD44u, 0x5EB7u, 0xC115u, 0x32E6u,
+ 0x4F15u, 0xBCE6u, 0x2344u, 0xD0B7u, 0x97B7u, 0x6444u, 0xFBE6u, 0x0815u,
+ 0xEBCCu, 0x183Fu, 0x879Du, 0x746Eu, 0x336Eu, 0xC09Du, 0x5F3Fu, 0xACCCu,
+ 0xD13Fu, 0x22CCu, 0xBD6Eu, 0x4E9Du, 0x099Du, 0xFA6Eu, 0x65CCu, 0x963Fu,
+ 0x9E2Au, 0x6DD9u, 0xF27Bu, 0x0188u, 0x4688u, 0xB57Bu, 0x2AD9u, 0xD92Au,
+ 0xA4D9u, 0x572Au, 0xC888u, 0x3B7Bu, 0x7C7Bu, 0x8F88u, 0x102Au, 0xE3D9u,
+ 0x5C2Fu, 0xAFDCu, 0x307Eu, 0xC38Du, 0x848Du, 0x777Eu, 0xE8DCu, 0x1B2Fu,
+ 0x66DCu, 0x952Fu, 0x0A8Du, 0xF97Eu, 0xBE7Eu, 0x4D8Du, 0xD22Fu, 0x21DCu,
+ 0x29C9u, 0xDA3Au, 0x4598u, 0xB66Bu, 0xF16Bu, 0x0298u, 0x9D3Au, 0x6EC9u,
+ 0x133Au, 0xE0C9u, 0x7F6Bu, 0x8C98u, 0xCB98u, 0x386Bu, 0xA7C9u, 0x543Au,
+ 0xB7E3u, 0x4410u, 0xDBB2u, 0x2841u, 0x6F41u, 0x9CB2u, 0x0310u, 0xF0E3u,
+ 0x8D10u, 0x7EE3u, 0xE141u, 0x12B2u, 0x55B2u, 0xA641u, 0x39E3u, 0xCA10u,
+ 0xC205u, 0x31F6u, 0xAE54u, 0x5DA7u, 0x1AA7u, 0xE954u, 0x76F6u, 0x8505u,
+ 0xF8F6u, 0x0B05u, 0x94A7u, 0x6754u, 0x2054u, 0xD3A7u, 0x4C05u, 0xBFF6u,
+ 0xB85Eu, 0x4BADu, 0xD40Fu, 0x27FCu, 0x60FCu, 0x930Fu, 0x0CADu, 0xFF5Eu,
+ 0x82ADu, 0x715Eu, 0xEEFCu, 0x1D0Fu, 0x5A0Fu, 0xA9FCu, 0x365Eu, 0xC5ADu,
+ 0xCDB8u, 0x3E4Bu, 0xA1E9u, 0x521Au, 0x151Au, 0xE6E9u, 0x794Bu, 0x8AB8u,
+ 0xF74Bu, 0x04B8u, 0x9B1Au, 0x68E9u, 0x2FE9u, 0xDC1Au, 0x43B8u, 0xB04Bu,
+ 0x5392u, 0xA061u, 0x3FC3u, 0xCC30u, 0x8B30u, 0x78C3u, 0xE761u, 0x1492u,
+ 0x6961u, 0x9A92u, 0x0530u, 0xF6C3u, 0xB1C3u, 0x4230u, 0xDD92u, 0x2E61u,
+ 0x2674u, 0xD587u, 0x4A25u, 0xB9D6u, 0xFED6u, 0x0D25u, 0x9287u, 0x6174u,
+ 0x1C87u, 0xEF74u, 0x70D6u, 0x8325u, 0xC425u, 0x37D6u, 0xA874u, 0x5B87u,
+ 0xE471u, 0x1782u, 0x8820u, 0x7BD3u, 0x3CD3u, 0xCF20u, 0x5082u, 0xA371u,
+ 0xDE82u, 0x2D71u, 0xB2D3u, 0x4120u, 0x0620u, 0xF5D3u, 0x6A71u, 0x9982u,
+ 0x9197u, 0x6264u, 0xFDC6u, 0x0E35u, 0x4935u, 0xBAC6u, 0x2564u, 0xD697u,
+ 0xAB64u, 0x5897u, 0xC735u, 0x34C6u, 0x73C6u, 0x8035u, 0x1F97u, 0xEC64u,
+ 0x0FBDu, 0xFC4Eu, 0x63ECu, 0x901Fu, 0xD71Fu, 0x24ECu, 0xBB4Eu, 0x48BDu,
+ 0x354Eu, 0xC6BDu, 0x591Fu, 0xAAECu, 0xEDECu, 0x1E1Fu, 0x81BDu, 0x724Eu,
+ 0x7A5Bu, 0x89A8u, 0x160Au, 0xE5F9u, 0xA2F9u, 0x510Au, 0xCEA8u, 0x3D5Bu,
+ 0x40A8u, 0xB35Bu, 0x2CF9u, 0xDF0Au, 0x980Au, 0x6BF9u, 0xF45Bu, 0x07A8u
+ },
+ {
+ 0x0000u, 0xFB0Bu, 0x7DA1u, 0x86AAu, 0xFB42u, 0x0049u, 0x86E3u, 0x7DE8u,
+ 0x7D33u, 0x8638u, 0x0092u, 0xFB99u, 0x8671u, 0x7D7Au, 0xFBD0u, 0x00DBu,
+ 0xFA66u, 0x016Du, 0x87C7u, 0x7CCCu, 0x0124u, 0xFA2Fu, 0x7C85u, 0x878Eu,
+ 0x8755u, 0x7C5Eu, 0xFAF4u, 0x01FFu, 0x7C17u, 0x871Cu, 0x01B6u, 0xFABDu,
+ 0x7F7Bu, 0x8470u, 0x02DAu, 0xF9D1u, 0x8439u, 0x7F32u, 0xF998u, 0x0293u,
+ 0x0248u, 0xF943u, 0x7FE9u, 0x84E2u, 0xF90Au, 0x0201u, 0x84ABu, 0x7FA0u,
+ 0x851Du, 0x7E16u, 0xF8BCu, 0x03B7u, 0x7E5Fu, 0x8554u, 0x03FEu, 0xF8F5u,
+ 0xF82Eu, 0x0325u, 0x858Fu, 0x7E84u, 0x036Cu, 0xF867u, 0x7ECDu, 0x85C6u,
+ 0xFEF6u, 0x05FDu, 0x8357u, 0x785Cu, 0x05B4u, 0xFEBFu, 0x7815u, 0x831Eu,
+ 0x83C5u, 0x78CEu, 0xFE64u, 0x056Fu, 0x7887u, 0x838Cu, 0x0526u, 0xFE2Du,
+ 0x0490u, 0xFF9Bu, 0x7931u, 0x823Au, 0xFFD2u, 0x04D9u, 0x8273u, 0x7978u,
+ 0x79A3u, 0x82A8u, 0x0402u, 0xFF09u, 0x82E1u, 0x79EAu, 0xFF40u, 0x044Bu,
+ 0x818Du, 0x7A86u, 0xFC2Cu, 0x0727u, 0x7ACFu, 0x81C4u, 0x076Eu, 0xFC65u,
+ 0xFCBEu, 0x07B5u, 0x811Fu, 0x7A14u, 0x07FCu, 0xFCF7u, 0x7A5Du, 0x8156u,
+ 0x7BEBu, 0x80E0u, 0x064Au, 0xFD41u, 0x80A9u, 0x7BA2u, 0xFD08u, 0x0603u,
+ 0x06D8u, 0xFDD3u, 0x7B79u, 0x8072u, 0xFD9Au, 0x0691u, 0x803Bu, 0x7B30u,
+ 0x765Bu, 0x8D50u, 0x0BFAu, 0xF0F1u, 0x8D19u, 0x7612u, 0xF0B8u, 0x0BB3u,
+ 0x0B68u, 0xF063u, 0x76C9u, 0x8DC2u, 0xF02Au, 0x0B21u, 0x8D8Bu, 0x7680u,
+ 0x8C3Du, 0x7736u, 0xF19Cu, 0x0A97u, 0x777Fu, 0x8C74u, 0x0ADEu, 0xF1D5u,
+ 0xF10Eu, 0x0A05u, 0x8CAFu, 0x77A4u, 0x0A4Cu, 0xF147u, 0x77EDu, 0x8CE6u,
+ 0x0920u, 0xF22Bu, 0x7481u, 0x8F8Au, 0xF262u, 0x0969u, 0x8FC3u, 0x74C8u,
+ 0x7413u, 0x8F18u, 0x09B2u, 0xF2B9u, 0x8F51u, 0x745Au, 0xF2F0u, 0x09FBu,
+ 0xF346u, 0x084Du, 0x8EE7u, 0x75ECu, 0x0804u, 0xF30Fu, 0x75A5u, 0x8EAEu,
+ 0x8E75u, 0x757Eu, 0xF3D4u, 0x08DFu, 0x7537u, 0x8E3Cu, 0x0896u, 0xF39Du,
+ 0x88ADu, 0x73A6u, 0xF50Cu, 0x0E07u, 0x73EFu, 0x88E4u, 0x0E4Eu, 0xF545u,
+ 0xF59Eu, 0x0E95u, 0x883Fu, 0x7334u, 0x0EDCu, 0xF5D7u, 0x737Du, 0x8876u,
+ 0x72CBu, 0x89C0u, 0x0F6Au, 0xF461u, 0x8989u, 0x7282u, 0xF428u, 0x0F23u,
+ 0x0FF8u, 0xF4F3u, 0x7259u, 0x8952u, 0xF4BAu, 0x0FB1u, 0x891Bu, 0x7210u,
+ 0xF7D6u, 0x0CDDu, 0x8A77u, 0x717Cu, 0x0C94u, 0xF79Fu, 0x7135u, 0x8A3Eu,
+ 0x8AE5u, 0x71EEu, 0xF744u, 0x0C4Fu, 0x71A7u, 0x8AACu, 0x0C06u, 0xF70Du,
+ 0x0DB0u, 0xF6BBu, 0x7011u, 0x8B1Au, 0xF6F2u, 0x0DF9u, 0x8B53u, 0x7058u,
+ 0x7083u, 0x8B88u, 0x0D22u, 0xF629u, 0x8BC1u, 0x70CAu, 0xF660u, 0x0D6Bu
+ },
+ {
+ 0x0000u, 0xECB6u, 0x52DBu, 0xBE6Du, 0xA5B6u, 0x4900u, 0xF76Du, 0x1BDBu,
+ 0xC0DBu, 0x2C6Du, 0x9200u, 0x7EB6u, 0x656Du, 0x89DBu, 0x37B6u, 0xDB00u,
+ 0x0A01u, 0xE6B7u, 0x58DAu, 0xB46Cu, 0xAFB7u, 0x4301u, 0xFD6Cu, 0x11DAu,
+ 0xCADAu, 0x266Cu, 0x9801u, 0x74B7u, 0x6F6Cu, 0x83DAu, 0x3DB7u, 0xD101u,
+ 0x1402u, 0xF8B4u, 0x46D9u, 0xAA6Fu, 0xB1B4u, 0x5D02u, 0xE36Fu, 0x0FD9u,
+ 0xD4D9u, 0x386Fu, 0x8602u, 0x6AB4u, 0x716Fu, 0x9DD9u, 0x23B4u, 0xCF02u,
+ 0x1E03u, 0xF2B5u, 0x4CD8u, 0xA06Eu, 0xBBB5u, 0x5703u, 0xE96Eu, 0x05D8u,
+ 0xDED8u, 0x326Eu, 0x8C03u, 0x60B5u, 0x7B6Eu, 0x97D8u, 0x29B5u, 0xC503u,
+ 0x2804u, 0xC4B2u, 0x7ADFu, 0x9669u, 0x8DB2u, 0x6104u, 0xDF69u, 0x33DFu,
+ 0xE8DFu, 0x0469u, 0xBA04u, 0x56B2u, 0x4D69u, 0xA1DFu, 0x1FB2u, 0xF304u,
+ 0x2205u, 0xCEB3u, 0x70DEu, 0x9C68u, 0x87B3u, 0x6B05u, 0xD568u, 0x39DEu,
+ 0xE2DEu, 0x0E68u, 0xB005u, 0x5CB3u, 0x4768u, 0xABDEu, 0x15B3u, 0xF905u,
+ 0x3C06u, 0xD0B0u, 0x6EDDu, 0x826Bu, 0x99B0u, 0x7506u, 0xCB6Bu, 0x27DDu,
+ 0xFCDDu, 0x106Bu, 0xAE06u, 0x42B0u, 0x596Bu, 0xB5DDu, 0x0BB0u, 0xE706u,
+ 0x3607u, 0xDAB1u, 0x64DCu, 0x886Au, 0x93B1u, 0x7F07u, 0xC16Au, 0x2DDCu,
+ 0xF6DCu, 0x1A6Au, 0xA407u, 0x48B1u, 0x536Au, 0xBFDCu, 0x01B1u, 0xED07u,
+ 0x5008u, 0xBCBEu, 0x02D3u, 0xEE65u, 0xF5BEu, 0x1908u, 0xA765u, 0x4BD3u,
+ 0x90D3u, 0x7C65u, 0xC208u, 0x2EBEu, 0x3565u, 0xD9D3u, 0x67BEu, 0x8B08u,
+ 0x5A09u, 0xB6BFu, 0x08D2u, 0xE464u, 0xFFBFu, 0x1309u, 0xAD64u, 0x41D2u,
+ 0x9AD2u, 0x7664u, 0xC809u, 0x24BFu, 0x3F64u, 0xD3D2u, 0x6DBFu, 0x8109u,
+ 0x440Au, 0xA8BCu, 0x16D1u, 0xFA67u, 0xE1BCu, 0x0D0Au, 0xB367u, 0x5FD1u,
+ 0x84D1u, 0x6867u, 0xD60Au, 0x3ABCu, 0x2167u, 0xCDD1u, 0x73BCu, 0x9F0Au,
+ 0x4E0Bu, 0xA2BDu, 0x1CD0u, 0xF066u, 0xEBBDu, 0x070Bu, 0xB966u, 0x55D0u,
+ 0x8ED0u, 0x6266u, 0xDC0Bu, 0x30BDu, 0x2B66u, 0xC7D0u, 0x79BDu, 0x950Bu,
+ 0x780Cu, 0x94BAu, 0x2AD7u, 0xC661u, 0xDDBAu, 0x310Cu, 0x8F61u, 0x63D7u,
+ 0xB8D7u, 0x5461u, 0xEA0Cu, 0x06BAu, 0x1D61u, 0xF1D7u, 0x4FBAu, 0xA30Cu,
+ 0x720Du, 0x9EBBu, 0x20D6u, 0xCC60u, 0xD7BBu, 0x3B0Du, 0x8560u, 0x69D6u,
+ 0xB2D6u, 0x5E60u, 0xE00Du, 0x0CBBu, 0x1760u, 0xFBD6u, 0x45BBu, 0xA90Du,
+ 0x6C0Eu, 0x80B8u, 0x3ED5u, 0xD263u, 0xC9B8u, 0x250Eu, 0x9B63u, 0x77D5u,
+ 0xACD5u, 0x4063u, 0xFE0Eu, 0x12B8u, 0x0963u, 0xE5D5u, 0x5BB8u, 0xB70Eu,
+ 0x660Fu, 0x8AB9u, 0x34D4u, 0xD862u, 0xC3B9u, 0x2F0Fu, 0x9162u, 0x7DD4u,
+ 0xA6D4u, 0x4A62u, 0xF40Fu, 0x18B9u, 0x0362u, 0xEFD4u, 0x51B9u, 0xBD0Fu
+ },
+ {
+ 0x0000u, 0xA010u, 0xCB97u, 0x6B87u, 0x1C99u, 0xBC89u, 0xD70Eu, 0x771Eu,
+ 0x3932u, 0x9922u, 0xF2A5u, 0x52B5u, 0x25ABu, 0x85BBu, 0xEE3Cu, 0x4E2Cu,
+ 0x7264u, 0xD274u, 0xB9F3u, 0x19E3u, 0x6EFDu, 0xCEEDu, 0xA56Au, 0x057Au,
+ 0x4B56u, 0xEB46u, 0x80C1u, 0x20D1u, 0x57CFu, 0xF7DFu, 0x9C58u, 0x3C48u,
+ 0xE4C8u, 0x44D8u, 0x2F5Fu, 0x8F4Fu, 0xF851u, 0x5841u, 0x33C6u, 0x93D6u,
+ 0xDDFAu, 0x7DEAu, 0x166Du, 0xB67Du, 0xC163u, 0x6173u, 0x0AF4u, 0xAAE4u,
+ 0x96ACu, 0x36BCu, 0x5D3Bu, 0xFD2Bu, 0x8A35u, 0x2A25u, 0x41A2u, 0xE1B2u,
+ 0xAF9Eu, 0x0F8Eu, 0x6409u, 0xC419u, 0xB307u, 0x1317u, 0x7890u, 0xD880u,
+ 0x4227u, 0xE237u, 0x89B0u, 0x29A0u, 0x5EBEu, 0xFEAEu, 0x9529u, 0x3539u,
+ 0x7B15u, 0xDB05u, 0xB082u, 0x1092u, 0x678Cu, 0xC79Cu, 0xAC1Bu, 0x0C0Bu,
+ 0x3043u, 0x9053u, 0xFBD4u, 0x5BC4u, 0x2CDAu, 0x8CCAu, 0xE74Du, 0x475Du,
+ 0x0971u, 0xA961u, 0xC2E6u, 0x62F6u, 0x15E8u, 0xB5F8u, 0xDE7Fu, 0x7E6Fu,
+ 0xA6EFu, 0x06FFu, 0x6D78u, 0xCD68u, 0xBA76u, 0x1A66u, 0x71E1u, 0xD1F1u,
+ 0x9FDDu, 0x3FCDu, 0x544Au, 0xF45Au, 0x8344u, 0x2354u, 0x48D3u, 0xE8C3u,
+ 0xD48Bu, 0x749Bu, 0x1F1Cu, 0xBF0Cu, 0xC812u, 0x6802u, 0x0385u, 0xA395u,
+ 0xEDB9u, 0x4DA9u, 0x262Eu, 0x863Eu, 0xF120u, 0x5130u, 0x3AB7u, 0x9AA7u,
+ 0x844Eu, 0x245Eu, 0x4FD9u, 0xEFC9u, 0x98D7u, 0x38C7u, 0x5340u, 0xF350u,
+ 0xBD7Cu, 0x1D6Cu, 0x76EBu, 0xD6FBu, 0xA1E5u, 0x01F5u, 0x6A72u, 0xCA62u,
+ 0xF62Au, 0x563Au, 0x3DBDu, 0x9DADu, 0xEAB3u, 0x4AA3u, 0x2124u, 0x8134u,
+ 0xCF18u, 0x6F08u, 0x048Fu, 0xA49Fu, 0xD381u, 0x7391u, 0x1816u, 0xB806u,
+ 0x6086u, 0xC096u, 0xAB11u, 0x0B01u, 0x7C1Fu, 0xDC0Fu, 0xB788u, 0x1798u,
+ 0x59B4u, 0xF9A4u, 0x9223u, 0x3233u, 0x452Du, 0xE53Du, 0x8EBAu, 0x2EAAu,
+ 0x12E2u, 0xB2F2u, 0xD975u, 0x7965u, 0x0E7Bu, 0xAE6Bu, 0xC5ECu, 0x65FCu,
+ 0x2BD0u, 0x8BC0u, 0xE047u, 0x4057u, 0x3749u, 0x9759u, 0xFCDEu, 0x5CCEu,
+ 0xC669u, 0x6679u, 0x0DFEu, 0xADEEu, 0xDAF0u, 0x7AE0u, 0x1167u, 0xB177u,
+ 0xFF5Bu, 0x5F4Bu, 0x34CCu, 0x94DCu, 0xE3C2u, 0x43D2u, 0x2855u, 0x8845u,
+ 0xB40Du, 0x141Du, 0x7F9Au, 0xDF8Au, 0xA894u, 0x0884u, 0x6303u, 0xC313u,
+ 0x8D3Fu, 0x2D2Fu, 0x46A8u, 0xE6B8u, 0x91A6u, 0x31B6u, 0x5A31u, 0xFA21u,
+ 0x22A1u, 0x82B1u, 0xE936u, 0x4926u, 0x3E38u, 0x9E28u, 0xF5AFu, 0x55BFu,
+ 0x1B93u, 0xBB83u, 0xD004u, 0x7014u, 0x070Au, 0xA71Au, 0xCC9Du, 0x6C8Du,
+ 0x50C5u, 0xF0D5u, 0x9B52u, 0x3B42u, 0x4C5Cu, 0xEC4Cu, 0x87CBu, 0x27DBu,
+ 0x69F7u, 0xC9E7u, 0xA260u, 0x0270u, 0x756Eu, 0xD57Eu, 0xBEF9u, 0x1EE9u
+ },
+ {
+ 0x0000u, 0x832Bu, 0x8DE1u, 0x0ECAu, 0x9075u, 0x135Eu, 0x1D94u, 0x9EBFu,
+ 0xAB5Du, 0x2876u, 0x26BCu, 0xA597u, 0x3B28u, 0xB803u, 0xB6C9u, 0x35E2u,
+ 0xDD0Du, 0x5E26u, 0x50ECu, 0xD3C7u, 0x4D78u, 0xCE53u, 0xC099u, 0x43B2u,
+ 0x7650u, 0xF57Bu, 0xFBB1u, 0x789Au, 0xE625u, 0x650Eu, 0x6BC4u, 0xE8EFu,
+ 0x31ADu, 0xB286u, 0xBC4Cu, 0x3F67u, 0xA1D8u, 0x22F3u, 0x2C39u, 0xAF12u,
+ 0x9AF0u, 0x19DBu, 0x1711u, 0x943Au, 0x0A85u, 0x89AEu, 0x8764u, 0x044Fu,
+ 0xECA0u, 0x6F8Bu, 0x6141u, 0xE26Au, 0x7CD5u, 0xFFFEu, 0xF134u, 0x721Fu,
+ 0x47FDu, 0xC4D6u, 0xCA1Cu, 0x4937u, 0xD788u, 0x54A3u, 0x5A69u, 0xD942u,
+ 0x635Au, 0xE071u, 0xEEBBu, 0x6D90u, 0xF32Fu, 0x7004u, 0x7ECEu, 0xFDE5u,
+ 0xC807u, 0x4B2Cu, 0x45E6u, 0xC6CDu, 0x5872u, 0xDB59u, 0xD593u, 0x56B8u,
+ 0xBE57u, 0x3D7Cu, 0x33B6u, 0xB09Du, 0x2E22u, 0xAD09u, 0xA3C3u, 0x20E8u,
+ 0x150Au, 0x9621u, 0x98EBu, 0x1BC0u, 0x857Fu, 0x0654u, 0x089Eu, 0x8BB5u,
+ 0x52F7u, 0xD1DCu, 0xDF16u, 0x5C3Du, 0xC282u, 0x41A9u, 0x4F63u, 0xCC48u,
+ 0xF9AAu, 0x7A81u, 0x744Bu, 0xF760u, 0x69DFu, 0xEAF4u, 0xE43Eu, 0x6715u,
+ 0x8FFAu, 0x0CD1u, 0x021Bu, 0x8130u, 0x1F8Fu, 0x9CA4u, 0x926Eu, 0x1145u,
+ 0x24A7u, 0xA78Cu, 0xA946u, 0x2A6Du, 0xB4D2u, 0x37F9u, 0x3933u, 0xBA18u,
+ 0xC6B4u, 0x459Fu, 0x4B55u, 0xC87Eu, 0x56C1u, 0xD5EAu, 0xDB20u, 0x580Bu,
+ 0x6DE9u, 0xEEC2u, 0xE008u, 0x6323u, 0xFD9Cu, 0x7EB7u, 0x707Du, 0xF356u,
+ 0x1BB9u, 0x9892u, 0x9658u, 0x1573u, 0x8BCCu, 0x08E7u, 0x062Du, 0x8506u,
+ 0xB0E4u, 0x33CFu, 0x3D05u, 0xBE2Eu, 0x2091u, 0xA3BAu, 0xAD70u, 0x2E5Bu,
+ 0xF719u, 0x7432u, 0x7AF8u, 0xF9D3u, 0x676Cu, 0xE447u, 0xEA8Du, 0x69A6u,
+ 0x5C44u, 0xDF6Fu, 0xD1A5u, 0x528Eu, 0xCC31u, 0x4F1Au, 0x41D0u, 0xC2FBu,
+ 0x2A14u, 0xA93Fu, 0xA7F5u, 0x24DEu, 0xBA61u, 0x394Au, 0x3780u, 0xB4ABu,
+ 0x8149u, 0x0262u, 0x0CA8u, 0x8F83u, 0x113Cu, 0x9217u, 0x9CDDu, 0x1FF6u,
+ 0xA5EEu, 0x26C5u, 0x280Fu, 0xAB24u, 0x359Bu, 0xB6B0u, 0xB87Au, 0x3B51u,
+ 0x0EB3u, 0x8D98u, 0x8352u, 0x0079u, 0x9EC6u, 0x1DEDu, 0x1327u, 0x900Cu,
+ 0x78E3u, 0xFBC8u, 0xF502u, 0x7629u, 0xE896u, 0x6BBDu, 0x6577u, 0xE65Cu,
+ 0xD3BEu, 0x5095u, 0x5E5Fu, 0xDD74u, 0x43CBu, 0xC0E0u, 0xCE2Au, 0x4D01u,
+ 0x9443u, 0x1768u, 0x19A2u, 0x9A89u, 0x0436u, 0x871Du, 0x89D7u, 0x0AFCu,
+ 0x3F1Eu, 0xBC35u, 0xB2FFu, 0x31D4u, 0xAF6Bu, 0x2C40u, 0x228Au, 0xA1A1u,
+ 0x494Eu, 0xCA65u, 0xC4AFu, 0x4784u, 0xD93Bu, 0x5A10u, 0x54DAu, 0xD7F1u,
+ 0xE213u, 0x6138u, 0x6FF2u, 0xECD9u, 0x7266u, 0xF14Du, 0xFF87u, 0x7CACu
+ }
+};
+
+static inline uint16_t
+crc_update_fast(uint16_t crc, const void *data, size_t data_len)
+{
+ const unsigned char *d = (const unsigned char *)data;
+ const unsigned char *d_end = d + data_len;
+ const unsigned char *d_last16 = d + (data_len & ~0x0F);
+
+ for (; d < d_last16 ; d += 16) {
+ crc = crc_table_fast[15][d[0] ^ (uint8_t)(crc >> 8)] ^
+ crc_table_fast[14][d[1] ^ (uint8_t)(crc >> 0)] ^
+ crc_table_fast[13][d[2]] ^
+ crc_table_fast[12][d[3]] ^
+ crc_table_fast[11][d[4]] ^
+ crc_table_fast[10][d[5]] ^
+ crc_table_fast[9][d[6]] ^
+ crc_table_fast[8][d[7]] ^
+ crc_table_fast[7][d[8]] ^
+ crc_table_fast[6][d[9]] ^
+ crc_table_fast[5][d[10]] ^
+ crc_table_fast[4][d[11]] ^
+ crc_table_fast[3][d[12]] ^
+ crc_table_fast[2][d[13]] ^
+ crc_table_fast[1][d[14]] ^
+ crc_table_fast[0][d[15]];
+ }
+ for (; d < d_end ; d++) {
+ crc = (crc << 8) ^ crc_table_fast[0][((uint8_t)(crc >> 8) ^ *d)];
+ }
+ return crc & 0xffff;
+}
+
+static inline uint16_t
+crc16_table_t10dif(uint16_t init_crc, const void *buf, size_t len)
+{
+ uint16_t crc;
+ const uint8_t *data = (const uint8_t *)buf;
+
+ crc = init_crc;
+ crc = crc_update_fast(crc, data, len);
+ return crc;
+}
+
+uint16_t
+spdk_crc16_t10dif(uint16_t init_crc, const void *buf, size_t len)
+{
+ return (crc16_table_t10dif(init_crc, buf, len));
+}
+
+uint16_t
+spdk_crc16_t10dif_copy(uint16_t init_crc, uint8_t *dst, uint8_t *src, size_t len)
+{
+ memcpy(dst, src, len);
+ return (crc16_table_t10dif(init_crc, src, len));
+}
+
+#endif
diff --git a/src/spdk/lib/util/crc32.c b/src/spdk/lib/util/crc32.c
new file mode 100644
index 000000000..34bb60b78
--- /dev/null
+++ b/src/spdk/lib/util/crc32.c
@@ -0,0 +1,95 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "util_internal.h"
+#include "spdk/crc32.h"
+
+void
+crc32_table_init(struct spdk_crc32_table *table, uint32_t polynomial_reflect)
+{
+ int i, j;
+ uint32_t val;
+
+ for (i = 0; i < 256; i++) {
+ val = i;
+ for (j = 0; j < 8; j++) {
+ if (val & 1) {
+ val = (val >> 1) ^ polynomial_reflect;
+ } else {
+ val = (val >> 1);
+ }
+ }
+ table->table[i] = val;
+ }
+}
+
+#ifdef SPDK_HAVE_ARM_CRC
+
+uint32_t
+crc32_update(const struct spdk_crc32_table *table, const void *buf, size_t len, uint32_t crc)
+{
+ size_t count;
+ const uint64_t *dword_buf;
+
+ count = len & 7;
+ while (count--) {
+ crc = __crc32b(crc, *(const uint8_t *)buf);
+ buf++;
+ }
+ dword_buf = (const uint64_t *)buf;
+
+ count = len / 8;
+ while (count--) {
+ crc = __crc32d(crc, *dword_buf);
+ dword_buf++;
+ }
+
+ return crc;
+}
+
+#else
+
+uint32_t
+crc32_update(const struct spdk_crc32_table *table, const void *buf, size_t len, uint32_t crc)
+{
+ const uint8_t *buf_u8 = buf;
+ size_t i;
+
+ for (i = 0; i < len; i++) {
+ crc = (crc >> 8) ^ table->table[(crc ^ buf_u8[i]) & 0xff];
+ }
+
+ return crc;
+}
+
+#endif
diff --git a/src/spdk/lib/util/crc32_ieee.c b/src/spdk/lib/util/crc32_ieee.c
new file mode 100644
index 000000000..ddc3c9901
--- /dev/null
+++ b/src/spdk/lib/util/crc32_ieee.c
@@ -0,0 +1,49 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "util_internal.h"
+#include "spdk/crc32.h"
+
+static struct spdk_crc32_table g_crc32_ieee_table;
+
+__attribute__((constructor)) static void
+crc32_ieee_init(void)
+{
+ crc32_table_init(&g_crc32_ieee_table, SPDK_CRC32_POLYNOMIAL_REFLECT);
+}
+
+uint32_t
+spdk_crc32_ieee_update(const void *buf, size_t len, uint32_t crc)
+{
+ return crc32_update(&g_crc32_ieee_table, buf, len, crc);
+}
diff --git a/src/spdk/lib/util/crc32c.c b/src/spdk/lib/util/crc32c.c
new file mode 100644
index 000000000..9acd8d80f
--- /dev/null
+++ b/src/spdk/lib/util/crc32c.c
@@ -0,0 +1,133 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "util_internal.h"
+#include "spdk/crc32.h"
+
+#ifdef SPDK_CONFIG_ISAL
+#define SPDK_HAVE_ISAL
+#include <isa-l/include/crc.h>
+#elif defined(__aarch64__) && defined(__ARM_FEATURE_CRC32)
+#define SPDK_HAVE_ARM_CRC
+#include <arm_acle.h>
+#elif defined(__x86_64__) && defined(__SSE4_2__)
+#define SPDK_HAVE_SSE4_2
+#include <x86intrin.h>
+#endif
+
+#ifdef SPDK_HAVE_ISAL
+
+uint32_t
+spdk_crc32c_update(const void *buf, size_t len, uint32_t crc)
+{
+ return crc32_iscsi((unsigned char *)buf, len, crc);
+}
+
+#elif defined(SPDK_HAVE_SSE4_2)
+
+uint32_t
+spdk_crc32c_update(const void *buf, size_t len, uint32_t crc)
+{
+ uint64_t crc_tmp64;
+ size_t count;
+
+ /* _mm_crc32_u64() needs a 64-bit intermediate value */
+ crc_tmp64 = crc;
+
+ /* Process as much of the buffer as possible in 64-bit blocks. */
+ count = len / 8;
+ while (count--) {
+ uint64_t block;
+
+ /*
+ * Use memcpy() to avoid unaligned loads, which are undefined behavior in C.
+ * The compiler will optimize out the memcpy() in release builds.
+ */
+ memcpy(&block, buf, sizeof(block));
+ crc_tmp64 = _mm_crc32_u64(crc_tmp64, block);
+ buf += sizeof(block);
+ }
+ crc = (uint32_t)crc_tmp64;
+
+ /* Handle any trailing bytes. */
+ count = len & 7;
+ while (count--) {
+ crc = _mm_crc32_u8(crc, *(const uint8_t *)buf);
+ buf++;
+ }
+
+ return crc;
+}
+
+#elif defined(SPDK_HAVE_ARM_CRC)
+
+uint32_t
+spdk_crc32c_update(const void *buf, size_t len, uint32_t crc)
+{
+ size_t count;
+
+ count = len / 8;
+ while (count--) {
+ uint64_t block;
+
+ memcpy(&block, buf, sizeof(block));
+ crc = __crc32cd(crc, block);
+ buf += sizeof(block);
+ }
+
+ count = len & 7;
+ while (count--) {
+ crc = __crc32cb(crc, *(const uint8_t *)buf);
+ buf++;
+ }
+
+ return crc;
+}
+
+#else /* Neither SSE 4.2 nor ARM CRC32 instructions available */
+
+static struct spdk_crc32_table g_crc32c_table;
+
+__attribute__((constructor)) static void
+crc32c_init(void)
+{
+ crc32_table_init(&g_crc32c_table, SPDK_CRC32C_POLYNOMIAL_REFLECT);
+}
+
+uint32_t
+spdk_crc32c_update(const void *buf, size_t len, uint32_t crc)
+{
+ return crc32_update(&g_crc32c_table, buf, len, crc);
+}
+
+#endif
diff --git a/src/spdk/lib/util/dif.c b/src/spdk/lib/util/dif.c
new file mode 100644
index 000000000..64bce1487
--- /dev/null
+++ b/src/spdk/lib/util/dif.c
@@ -0,0 +1,1999 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/dif.h"
+#include "spdk/crc16.h"
+#include "spdk/crc32.h"
+#include "spdk/endian.h"
+#include "spdk/log.h"
+#include "spdk/util.h"
+
+/* Context to iterate or create a iovec array.
+ * Each sgl is either iterated or created at a time.
+ */
+struct _dif_sgl {
+ /* Current iovec in the iteration or creation */
+ struct iovec *iov;
+
+ /* Remaining count of iovecs in the iteration or creation. */
+ int iovcnt;
+
+ /* Current offset in the iovec */
+ uint32_t iov_offset;
+
+ /* Size of the created iovec array in bytes */
+ uint32_t total_size;
+};
+
+static inline void
+_dif_sgl_init(struct _dif_sgl *s, struct iovec *iovs, int iovcnt)
+{
+ s->iov = iovs;
+ s->iovcnt = iovcnt;
+ s->iov_offset = 0;
+ s->total_size = 0;
+}
+
+static void
+_dif_sgl_advance(struct _dif_sgl *s, uint32_t step)
+{
+ s->iov_offset += step;
+ while (s->iovcnt != 0) {
+ if (s->iov_offset < s->iov->iov_len) {
+ break;
+ }
+
+ s->iov_offset -= s->iov->iov_len;
+ s->iov++;
+ s->iovcnt--;
+ }
+}
+
+static inline void
+_dif_sgl_get_buf(struct _dif_sgl *s, void **_buf, uint32_t *_buf_len)
+{
+ if (_buf != NULL) {
+ *_buf = s->iov->iov_base + s->iov_offset;
+ }
+ if (_buf_len != NULL) {
+ *_buf_len = s->iov->iov_len - s->iov_offset;
+ }
+}
+
+static inline bool
+_dif_sgl_append(struct _dif_sgl *s, uint8_t *data, uint32_t data_len)
+{
+ assert(s->iovcnt > 0);
+ s->iov->iov_base = data;
+ s->iov->iov_len = data_len;
+ s->total_size += data_len;
+ s->iov++;
+ s->iovcnt--;
+
+ if (s->iovcnt > 0) {
+ return true;
+ } else {
+ return false;
+ }
+}
+
+static inline bool
+_dif_sgl_append_split(struct _dif_sgl *dst, struct _dif_sgl *src, uint32_t data_len)
+{
+ uint8_t *buf;
+ uint32_t buf_len;
+
+ while (data_len != 0) {
+ _dif_sgl_get_buf(src, (void *)&buf, &buf_len);
+ buf_len = spdk_min(buf_len, data_len);
+
+ if (!_dif_sgl_append(dst, buf, buf_len)) {
+ return false;
+ }
+
+ _dif_sgl_advance(src, buf_len);
+ data_len -= buf_len;
+ }
+
+ return true;
+}
+
+/* This function must be used before starting iteration. */
+static bool
+_dif_sgl_is_bytes_multiple(struct _dif_sgl *s, uint32_t bytes)
+{
+ int i;
+
+ for (i = 0; i < s->iovcnt; i++) {
+ if (s->iov[i].iov_len % bytes) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+/* This function must be used before starting iteration. */
+static bool
+_dif_sgl_is_valid(struct _dif_sgl *s, uint32_t bytes)
+{
+ uint64_t total = 0;
+ int i;
+
+ for (i = 0; i < s->iovcnt; i++) {
+ total += s->iov[i].iov_len;
+ }
+
+ return total >= bytes;
+}
+
+static void
+_dif_sgl_copy(struct _dif_sgl *to, struct _dif_sgl *from)
+{
+ memcpy(to, from, sizeof(struct _dif_sgl));
+}
+
+static bool
+_dif_type_is_valid(enum spdk_dif_type dif_type, uint32_t dif_flags)
+{
+ switch (dif_type) {
+ case SPDK_DIF_TYPE1:
+ case SPDK_DIF_TYPE2:
+ case SPDK_DIF_DISABLE:
+ break;
+ case SPDK_DIF_TYPE3:
+ if (dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK) {
+ SPDK_ERRLOG("Reference Tag should not be checked for Type 3\n");
+ return false;
+ }
+ break;
+ default:
+ SPDK_ERRLOG("Unknown DIF Type: %d\n", dif_type);
+ return false;
+ }
+
+ return true;
+}
+
+static bool
+_dif_is_disabled(enum spdk_dif_type dif_type)
+{
+ if (dif_type == SPDK_DIF_DISABLE) {
+ return true;
+ } else {
+ return false;
+ }
+}
+
+
+static uint32_t
+_get_guard_interval(uint32_t block_size, uint32_t md_size, bool dif_loc, bool md_interleave)
+{
+ if (!dif_loc) {
+ /* For metadata formats with more than 8 bytes, if the DIF is
+ * contained in the last 8 bytes of metadata, then the CRC
+ * covers all metadata up to but excluding these last 8 bytes.
+ */
+ if (md_interleave) {
+ return block_size - sizeof(struct spdk_dif);
+ } else {
+ return md_size - sizeof(struct spdk_dif);
+ }
+ } else {
+ /* For metadata formats with more than 8 bytes, if the DIF is
+ * contained in the first 8 bytes of metadata, then the CRC
+ * does not cover any metadata.
+ */
+ if (md_interleave) {
+ return block_size - md_size;
+ } else {
+ return 0;
+ }
+ }
+}
+
+int
+spdk_dif_ctx_init(struct spdk_dif_ctx *ctx, uint32_t block_size, uint32_t md_size,
+ bool md_interleave, bool dif_loc, enum spdk_dif_type dif_type, uint32_t dif_flags,
+ uint32_t init_ref_tag, uint16_t apptag_mask, uint16_t app_tag,
+ uint32_t data_offset, uint16_t guard_seed)
+{
+ uint32_t data_block_size;
+
+ if (md_size < sizeof(struct spdk_dif)) {
+ SPDK_ERRLOG("Metadata size is smaller than DIF size.\n");
+ return -EINVAL;
+ }
+
+ if (md_interleave) {
+ if (block_size < md_size) {
+ SPDK_ERRLOG("Block size is smaller than DIF size.\n");
+ return -EINVAL;
+ }
+ data_block_size = block_size - md_size;
+ } else {
+ if (block_size == 0 || (block_size % 512) != 0) {
+ SPDK_ERRLOG("Zero block size is not allowed\n");
+ return -EINVAL;
+ }
+ data_block_size = block_size;
+ }
+
+ if (!_dif_type_is_valid(dif_type, dif_flags)) {
+ SPDK_ERRLOG("DIF type is invalid.\n");
+ return -EINVAL;
+ }
+
+ ctx->block_size = block_size;
+ ctx->md_size = md_size;
+ ctx->md_interleave = md_interleave;
+ ctx->guard_interval = _get_guard_interval(block_size, md_size, dif_loc, md_interleave);
+ ctx->dif_type = dif_type;
+ ctx->dif_flags = dif_flags;
+ ctx->init_ref_tag = init_ref_tag;
+ ctx->apptag_mask = apptag_mask;
+ ctx->app_tag = app_tag;
+ ctx->data_offset = data_offset;
+ ctx->ref_tag_offset = data_offset / data_block_size;
+ ctx->last_guard = guard_seed;
+ ctx->guard_seed = guard_seed;
+ ctx->remapped_init_ref_tag = 0;
+
+ return 0;
+}
+
+void
+spdk_dif_ctx_set_data_offset(struct spdk_dif_ctx *ctx, uint32_t data_offset)
+{
+ uint32_t data_block_size;
+
+ if (ctx->md_interleave) {
+ data_block_size = ctx->block_size - ctx->md_size;
+ } else {
+ data_block_size = ctx->block_size;
+ }
+
+ ctx->data_offset = data_offset;
+ ctx->ref_tag_offset = data_offset / data_block_size;
+}
+
+void
+spdk_dif_ctx_set_remapped_init_ref_tag(struct spdk_dif_ctx *ctx,
+ uint32_t remapped_init_ref_tag)
+{
+ ctx->remapped_init_ref_tag = remapped_init_ref_tag;
+}
+
+static void
+_dif_generate(void *_dif, uint16_t guard, uint32_t offset_blocks,
+ const struct spdk_dif_ctx *ctx)
+{
+ struct spdk_dif *dif = _dif;
+ uint32_t ref_tag;
+
+ if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) {
+ to_be16(&dif->guard, guard);
+ }
+
+ if (ctx->dif_flags & SPDK_DIF_FLAGS_APPTAG_CHECK) {
+ to_be16(&dif->app_tag, ctx->app_tag);
+ }
+
+ if (ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK) {
+ /* For type 1 and 2, the reference tag is incremented for each
+ * subsequent logical block. For type 3, the reference tag
+ * remains the same as the initial reference tag.
+ */
+ if (ctx->dif_type != SPDK_DIF_TYPE3) {
+ ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset + offset_blocks;
+ } else {
+ ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset;
+ }
+
+ to_be32(&dif->ref_tag, ref_tag);
+ }
+}
+
+static void
+dif_generate(struct _dif_sgl *sgl, uint32_t num_blocks, const struct spdk_dif_ctx *ctx)
+{
+ uint32_t offset_blocks = 0;
+ void *buf;
+ uint16_t guard = 0;
+
+ while (offset_blocks < num_blocks) {
+ _dif_sgl_get_buf(sgl, &buf, NULL);
+
+ if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) {
+ guard = spdk_crc16_t10dif(ctx->guard_seed, buf, ctx->guard_interval);
+ }
+
+ _dif_generate(buf + ctx->guard_interval, guard, offset_blocks, ctx);
+
+ _dif_sgl_advance(sgl, ctx->block_size);
+ offset_blocks++;
+ }
+}
+
+static uint16_t
+_dif_generate_split(struct _dif_sgl *sgl, uint32_t offset_in_block, uint32_t data_len,
+ uint16_t guard, uint32_t offset_blocks, const struct spdk_dif_ctx *ctx)
+{
+ uint32_t offset_in_dif, buf_len;
+ void *buf;
+ struct spdk_dif dif = {};
+
+ assert(offset_in_block < ctx->guard_interval);
+ assert(offset_in_block + data_len < ctx->guard_interval ||
+ offset_in_block + data_len == ctx->block_size);
+
+ /* Compute CRC over split logical block data. */
+ while (data_len != 0 && offset_in_block < ctx->guard_interval) {
+ _dif_sgl_get_buf(sgl, &buf, &buf_len);
+ buf_len = spdk_min(buf_len, data_len);
+ buf_len = spdk_min(buf_len, ctx->guard_interval - offset_in_block);
+
+ if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) {
+ guard = spdk_crc16_t10dif(guard, buf, buf_len);
+ }
+
+ _dif_sgl_advance(sgl, buf_len);
+ offset_in_block += buf_len;
+ data_len -= buf_len;
+ }
+
+ if (offset_in_block < ctx->guard_interval) {
+ return guard;
+ }
+
+ /* If a whole logical block data is parsed, generate DIF
+ * and save it to the temporary DIF area.
+ */
+ _dif_generate(&dif, guard, offset_blocks, ctx);
+
+ /* Copy generated DIF field to the split DIF field, and then
+ * skip metadata field after DIF field (if any).
+ */
+ while (offset_in_block < ctx->block_size) {
+ _dif_sgl_get_buf(sgl, &buf, &buf_len);
+
+ if (offset_in_block < ctx->guard_interval + sizeof(struct spdk_dif)) {
+ offset_in_dif = offset_in_block - ctx->guard_interval;
+ buf_len = spdk_min(buf_len, sizeof(struct spdk_dif) - offset_in_dif);
+
+ memcpy(buf, ((uint8_t *)&dif) + offset_in_dif, buf_len);
+ } else {
+ buf_len = spdk_min(buf_len, ctx->block_size - offset_in_block);
+ }
+
+ _dif_sgl_advance(sgl, buf_len);
+ offset_in_block += buf_len;
+ }
+
+ if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) {
+ guard = ctx->guard_seed;
+ }
+
+ return guard;
+}
+
+static void
+dif_generate_split(struct _dif_sgl *sgl, uint32_t num_blocks,
+ const struct spdk_dif_ctx *ctx)
+{
+ uint32_t offset_blocks;
+ uint16_t guard = 0;
+
+ if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) {
+ guard = ctx->guard_seed;
+ }
+
+ for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) {
+ _dif_generate_split(sgl, 0, ctx->block_size, guard, offset_blocks, ctx);
+ }
+}
+
+int
+spdk_dif_generate(struct iovec *iovs, int iovcnt, uint32_t num_blocks,
+ const struct spdk_dif_ctx *ctx)
+{
+ struct _dif_sgl sgl;
+
+ _dif_sgl_init(&sgl, iovs, iovcnt);
+
+ if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) {
+ SPDK_ERRLOG("Size of iovec array is not valid.\n");
+ return -EINVAL;
+ }
+
+ if (_dif_is_disabled(ctx->dif_type)) {
+ return 0;
+ }
+
+ if (_dif_sgl_is_bytes_multiple(&sgl, ctx->block_size)) {
+ dif_generate(&sgl, num_blocks, ctx);
+ } else {
+ dif_generate_split(&sgl, num_blocks, ctx);
+ }
+
+ return 0;
+}
+
+static void
+_dif_error_set(struct spdk_dif_error *err_blk, uint8_t err_type,
+ uint32_t expected, uint32_t actual, uint32_t err_offset)
+{
+ if (err_blk) {
+ err_blk->err_type = err_type;
+ err_blk->expected = expected;
+ err_blk->actual = actual;
+ err_blk->err_offset = err_offset;
+ }
+}
+
+static int
+_dif_verify(void *_dif, uint16_t guard, uint32_t offset_blocks,
+ const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk)
+{
+ struct spdk_dif *dif = _dif;
+ uint16_t _guard;
+ uint16_t _app_tag;
+ uint32_t ref_tag, _ref_tag;
+
+ switch (ctx->dif_type) {
+ case SPDK_DIF_TYPE1:
+ case SPDK_DIF_TYPE2:
+ /* If Type 1 or 2 is used, then all DIF checks are disabled when
+ * the Application Tag is 0xFFFF.
+ */
+ if (dif->app_tag == 0xFFFF) {
+ return 0;
+ }
+ break;
+ case SPDK_DIF_TYPE3:
+ /* If Type 3 is used, then all DIF checks are disabled when the
+ * Application Tag is 0xFFFF and the Reference Tag is 0xFFFFFFFF.
+ */
+ if (dif->app_tag == 0xFFFF && dif->ref_tag == 0xFFFFFFFF) {
+ return 0;
+ }
+ break;
+ default:
+ break;
+ }
+
+ /* For type 1 and 2, the reference tag is incremented for each
+ * subsequent logical block. For type 3, the reference tag
+ * remains the same as the initial reference tag.
+ */
+ if (ctx->dif_type != SPDK_DIF_TYPE3) {
+ ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset + offset_blocks;
+ } else {
+ ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset;
+ }
+
+ if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) {
+ /* Compare the DIF Guard field to the CRC computed over the logical
+ * block data.
+ */
+ _guard = from_be16(&dif->guard);
+ if (_guard != guard) {
+ _dif_error_set(err_blk, SPDK_DIF_GUARD_ERROR, _guard, guard,
+ offset_blocks);
+ SPDK_ERRLOG("Failed to compare Guard: LBA=%" PRIu32 "," \
+ " Expected=%x, Actual=%x\n",
+ ref_tag, _guard, guard);
+ return -1;
+ }
+ }
+
+ if (ctx->dif_flags & SPDK_DIF_FLAGS_APPTAG_CHECK) {
+ /* Compare unmasked bits in the DIF Application Tag field to the
+ * passed Application Tag.
+ */
+ _app_tag = from_be16(&dif->app_tag);
+ if ((_app_tag & ctx->apptag_mask) != ctx->app_tag) {
+ _dif_error_set(err_blk, SPDK_DIF_APPTAG_ERROR, ctx->app_tag,
+ (_app_tag & ctx->apptag_mask), offset_blocks);
+ SPDK_ERRLOG("Failed to compare App Tag: LBA=%" PRIu32 "," \
+ " Expected=%x, Actual=%x\n",
+ ref_tag, ctx->app_tag, (_app_tag & ctx->apptag_mask));
+ return -1;
+ }
+ }
+
+ if (ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK) {
+ switch (ctx->dif_type) {
+ case SPDK_DIF_TYPE1:
+ case SPDK_DIF_TYPE2:
+ /* Compare the DIF Reference Tag field to the passed Reference Tag.
+ * The passed Reference Tag will be the least significant 4 bytes
+ * of the LBA when Type 1 is used, and application specific value
+ * if Type 2 is used,
+ */
+ _ref_tag = from_be32(&dif->ref_tag);
+ if (_ref_tag != ref_tag) {
+ _dif_error_set(err_blk, SPDK_DIF_REFTAG_ERROR, ref_tag,
+ _ref_tag, offset_blocks);
+ SPDK_ERRLOG("Failed to compare Ref Tag: LBA=%" PRIu32 "," \
+ " Expected=%x, Actual=%x\n",
+ ref_tag, ref_tag, _ref_tag);
+ return -1;
+ }
+ break;
+ case SPDK_DIF_TYPE3:
+ /* For Type 3, computed Reference Tag remains unchanged.
+ * Hence ignore the Reference Tag field.
+ */
+ break;
+ default:
+ break;
+ }
+ }
+
+ return 0;
+}
+
+static int
+dif_verify(struct _dif_sgl *sgl, uint32_t num_blocks,
+ const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk)
+{
+ uint32_t offset_blocks = 0;
+ int rc;
+ void *buf;
+ uint16_t guard = 0;
+
+ while (offset_blocks < num_blocks) {
+ _dif_sgl_get_buf(sgl, &buf, NULL);
+
+ if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) {
+ guard = spdk_crc16_t10dif(ctx->guard_seed, buf, ctx->guard_interval);
+ }
+
+ rc = _dif_verify(buf + ctx->guard_interval, guard, offset_blocks, ctx, err_blk);
+ if (rc != 0) {
+ return rc;
+ }
+
+ _dif_sgl_advance(sgl, ctx->block_size);
+ offset_blocks++;
+ }
+
+ return 0;
+}
+
+static int
+_dif_verify_split(struct _dif_sgl *sgl, uint32_t offset_in_block, uint32_t data_len,
+ uint16_t *_guard, uint32_t offset_blocks,
+ const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk)
+{
+ uint32_t offset_in_dif, buf_len;
+ void *buf;
+ uint16_t guard;
+ struct spdk_dif dif = {};
+ int rc;
+
+ assert(_guard != NULL);
+ assert(offset_in_block < ctx->guard_interval);
+ assert(offset_in_block + data_len < ctx->guard_interval ||
+ offset_in_block + data_len == ctx->block_size);
+
+ guard = *_guard;
+
+ /* Compute CRC over split logical block data. */
+ while (data_len != 0 && offset_in_block < ctx->guard_interval) {
+ _dif_sgl_get_buf(sgl, &buf, &buf_len);
+ buf_len = spdk_min(buf_len, data_len);
+ buf_len = spdk_min(buf_len, ctx->guard_interval - offset_in_block);
+
+ if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) {
+ guard = spdk_crc16_t10dif(guard, buf, buf_len);
+ }
+
+ _dif_sgl_advance(sgl, buf_len);
+ offset_in_block += buf_len;
+ data_len -= buf_len;
+ }
+
+ if (offset_in_block < ctx->guard_interval) {
+ *_guard = guard;
+ return 0;
+ }
+
+ /* Copy the split DIF field to the temporary DIF buffer, and then
+ * skip metadata field after DIF field (if any). */
+ while (offset_in_block < ctx->block_size) {
+ _dif_sgl_get_buf(sgl, &buf, &buf_len);
+
+ if (offset_in_block < ctx->guard_interval + sizeof(struct spdk_dif)) {
+ offset_in_dif = offset_in_block - ctx->guard_interval;
+ buf_len = spdk_min(buf_len, sizeof(struct spdk_dif) - offset_in_dif);
+
+ memcpy((uint8_t *)&dif + offset_in_dif, buf, buf_len);
+ } else {
+ buf_len = spdk_min(buf_len, ctx->block_size - offset_in_block);
+ }
+ _dif_sgl_advance(sgl, buf_len);
+ offset_in_block += buf_len;
+ }
+
+ rc = _dif_verify(&dif, guard, offset_blocks, ctx, err_blk);
+ if (rc != 0) {
+ return rc;
+ }
+
+ if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) {
+ guard = ctx->guard_seed;
+ }
+
+ *_guard = guard;
+ return 0;
+}
+
+static int
+dif_verify_split(struct _dif_sgl *sgl, uint32_t num_blocks,
+ const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk)
+{
+ uint32_t offset_blocks;
+ uint16_t guard = 0;
+ int rc;
+
+ if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) {
+ guard = ctx->guard_seed;
+ }
+
+ for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) {
+ rc = _dif_verify_split(sgl, 0, ctx->block_size, &guard, offset_blocks,
+ ctx, err_blk);
+ if (rc != 0) {
+ return rc;
+ }
+ }
+
+ return 0;
+}
+
+int
+spdk_dif_verify(struct iovec *iovs, int iovcnt, uint32_t num_blocks,
+ const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk)
+{
+ struct _dif_sgl sgl;
+
+ _dif_sgl_init(&sgl, iovs, iovcnt);
+
+ if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) {
+ SPDK_ERRLOG("Size of iovec array is not valid.\n");
+ return -EINVAL;
+ }
+
+ if (_dif_is_disabled(ctx->dif_type)) {
+ return 0;
+ }
+
+ if (_dif_sgl_is_bytes_multiple(&sgl, ctx->block_size)) {
+ return dif_verify(&sgl, num_blocks, ctx, err_blk);
+ } else {
+ return dif_verify_split(&sgl, num_blocks, ctx, err_blk);
+ }
+}
+
+static uint32_t
+dif_update_crc32c(struct _dif_sgl *sgl, uint32_t num_blocks,
+ uint32_t crc32c, const struct spdk_dif_ctx *ctx)
+{
+ uint32_t offset_blocks;
+ void *buf;
+
+ for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) {
+ _dif_sgl_get_buf(sgl, &buf, NULL);
+
+ crc32c = spdk_crc32c_update(buf, ctx->block_size - ctx->md_size, crc32c);
+
+ _dif_sgl_advance(sgl, ctx->block_size);
+ }
+
+ return crc32c;
+}
+
+static uint32_t
+_dif_update_crc32c_split(struct _dif_sgl *sgl, uint32_t offset_in_block, uint32_t data_len,
+ uint32_t crc32c, const struct spdk_dif_ctx *ctx)
+{
+ uint32_t data_block_size, buf_len;
+ void *buf;
+
+ data_block_size = ctx->block_size - ctx->md_size;
+
+ assert(offset_in_block + data_len <= ctx->block_size);
+
+ while (data_len != 0) {
+ _dif_sgl_get_buf(sgl, &buf, &buf_len);
+ buf_len = spdk_min(buf_len, data_len);
+
+ if (offset_in_block < data_block_size) {
+ buf_len = spdk_min(buf_len, data_block_size - offset_in_block);
+ crc32c = spdk_crc32c_update(buf, buf_len, crc32c);
+ }
+
+ _dif_sgl_advance(sgl, buf_len);
+ offset_in_block += buf_len;
+ data_len -= buf_len;
+ }
+
+ return crc32c;
+}
+
+static uint32_t
+dif_update_crc32c_split(struct _dif_sgl *sgl, uint32_t num_blocks,
+ uint32_t crc32c, const struct spdk_dif_ctx *ctx)
+{
+ uint32_t offset_blocks;
+
+ for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) {
+ crc32c = _dif_update_crc32c_split(sgl, 0, ctx->block_size, crc32c, ctx);
+ }
+
+ return crc32c;
+}
+
+int
+spdk_dif_update_crc32c(struct iovec *iovs, int iovcnt, uint32_t num_blocks,
+ uint32_t *_crc32c, const struct spdk_dif_ctx *ctx)
+{
+ struct _dif_sgl sgl;
+
+ if (_crc32c == NULL) {
+ return -EINVAL;
+ }
+
+ _dif_sgl_init(&sgl, iovs, iovcnt);
+
+ if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) {
+ SPDK_ERRLOG("Size of iovec array is not valid.\n");
+ return -EINVAL;
+ }
+
+ if (_dif_sgl_is_bytes_multiple(&sgl, ctx->block_size)) {
+ *_crc32c = dif_update_crc32c(&sgl, num_blocks, *_crc32c, ctx);
+ } else {
+ *_crc32c = dif_update_crc32c_split(&sgl, num_blocks, *_crc32c, ctx);
+ }
+
+ return 0;
+}
+
+static void
+dif_generate_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl,
+ uint32_t num_blocks, const struct spdk_dif_ctx *ctx)
+{
+ uint32_t offset_blocks = 0, data_block_size;
+ void *src, *dst;
+ uint16_t guard;
+
+ data_block_size = ctx->block_size - ctx->md_size;
+
+ while (offset_blocks < num_blocks) {
+ _dif_sgl_get_buf(src_sgl, &src, NULL);
+ _dif_sgl_get_buf(dst_sgl, &dst, NULL);
+
+ guard = 0;
+ if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) {
+ guard = spdk_crc16_t10dif_copy(ctx->guard_seed, dst, src, data_block_size);
+ guard = spdk_crc16_t10dif(guard, dst + data_block_size,
+ ctx->guard_interval - data_block_size);
+ } else {
+ memcpy(dst, src, data_block_size);
+ }
+
+ _dif_generate(dst + ctx->guard_interval, guard, offset_blocks, ctx);
+
+ _dif_sgl_advance(src_sgl, data_block_size);
+ _dif_sgl_advance(dst_sgl, ctx->block_size);
+ offset_blocks++;
+ }
+}
+
+static void
+_dif_generate_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl,
+ uint32_t offset_blocks, const struct spdk_dif_ctx *ctx)
+{
+ uint32_t offset_in_block, src_len, data_block_size;
+ uint16_t guard = 0;
+ void *src, *dst;
+
+ _dif_sgl_get_buf(dst_sgl, &dst, NULL);
+
+ data_block_size = ctx->block_size - ctx->md_size;
+
+ if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) {
+ guard = ctx->guard_seed;
+ }
+ offset_in_block = 0;
+
+ while (offset_in_block < data_block_size) {
+ /* Compute CRC over split logical block data and copy
+ * data to bounce buffer.
+ */
+ _dif_sgl_get_buf(src_sgl, &src, &src_len);
+ src_len = spdk_min(src_len, data_block_size - offset_in_block);
+
+ if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) {
+ guard = spdk_crc16_t10dif_copy(guard, dst + offset_in_block,
+ src, src_len);
+ } else {
+ memcpy(dst + offset_in_block, src, src_len);
+ }
+
+ _dif_sgl_advance(src_sgl, src_len);
+ offset_in_block += src_len;
+ }
+
+ if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) {
+ guard = spdk_crc16_t10dif(guard, dst + data_block_size,
+ ctx->guard_interval - data_block_size);
+ }
+
+ _dif_sgl_advance(dst_sgl, ctx->block_size);
+
+ _dif_generate(dst + ctx->guard_interval, guard, offset_blocks, ctx);
+}
+
+static void
+dif_generate_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl,
+ uint32_t num_blocks, const struct spdk_dif_ctx *ctx)
+{
+ uint32_t offset_blocks;
+
+ for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) {
+ _dif_generate_copy_split(src_sgl, dst_sgl, offset_blocks, ctx);
+ }
+}
+
+int
+spdk_dif_generate_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iov,
+ uint32_t num_blocks, const struct spdk_dif_ctx *ctx)
+{
+ struct _dif_sgl src_sgl, dst_sgl;
+ uint32_t data_block_size;
+
+ _dif_sgl_init(&src_sgl, iovs, iovcnt);
+ _dif_sgl_init(&dst_sgl, bounce_iov, 1);
+
+ data_block_size = ctx->block_size - ctx->md_size;
+
+ if (!_dif_sgl_is_valid(&src_sgl, data_block_size * num_blocks) ||
+ !_dif_sgl_is_valid(&dst_sgl, ctx->block_size * num_blocks)) {
+ SPDK_ERRLOG("Size of iovec arrays are not valid.\n");
+ return -EINVAL;
+ }
+
+ if (_dif_is_disabled(ctx->dif_type)) {
+ return 0;
+ }
+
+ if (_dif_sgl_is_bytes_multiple(&src_sgl, data_block_size)) {
+ dif_generate_copy(&src_sgl, &dst_sgl, num_blocks, ctx);
+ } else {
+ dif_generate_copy_split(&src_sgl, &dst_sgl, num_blocks, ctx);
+ }
+
+ return 0;
+}
+
+static int
+dif_verify_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl,
+ uint32_t num_blocks, const struct spdk_dif_ctx *ctx,
+ struct spdk_dif_error *err_blk)
+{
+ uint32_t offset_blocks = 0, data_block_size;
+ void *src, *dst;
+ int rc;
+ uint16_t guard;
+
+ data_block_size = ctx->block_size - ctx->md_size;
+
+ while (offset_blocks < num_blocks) {
+ _dif_sgl_get_buf(src_sgl, &src, NULL);
+ _dif_sgl_get_buf(dst_sgl, &dst, NULL);
+
+ guard = 0;
+ if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) {
+ guard = spdk_crc16_t10dif_copy(ctx->guard_seed, dst, src, data_block_size);
+ guard = spdk_crc16_t10dif(guard, src + data_block_size,
+ ctx->guard_interval - data_block_size);
+ } else {
+ memcpy(dst, src, data_block_size);
+ }
+
+ rc = _dif_verify(src + ctx->guard_interval, guard, offset_blocks, ctx, err_blk);
+ if (rc != 0) {
+ return rc;
+ }
+
+ _dif_sgl_advance(src_sgl, ctx->block_size);
+ _dif_sgl_advance(dst_sgl, data_block_size);
+ offset_blocks++;
+ }
+
+ return 0;
+}
+
+static int
+_dif_verify_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl,
+ uint32_t offset_blocks, const struct spdk_dif_ctx *ctx,
+ struct spdk_dif_error *err_blk)
+{
+ uint32_t offset_in_block, dst_len, data_block_size;
+ uint16_t guard = 0;
+ void *src, *dst;
+
+ _dif_sgl_get_buf(src_sgl, &src, NULL);
+
+ data_block_size = ctx->block_size - ctx->md_size;
+
+ if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) {
+ guard = ctx->guard_seed;
+ }
+ offset_in_block = 0;
+
+ while (offset_in_block < data_block_size) {
+ /* Compute CRC over split logical block data and copy
+ * data to bounce buffer.
+ */
+ _dif_sgl_get_buf(dst_sgl, &dst, &dst_len);
+ dst_len = spdk_min(dst_len, data_block_size - offset_in_block);
+
+ if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) {
+ guard = spdk_crc16_t10dif_copy(guard, dst,
+ src + offset_in_block, dst_len);
+ } else {
+ memcpy(dst, src + offset_in_block, dst_len);
+ }
+
+ _dif_sgl_advance(dst_sgl, dst_len);
+ offset_in_block += dst_len;
+ }
+
+ if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) {
+ guard = spdk_crc16_t10dif(guard, src + data_block_size,
+ ctx->guard_interval - data_block_size);
+ }
+
+ _dif_sgl_advance(src_sgl, ctx->block_size);
+
+ return _dif_verify(src + ctx->guard_interval, guard, offset_blocks, ctx, err_blk);
+}
+
+static int
+dif_verify_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl,
+ uint32_t num_blocks, const struct spdk_dif_ctx *ctx,
+ struct spdk_dif_error *err_blk)
+{
+ uint32_t offset_blocks;
+ int rc;
+
+ for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) {
+ rc = _dif_verify_copy_split(src_sgl, dst_sgl, offset_blocks, ctx, err_blk);
+ if (rc != 0) {
+ return rc;
+ }
+ }
+
+ return 0;
+}
+
+int
+spdk_dif_verify_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iov,
+ uint32_t num_blocks, const struct spdk_dif_ctx *ctx,
+ struct spdk_dif_error *err_blk)
+{
+ struct _dif_sgl src_sgl, dst_sgl;
+ uint32_t data_block_size;
+
+ _dif_sgl_init(&src_sgl, bounce_iov, 1);
+ _dif_sgl_init(&dst_sgl, iovs, iovcnt);
+
+ data_block_size = ctx->block_size - ctx->md_size;
+
+ if (!_dif_sgl_is_valid(&dst_sgl, data_block_size * num_blocks) ||
+ !_dif_sgl_is_valid(&src_sgl, ctx->block_size * num_blocks)) {
+ SPDK_ERRLOG("Size of iovec arrays are not valid\n");
+ return -EINVAL;
+ }
+
+ if (_dif_is_disabled(ctx->dif_type)) {
+ return 0;
+ }
+
+ if (_dif_sgl_is_bytes_multiple(&dst_sgl, data_block_size)) {
+ return dif_verify_copy(&src_sgl, &dst_sgl, num_blocks, ctx, err_blk);
+ } else {
+ return dif_verify_copy_split(&src_sgl, &dst_sgl, num_blocks, ctx, err_blk);
+ }
+}
+
+static void
+_bit_flip(uint8_t *buf, uint32_t flip_bit)
+{
+ uint8_t byte;
+
+ byte = *buf;
+ byte ^= 1 << flip_bit;
+ *buf = byte;
+}
+
+static int
+_dif_inject_error(struct _dif_sgl *sgl,
+ uint32_t block_size, uint32_t num_blocks,
+ uint32_t inject_offset_blocks,
+ uint32_t inject_offset_bytes,
+ uint32_t inject_offset_bits)
+{
+ uint32_t offset_in_block, buf_len;
+ void *buf;
+
+ _dif_sgl_advance(sgl, block_size * inject_offset_blocks);
+
+ offset_in_block = 0;
+
+ while (offset_in_block < block_size) {
+ _dif_sgl_get_buf(sgl, &buf, &buf_len);
+ buf_len = spdk_min(buf_len, block_size - offset_in_block);
+
+ if (inject_offset_bytes >= offset_in_block &&
+ inject_offset_bytes < offset_in_block + buf_len) {
+ buf += inject_offset_bytes - offset_in_block;
+ _bit_flip(buf, inject_offset_bits);
+ return 0;
+ }
+
+ _dif_sgl_advance(sgl, buf_len);
+ offset_in_block += buf_len;
+ }
+
+ return -1;
+}
+
+static int
+dif_inject_error(struct _dif_sgl *sgl, uint32_t block_size, uint32_t num_blocks,
+ uint32_t start_inject_bytes, uint32_t inject_range_bytes,
+ uint32_t *inject_offset)
+{
+ uint32_t inject_offset_blocks, inject_offset_bytes, inject_offset_bits;
+ uint32_t offset_blocks;
+ int rc;
+
+ srand(time(0));
+
+ inject_offset_blocks = rand() % num_blocks;
+ inject_offset_bytes = start_inject_bytes + (rand() % inject_range_bytes);
+ inject_offset_bits = rand() % 8;
+
+ for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) {
+ if (offset_blocks == inject_offset_blocks) {
+ rc = _dif_inject_error(sgl, block_size, num_blocks,
+ inject_offset_blocks,
+ inject_offset_bytes,
+ inject_offset_bits);
+ if (rc == 0) {
+ *inject_offset = inject_offset_blocks;
+ }
+ return rc;
+ }
+ }
+
+ return -1;
+}
+
+#define _member_size(type, member) sizeof(((type *)0)->member)
+
+int
+spdk_dif_inject_error(struct iovec *iovs, int iovcnt, uint32_t num_blocks,
+ const struct spdk_dif_ctx *ctx, uint32_t inject_flags,
+ uint32_t *inject_offset)
+{
+ struct _dif_sgl sgl;
+ int rc;
+
+ _dif_sgl_init(&sgl, iovs, iovcnt);
+
+ if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) {
+ SPDK_ERRLOG("Size of iovec array is not valid.\n");
+ return -EINVAL;
+ }
+
+ if (inject_flags & SPDK_DIF_REFTAG_ERROR) {
+ rc = dif_inject_error(&sgl, ctx->block_size, num_blocks,
+ ctx->guard_interval + offsetof(struct spdk_dif, ref_tag),
+ _member_size(struct spdk_dif, ref_tag),
+ inject_offset);
+ if (rc != 0) {
+ SPDK_ERRLOG("Failed to inject error to Reference Tag.\n");
+ return rc;
+ }
+ }
+
+ if (inject_flags & SPDK_DIF_APPTAG_ERROR) {
+ rc = dif_inject_error(&sgl, ctx->block_size, num_blocks,
+ ctx->guard_interval + offsetof(struct spdk_dif, app_tag),
+ _member_size(struct spdk_dif, app_tag),
+ inject_offset);
+ if (rc != 0) {
+ SPDK_ERRLOG("Failed to inject error to Application Tag.\n");
+ return rc;
+ }
+ }
+ if (inject_flags & SPDK_DIF_GUARD_ERROR) {
+ rc = dif_inject_error(&sgl, ctx->block_size, num_blocks,
+ ctx->guard_interval,
+ _member_size(struct spdk_dif, guard),
+ inject_offset);
+ if (rc != 0) {
+ SPDK_ERRLOG("Failed to inject error to Guard.\n");
+ return rc;
+ }
+ }
+
+ if (inject_flags & SPDK_DIF_DATA_ERROR) {
+ /* If the DIF information is contained within the last 8 bytes of
+ * metadata, then the CRC covers all metadata bytes up to but excluding
+ * the last 8 bytes. But error injection does not cover these metadata
+ * because classification is not determined yet.
+ *
+ * Note: Error injection to data block is expected to be detected as
+ * guard error.
+ */
+ rc = dif_inject_error(&sgl, ctx->block_size, num_blocks,
+ 0,
+ ctx->block_size - ctx->md_size,
+ inject_offset);
+ if (rc != 0) {
+ SPDK_ERRLOG("Failed to inject error to data block.\n");
+ return rc;
+ }
+ }
+
+ return 0;
+}
+
+static void
+dix_generate(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl,
+ uint32_t num_blocks, const struct spdk_dif_ctx *ctx)
+{
+ uint32_t offset_blocks = 0;
+ uint16_t guard;
+ void *data_buf, *md_buf;
+
+ while (offset_blocks < num_blocks) {
+ _dif_sgl_get_buf(data_sgl, &data_buf, NULL);
+ _dif_sgl_get_buf(md_sgl, &md_buf, NULL);
+
+ guard = 0;
+ if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) {
+ guard = spdk_crc16_t10dif(ctx->guard_seed, data_buf, ctx->block_size);
+ guard = spdk_crc16_t10dif(guard, md_buf, ctx->guard_interval);
+ }
+
+ _dif_generate(md_buf + ctx->guard_interval, guard, offset_blocks, ctx);
+
+ _dif_sgl_advance(data_sgl, ctx->block_size);
+ _dif_sgl_advance(md_sgl, ctx->md_size);
+ offset_blocks++;
+ }
+}
+
+static void
+_dix_generate_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl,
+ uint32_t offset_blocks, const struct spdk_dif_ctx *ctx)
+{
+ uint32_t offset_in_block, data_buf_len;
+ uint16_t guard = 0;
+ void *data_buf, *md_buf;
+
+ _dif_sgl_get_buf(md_sgl, &md_buf, NULL);
+
+ if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) {
+ guard = ctx->guard_seed;
+ }
+ offset_in_block = 0;
+
+ while (offset_in_block < ctx->block_size) {
+ _dif_sgl_get_buf(data_sgl, &data_buf, &data_buf_len);
+ data_buf_len = spdk_min(data_buf_len, ctx->block_size - offset_in_block);
+
+ if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) {
+ guard = spdk_crc16_t10dif(guard, data_buf, data_buf_len);
+ }
+
+ _dif_sgl_advance(data_sgl, data_buf_len);
+ offset_in_block += data_buf_len;
+ }
+
+ if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) {
+ guard = spdk_crc16_t10dif(guard, md_buf, ctx->guard_interval);
+ }
+
+ _dif_sgl_advance(md_sgl, ctx->md_size);
+
+ _dif_generate(md_buf + ctx->guard_interval, guard, offset_blocks, ctx);
+}
+
+static void
+dix_generate_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl,
+ uint32_t num_blocks, const struct spdk_dif_ctx *ctx)
+{
+ uint32_t offset_blocks;
+
+ for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) {
+ _dix_generate_split(data_sgl, md_sgl, offset_blocks, ctx);
+ }
+}
+
+int
+spdk_dix_generate(struct iovec *iovs, int iovcnt, struct iovec *md_iov,
+ uint32_t num_blocks, const struct spdk_dif_ctx *ctx)
+{
+ struct _dif_sgl data_sgl, md_sgl;
+
+ _dif_sgl_init(&data_sgl, iovs, iovcnt);
+ _dif_sgl_init(&md_sgl, md_iov, 1);
+
+ if (!_dif_sgl_is_valid(&data_sgl, ctx->block_size * num_blocks) ||
+ !_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) {
+ SPDK_ERRLOG("Size of iovec array is not valid.\n");
+ return -EINVAL;
+ }
+
+ if (_dif_is_disabled(ctx->dif_type)) {
+ return 0;
+ }
+
+ if (_dif_sgl_is_bytes_multiple(&data_sgl, ctx->block_size)) {
+ dix_generate(&data_sgl, &md_sgl, num_blocks, ctx);
+ } else {
+ dix_generate_split(&data_sgl, &md_sgl, num_blocks, ctx);
+ }
+
+ return 0;
+}
+
+static int
+dix_verify(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl,
+ uint32_t num_blocks, const struct spdk_dif_ctx *ctx,
+ struct spdk_dif_error *err_blk)
+{
+ uint32_t offset_blocks = 0;
+ uint16_t guard;
+ void *data_buf, *md_buf;
+ int rc;
+
+ while (offset_blocks < num_blocks) {
+ _dif_sgl_get_buf(data_sgl, &data_buf, NULL);
+ _dif_sgl_get_buf(md_sgl, &md_buf, NULL);
+
+ guard = 0;
+ if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) {
+ guard = spdk_crc16_t10dif(ctx->guard_seed, data_buf, ctx->block_size);
+ guard = spdk_crc16_t10dif(guard, md_buf, ctx->guard_interval);
+ }
+
+ rc = _dif_verify(md_buf + ctx->guard_interval, guard, offset_blocks, ctx, err_blk);
+ if (rc != 0) {
+ return rc;
+ }
+
+ _dif_sgl_advance(data_sgl, ctx->block_size);
+ _dif_sgl_advance(md_sgl, ctx->md_size);
+ offset_blocks++;
+ }
+
+ return 0;
+}
+
+static int
+_dix_verify_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl,
+ uint32_t offset_blocks, const struct spdk_dif_ctx *ctx,
+ struct spdk_dif_error *err_blk)
+{
+ uint32_t offset_in_block, data_buf_len;
+ uint16_t guard = 0;
+ void *data_buf, *md_buf;
+
+ _dif_sgl_get_buf(md_sgl, &md_buf, NULL);
+
+ if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) {
+ guard = ctx->guard_seed;
+ }
+ offset_in_block = 0;
+
+ while (offset_in_block < ctx->block_size) {
+ _dif_sgl_get_buf(data_sgl, &data_buf, &data_buf_len);
+ data_buf_len = spdk_min(data_buf_len, ctx->block_size - offset_in_block);
+
+ if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) {
+ guard = spdk_crc16_t10dif(guard, data_buf, data_buf_len);
+ }
+
+ _dif_sgl_advance(data_sgl, data_buf_len);
+ offset_in_block += data_buf_len;
+ }
+
+ if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) {
+ guard = spdk_crc16_t10dif(guard, md_buf, ctx->guard_interval);
+ }
+
+ _dif_sgl_advance(md_sgl, ctx->md_size);
+
+ return _dif_verify(md_buf + ctx->guard_interval, guard, offset_blocks, ctx, err_blk);
+}
+
+static int
+dix_verify_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl,
+ uint32_t num_blocks, const struct spdk_dif_ctx *ctx,
+ struct spdk_dif_error *err_blk)
+{
+ uint32_t offset_blocks;
+ int rc;
+
+ for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) {
+ rc = _dix_verify_split(data_sgl, md_sgl, offset_blocks, ctx, err_blk);
+ if (rc != 0) {
+ return rc;
+ }
+ }
+
+ return 0;
+}
+
+int
+spdk_dix_verify(struct iovec *iovs, int iovcnt, struct iovec *md_iov,
+ uint32_t num_blocks, const struct spdk_dif_ctx *ctx,
+ struct spdk_dif_error *err_blk)
+{
+ struct _dif_sgl data_sgl, md_sgl;
+
+ _dif_sgl_init(&data_sgl, iovs, iovcnt);
+ _dif_sgl_init(&md_sgl, md_iov, 1);
+
+ if (!_dif_sgl_is_valid(&data_sgl, ctx->block_size * num_blocks) ||
+ !_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) {
+ SPDK_ERRLOG("Size of iovec array is not valid.\n");
+ return -EINVAL;
+ }
+
+ if (_dif_is_disabled(ctx->dif_type)) {
+ return 0;
+ }
+
+ if (_dif_sgl_is_bytes_multiple(&data_sgl, ctx->block_size)) {
+ return dix_verify(&data_sgl, &md_sgl, num_blocks, ctx, err_blk);
+ } else {
+ return dix_verify_split(&data_sgl, &md_sgl, num_blocks, ctx, err_blk);
+ }
+}
+
+int
+spdk_dix_inject_error(struct iovec *iovs, int iovcnt, struct iovec *md_iov,
+ uint32_t num_blocks, const struct spdk_dif_ctx *ctx,
+ uint32_t inject_flags, uint32_t *inject_offset)
+{
+ struct _dif_sgl data_sgl, md_sgl;
+ int rc;
+
+ _dif_sgl_init(&data_sgl, iovs, iovcnt);
+ _dif_sgl_init(&md_sgl, md_iov, 1);
+
+ if (!_dif_sgl_is_valid(&data_sgl, ctx->block_size * num_blocks) ||
+ !_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) {
+ SPDK_ERRLOG("Size of iovec array is not valid.\n");
+ return -EINVAL;
+ }
+
+ if (inject_flags & SPDK_DIF_REFTAG_ERROR) {
+ rc = dif_inject_error(&md_sgl, ctx->md_size, num_blocks,
+ ctx->guard_interval + offsetof(struct spdk_dif, ref_tag),
+ _member_size(struct spdk_dif, ref_tag),
+ inject_offset);
+ if (rc != 0) {
+ SPDK_ERRLOG("Failed to inject error to Reference Tag.\n");
+ return rc;
+ }
+ }
+
+ if (inject_flags & SPDK_DIF_APPTAG_ERROR) {
+ rc = dif_inject_error(&md_sgl, ctx->md_size, num_blocks,
+ ctx->guard_interval + offsetof(struct spdk_dif, app_tag),
+ _member_size(struct spdk_dif, app_tag),
+ inject_offset);
+ if (rc != 0) {
+ SPDK_ERRLOG("Failed to inject error to Application Tag.\n");
+ return rc;
+ }
+ }
+
+ if (inject_flags & SPDK_DIF_GUARD_ERROR) {
+ rc = dif_inject_error(&md_sgl, ctx->md_size, num_blocks,
+ ctx->guard_interval,
+ _member_size(struct spdk_dif, guard),
+ inject_offset);
+ if (rc != 0) {
+ SPDK_ERRLOG("Failed to inject error to Guard.\n");
+ return rc;
+ }
+ }
+
+ if (inject_flags & SPDK_DIF_DATA_ERROR) {
+ /* Note: Error injection to data block is expected to be detected
+ * as guard error.
+ */
+ rc = dif_inject_error(&data_sgl, ctx->block_size, num_blocks,
+ 0,
+ ctx->block_size,
+ inject_offset);
+ if (rc != 0) {
+ SPDK_ERRLOG("Failed to inject error to Guard.\n");
+ return rc;
+ }
+ }
+
+ return 0;
+}
+
+static uint32_t
+_to_next_boundary(uint32_t offset, uint32_t boundary)
+{
+ return boundary - (offset % boundary);
+}
+
+static uint32_t
+_to_size_with_md(uint32_t size, uint32_t data_block_size, uint32_t block_size)
+{
+ return (size / data_block_size) * block_size + (size % data_block_size);
+}
+
+int
+spdk_dif_set_md_interleave_iovs(struct iovec *iovs, int iovcnt,
+ struct iovec *buf_iovs, int buf_iovcnt,
+ uint32_t data_offset, uint32_t data_len,
+ uint32_t *_mapped_len,
+ const struct spdk_dif_ctx *ctx)
+{
+ uint32_t data_block_size, data_unalign, buf_len, buf_offset, len;
+ struct _dif_sgl dif_sgl;
+ struct _dif_sgl buf_sgl;
+
+ if (iovs == NULL || iovcnt == 0 || buf_iovs == NULL || buf_iovcnt == 0) {
+ return -EINVAL;
+ }
+
+ data_block_size = ctx->block_size - ctx->md_size;
+
+ data_unalign = ctx->data_offset % data_block_size;
+
+ buf_len = _to_size_with_md(data_unalign + data_offset + data_len, data_block_size,
+ ctx->block_size);
+ buf_len -= data_unalign;
+
+ _dif_sgl_init(&dif_sgl, iovs, iovcnt);
+ _dif_sgl_init(&buf_sgl, buf_iovs, buf_iovcnt);
+
+ if (!_dif_sgl_is_valid(&buf_sgl, buf_len)) {
+ SPDK_ERRLOG("Buffer overflow will occur.\n");
+ return -ERANGE;
+ }
+
+ buf_offset = _to_size_with_md(data_unalign + data_offset, data_block_size, ctx->block_size);
+ buf_offset -= data_unalign;
+
+ _dif_sgl_advance(&buf_sgl, buf_offset);
+
+ while (data_len != 0) {
+ len = spdk_min(data_len, _to_next_boundary(ctx->data_offset + data_offset, data_block_size));
+ if (!_dif_sgl_append_split(&dif_sgl, &buf_sgl, len)) {
+ break;
+ }
+ _dif_sgl_advance(&buf_sgl, ctx->md_size);
+ data_offset += len;
+ data_len -= len;
+ }
+
+ if (_mapped_len != NULL) {
+ *_mapped_len = dif_sgl.total_size;
+ }
+
+ return iovcnt - dif_sgl.iovcnt;
+}
+
+static int
+_dif_sgl_setup_stream(struct _dif_sgl *sgl, uint32_t *_buf_offset, uint32_t *_buf_len,
+ uint32_t data_offset, uint32_t data_len,
+ const struct spdk_dif_ctx *ctx)
+{
+ uint32_t data_block_size, data_unalign, buf_len, buf_offset;
+
+ data_block_size = ctx->block_size - ctx->md_size;
+
+ data_unalign = ctx->data_offset % data_block_size;
+
+ /* If the last data block is complete, DIF of the data block is
+ * inserted or verified in this turn.
+ */
+ buf_len = _to_size_with_md(data_unalign + data_offset + data_len, data_block_size,
+ ctx->block_size);
+ buf_len -= data_unalign;
+
+ if (!_dif_sgl_is_valid(sgl, buf_len)) {
+ return -ERANGE;
+ }
+
+ buf_offset = _to_size_with_md(data_unalign + data_offset, data_block_size, ctx->block_size);
+ buf_offset -= data_unalign;
+
+ _dif_sgl_advance(sgl, buf_offset);
+ buf_len -= buf_offset;
+
+ buf_offset += data_unalign;
+
+ *_buf_offset = buf_offset;
+ *_buf_len = buf_len;
+
+ return 0;
+}
+
+int
+spdk_dif_generate_stream(struct iovec *iovs, int iovcnt,
+ uint32_t data_offset, uint32_t data_len,
+ struct spdk_dif_ctx *ctx)
+{
+ uint32_t buf_len = 0, buf_offset = 0;
+ uint32_t len, offset_in_block, offset_blocks;
+ uint16_t guard = 0;
+ struct _dif_sgl sgl;
+ int rc;
+
+ if (iovs == NULL || iovcnt == 0) {
+ return -EINVAL;
+ }
+
+ if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) {
+ guard = ctx->last_guard;
+ }
+
+ _dif_sgl_init(&sgl, iovs, iovcnt);
+
+ rc = _dif_sgl_setup_stream(&sgl, &buf_offset, &buf_len, data_offset, data_len, ctx);
+ if (rc != 0) {
+ return rc;
+ }
+
+ while (buf_len != 0) {
+ len = spdk_min(buf_len, _to_next_boundary(buf_offset, ctx->block_size));
+ offset_in_block = buf_offset % ctx->block_size;
+ offset_blocks = buf_offset / ctx->block_size;
+
+ guard = _dif_generate_split(&sgl, offset_in_block, len, guard, offset_blocks, ctx);
+
+ buf_len -= len;
+ buf_offset += len;
+ }
+
+ if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) {
+ ctx->last_guard = guard;
+ }
+
+ return 0;
+}
+
+int
+spdk_dif_verify_stream(struct iovec *iovs, int iovcnt,
+ uint32_t data_offset, uint32_t data_len,
+ struct spdk_dif_ctx *ctx,
+ struct spdk_dif_error *err_blk)
+{
+ uint32_t buf_len = 0, buf_offset = 0;
+ uint32_t len, offset_in_block, offset_blocks;
+ uint16_t guard = 0;
+ struct _dif_sgl sgl;
+ int rc = 0;
+
+ if (iovs == NULL || iovcnt == 0) {
+ return -EINVAL;
+ }
+
+ if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) {
+ guard = ctx->last_guard;
+ }
+
+ _dif_sgl_init(&sgl, iovs, iovcnt);
+
+ rc = _dif_sgl_setup_stream(&sgl, &buf_offset, &buf_len, data_offset, data_len, ctx);
+ if (rc != 0) {
+ return rc;
+ }
+
+ while (buf_len != 0) {
+ len = spdk_min(buf_len, _to_next_boundary(buf_offset, ctx->block_size));
+ offset_in_block = buf_offset % ctx->block_size;
+ offset_blocks = buf_offset / ctx->block_size;
+
+ rc = _dif_verify_split(&sgl, offset_in_block, len, &guard, offset_blocks,
+ ctx, err_blk);
+ if (rc != 0) {
+ goto error;
+ }
+
+ buf_len -= len;
+ buf_offset += len;
+ }
+
+ if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) {
+ ctx->last_guard = guard;
+ }
+error:
+ return rc;
+}
+
+int
+spdk_dif_update_crc32c_stream(struct iovec *iovs, int iovcnt,
+ uint32_t data_offset, uint32_t data_len,
+ uint32_t *_crc32c, const struct spdk_dif_ctx *ctx)
+{
+ uint32_t buf_len = 0, buf_offset = 0, len, offset_in_block;
+ uint32_t crc32c;
+ struct _dif_sgl sgl;
+ int rc;
+
+ if (iovs == NULL || iovcnt == 0) {
+ return -EINVAL;
+ }
+
+ crc32c = *_crc32c;
+ _dif_sgl_init(&sgl, iovs, iovcnt);
+
+ rc = _dif_sgl_setup_stream(&sgl, &buf_offset, &buf_len, data_offset, data_len, ctx);
+ if (rc != 0) {
+ return rc;
+ }
+
+ while (buf_len != 0) {
+ len = spdk_min(buf_len, _to_next_boundary(buf_offset, ctx->block_size));
+ offset_in_block = buf_offset % ctx->block_size;
+
+ crc32c = _dif_update_crc32c_split(&sgl, offset_in_block, len, crc32c, ctx);
+
+ buf_len -= len;
+ buf_offset += len;
+ }
+
+ *_crc32c = crc32c;
+
+ return 0;
+}
+
+void
+spdk_dif_get_range_with_md(uint32_t data_offset, uint32_t data_len,
+ uint32_t *_buf_offset, uint32_t *_buf_len,
+ const struct spdk_dif_ctx *ctx)
+{
+ uint32_t data_block_size, data_unalign, buf_offset, buf_len;
+
+ if (!ctx->md_interleave) {
+ buf_offset = data_offset;
+ buf_len = data_len;
+ } else {
+ data_block_size = ctx->block_size - ctx->md_size;
+
+ data_unalign = data_offset % data_block_size;
+
+ buf_offset = _to_size_with_md(data_offset, data_block_size, ctx->block_size);
+ buf_len = _to_size_with_md(data_unalign + data_len, data_block_size, ctx->block_size) -
+ data_unalign;
+ }
+
+ if (_buf_offset != NULL) {
+ *_buf_offset = buf_offset;
+ }
+
+ if (_buf_len != NULL) {
+ *_buf_len = buf_len;
+ }
+}
+
+uint32_t
+spdk_dif_get_length_with_md(uint32_t data_len, const struct spdk_dif_ctx *ctx)
+{
+ uint32_t data_block_size;
+
+ if (!ctx->md_interleave) {
+ return data_len;
+ } else {
+ data_block_size = ctx->block_size - ctx->md_size;
+
+ return _to_size_with_md(data_len, data_block_size, ctx->block_size);
+ }
+}
+
+static int
+_dif_remap_ref_tag(struct _dif_sgl *sgl, uint32_t offset_blocks,
+ const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk)
+{
+ uint32_t offset, buf_len, expected = 0, _actual, remapped;
+ void *buf;
+ struct _dif_sgl tmp_sgl;
+ struct spdk_dif dif;
+
+ /* Fast forward to DIF field. */
+ _dif_sgl_advance(sgl, ctx->guard_interval);
+ _dif_sgl_copy(&tmp_sgl, sgl);
+
+ /* Copy the split DIF field to the temporary DIF buffer */
+ offset = 0;
+ while (offset < sizeof(struct spdk_dif)) {
+ _dif_sgl_get_buf(sgl, &buf, &buf_len);
+ buf_len = spdk_min(buf_len, sizeof(struct spdk_dif) - offset);
+
+ memcpy((uint8_t *)&dif + offset, buf, buf_len);
+
+ _dif_sgl_advance(sgl, buf_len);
+ offset += buf_len;
+ }
+
+ switch (ctx->dif_type) {
+ case SPDK_DIF_TYPE1:
+ case SPDK_DIF_TYPE2:
+ /* If Type 1 or 2 is used, then all DIF checks are disabled when
+ * the Application Tag is 0xFFFF.
+ */
+ if (dif.app_tag == 0xFFFF) {
+ goto end;
+ }
+ break;
+ case SPDK_DIF_TYPE3:
+ /* If Type 3 is used, then all DIF checks are disabled when the
+ * Application Tag is 0xFFFF and the Reference Tag is 0xFFFFFFFF.
+ */
+ if (dif.app_tag == 0xFFFF && dif.ref_tag == 0xFFFFFFFF) {
+ goto end;
+ }
+ break;
+ default:
+ break;
+ }
+
+ /* For type 1 and 2, the Reference Tag is incremented for each
+ * subsequent logical block. For type 3, the Reference Tag
+ * remains the same as the initial Reference Tag.
+ */
+ if (ctx->dif_type != SPDK_DIF_TYPE3) {
+ expected = ctx->init_ref_tag + ctx->ref_tag_offset + offset_blocks;
+ remapped = ctx->remapped_init_ref_tag + ctx->ref_tag_offset + offset_blocks;
+ } else {
+ remapped = ctx->remapped_init_ref_tag;
+ }
+
+ /* Verify the stored Reference Tag. */
+ switch (ctx->dif_type) {
+ case SPDK_DIF_TYPE1:
+ case SPDK_DIF_TYPE2:
+ /* Compare the DIF Reference Tag field to the computed Reference Tag.
+ * The computed Reference Tag will be the least significant 4 bytes
+ * of the LBA when Type 1 is used, and application specific value
+ * if Type 2 is used.
+ */
+ _actual = from_be32(&dif.ref_tag);
+ if (_actual != expected) {
+ _dif_error_set(err_blk, SPDK_DIF_REFTAG_ERROR, expected,
+ _actual, offset_blocks);
+ SPDK_ERRLOG("Failed to compare Ref Tag: LBA=%" PRIu32 "," \
+ " Expected=%x, Actual=%x\n",
+ expected, expected, _actual);
+ return -1;
+ }
+ break;
+ case SPDK_DIF_TYPE3:
+ /* For type 3, the computed Reference Tag remains unchanged.
+ * Hence ignore the Reference Tag field.
+ */
+ break;
+ default:
+ break;
+ }
+
+ /* Update the stored Reference Tag to the remapped one. */
+ to_be32(&dif.ref_tag, remapped);
+
+ offset = 0;
+ while (offset < sizeof(struct spdk_dif)) {
+ _dif_sgl_get_buf(&tmp_sgl, &buf, &buf_len);
+ buf_len = spdk_min(buf_len, sizeof(struct spdk_dif) - offset);
+
+ memcpy(buf, (uint8_t *)&dif + offset, buf_len);
+
+ _dif_sgl_advance(&tmp_sgl, buf_len);
+ offset += buf_len;
+ }
+
+end:
+ _dif_sgl_advance(sgl, ctx->block_size - ctx->guard_interval - sizeof(struct spdk_dif));
+
+ return 0;
+}
+
+int
+spdk_dif_remap_ref_tag(struct iovec *iovs, int iovcnt, uint32_t num_blocks,
+ const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk)
+{
+ struct _dif_sgl sgl;
+ uint32_t offset_blocks;
+ int rc;
+
+ _dif_sgl_init(&sgl, iovs, iovcnt);
+
+ if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) {
+ SPDK_ERRLOG("Size of iovec array is not valid.\n");
+ return -EINVAL;
+ }
+
+ if (_dif_is_disabled(ctx->dif_type)) {
+ return 0;
+ }
+
+ if (!(ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK)) {
+ return 0;
+ }
+
+ for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) {
+ rc = _dif_remap_ref_tag(&sgl, offset_blocks, ctx, err_blk);
+ if (rc != 0) {
+ return rc;
+ }
+ }
+
+ return 0;
+}
+
+static int
+_dix_remap_ref_tag(struct _dif_sgl *md_sgl, uint32_t offset_blocks,
+ const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk)
+{
+ uint32_t expected = 0, _actual, remapped;
+ uint8_t *md_buf;
+ struct spdk_dif *dif;
+
+ _dif_sgl_get_buf(md_sgl, (void *)&md_buf, NULL);
+
+ dif = (struct spdk_dif *)(md_buf + ctx->guard_interval);
+
+ switch (ctx->dif_type) {
+ case SPDK_DIF_TYPE1:
+ case SPDK_DIF_TYPE2:
+ /* If Type 1 or 2 is used, then all DIF checks are disabled when
+ * the Application Tag is 0xFFFF.
+ */
+ if (dif->app_tag == 0xFFFF) {
+ goto end;
+ }
+ break;
+ case SPDK_DIF_TYPE3:
+ /* If Type 3 is used, then all DIF checks are disabled when the
+ * Application Tag is 0xFFFF and the Reference Tag is 0xFFFFFFFF.
+ */
+ if (dif->app_tag == 0xFFFF && dif->ref_tag == 0xFFFFFFFF) {
+ goto end;
+ }
+ break;
+ default:
+ break;
+ }
+
+ /* For type 1 and 2, the Reference Tag is incremented for each
+ * subsequent logical block. For type 3, the Reference Tag
+ * remains the same as the initialReference Tag.
+ */
+ if (ctx->dif_type != SPDK_DIF_TYPE3) {
+ expected = ctx->init_ref_tag + ctx->ref_tag_offset + offset_blocks;
+ remapped = ctx->remapped_init_ref_tag + ctx->ref_tag_offset + offset_blocks;
+ } else {
+ remapped = ctx->remapped_init_ref_tag;
+ }
+
+ /* Verify the stored Reference Tag. */
+ switch (ctx->dif_type) {
+ case SPDK_DIF_TYPE1:
+ case SPDK_DIF_TYPE2:
+ /* Compare the DIF Reference Tag field to the computed Reference Tag.
+ * The computed Reference Tag will be the least significant 4 bytes
+ * of the LBA when Type 1 is used, and application specific value
+ * if Type 2 is used.
+ */
+ _actual = from_be32(&dif->ref_tag);
+ if (_actual != expected) {
+ _dif_error_set(err_blk, SPDK_DIF_REFTAG_ERROR, expected,
+ _actual, offset_blocks);
+ SPDK_ERRLOG("Failed to compare Ref Tag: LBA=%" PRIu32 "," \
+ " Expected=%x, Actual=%x\n",
+ expected, expected, _actual);
+ return -1;
+ }
+ break;
+ case SPDK_DIF_TYPE3:
+ /* For type 3, the computed Reference Tag remains unchanged.
+ * Hence ignore the Reference Tag field.
+ */
+ break;
+ default:
+ break;
+ }
+
+ /* Update the stored Reference Tag to the remapped one. */
+ to_be32(&dif->ref_tag, remapped);
+
+end:
+ _dif_sgl_advance(md_sgl, ctx->md_size);
+
+ return 0;
+}
+
+int
+spdk_dix_remap_ref_tag(struct iovec *md_iov, uint32_t num_blocks,
+ const struct spdk_dif_ctx *ctx,
+ struct spdk_dif_error *err_blk)
+{
+ struct _dif_sgl md_sgl;
+ uint32_t offset_blocks;
+ int rc;
+
+ _dif_sgl_init(&md_sgl, md_iov, 1);
+
+ if (!_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) {
+ SPDK_ERRLOG("Size of metadata iovec array is not valid.\n");
+ return -EINVAL;
+ }
+
+ if (_dif_is_disabled(ctx->dif_type)) {
+ return 0;
+ }
+
+ if (!(ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK)) {
+ return 0;
+ }
+
+ for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) {
+ rc = _dix_remap_ref_tag(&md_sgl, offset_blocks, ctx, err_blk);
+ if (rc != 0) {
+ return rc;
+ }
+ }
+
+ return 0;
+}
diff --git a/src/spdk/lib/util/fd.c b/src/spdk/lib/util/fd.c
new file mode 100644
index 000000000..6b0d0d554
--- /dev/null
+++ b/src/spdk/lib/util/fd.c
@@ -0,0 +1,103 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+
+#include "spdk/fd.h"
+
+#ifdef __linux__
+#include <linux/fs.h>
+#endif
+
+static uint64_t
+dev_get_size(int fd)
+{
+#if defined(DIOCGMEDIASIZE) /* FreeBSD */
+ off_t size;
+
+ if (ioctl(fd, DIOCGMEDIASIZE, &size) == 0) {
+ return size;
+ }
+#elif defined(__linux__) && defined(BLKGETSIZE64)
+ uint64_t size;
+
+ if (ioctl(fd, BLKGETSIZE64, &size) == 0) {
+ return size;
+ }
+#endif
+
+ return 0;
+}
+
+uint32_t
+spdk_fd_get_blocklen(int fd)
+{
+#if defined(DKIOCGETBLOCKSIZE) /* FreeBSD */
+ uint32_t blocklen;
+
+ if (ioctl(fd, DKIOCGETBLOCKSIZE, &blocklen) == 0) {
+ return blocklen;
+ }
+#elif defined(__linux__) && defined(BLKSSZGET)
+ uint32_t blocklen;
+
+ if (ioctl(fd, BLKSSZGET, &blocklen) == 0) {
+ return blocklen;
+ }
+#endif
+
+ return 0;
+}
+
+uint64_t
+spdk_fd_get_size(int fd)
+{
+ struct stat st;
+
+ if (fstat(fd, &st) != 0) {
+ return 0;
+ }
+
+ if (S_ISLNK(st.st_mode)) {
+ return 0;
+ }
+
+ if (S_ISBLK(st.st_mode) || S_ISCHR(st.st_mode)) {
+ return dev_get_size(fd);
+ } else if (S_ISREG(st.st_mode)) {
+ return st.st_size;
+ }
+
+ /* Not REG, CHR or BLK */
+ return 0;
+}
diff --git a/src/spdk/lib/util/file.c b/src/spdk/lib/util/file.c
new file mode 100644
index 000000000..2ba08547b
--- /dev/null
+++ b/src/spdk/lib/util/file.c
@@ -0,0 +1,71 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/file.h"
+
+void *
+spdk_posix_file_load(FILE *file, size_t *size)
+{
+ void *newbuf, *buf = NULL;
+ size_t rc, buf_size, cur_size = 0;
+
+ *size = 0;
+ buf_size = 128 * 1024;
+
+ while (buf_size <= 1024 * 1024 * 1024) {
+ newbuf = realloc(buf, buf_size);
+ if (newbuf == NULL) {
+ free(buf);
+ return NULL;
+ }
+ buf = newbuf;
+
+ rc = fread(buf + cur_size, 1, buf_size - cur_size, file);
+ cur_size += rc;
+
+ if (feof(file)) {
+ *size = cur_size;
+ return buf;
+ }
+
+ if (ferror(file)) {
+ free(buf);
+ return NULL;
+ }
+
+ buf_size *= 2;
+ }
+
+ free(buf);
+ return NULL;
+}
diff --git a/src/spdk/lib/util/iov.c b/src/spdk/lib/util/iov.c
new file mode 100644
index 000000000..e89ef9d21
--- /dev/null
+++ b/src/spdk/lib/util/iov.c
@@ -0,0 +1,111 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/util.h"
+
+size_t
+spdk_iovcpy(struct iovec *siov, size_t siovcnt, struct iovec *diov, size_t diovcnt)
+{
+ size_t total_sz;
+ size_t sidx;
+ size_t didx;
+ int siov_len;
+ uint8_t *siov_base;
+ int diov_len;
+ uint8_t *diov_base;
+
+ /* d prefix = destination. s prefix = source. */
+
+ assert(diovcnt > 0);
+ assert(siovcnt > 0);
+
+ total_sz = 0;
+ sidx = 0;
+ didx = 0;
+ siov_len = siov[0].iov_len;
+ siov_base = siov[0].iov_base;
+ diov_len = diov[0].iov_len;
+ diov_base = diov[0].iov_base;
+ while (siov_len > 0 && diov_len > 0) {
+ if (siov_len == diov_len) {
+ memcpy(diov_base, siov_base, siov_len);
+ total_sz += siov_len;
+
+ /* Advance both iovs to the next element */
+ sidx++;
+ if (sidx == siovcnt) {
+ break;
+ }
+
+ didx++;
+ if (didx == diovcnt) {
+ break;
+ }
+
+ siov_len = siov[sidx].iov_len;
+ siov_base = siov[sidx].iov_base;
+ diov_len = diov[didx].iov_len;
+ diov_base = diov[didx].iov_base;
+ } else if (siov_len < diov_len) {
+ memcpy(diov_base, siov_base, siov_len);
+ total_sz += siov_len;
+
+ /* Advance only the source to the next element */
+ sidx++;
+ if (sidx == siovcnt) {
+ break;
+ }
+
+ diov_base += siov_len;
+ diov_len -= siov_len;
+ siov_len = siov[sidx].iov_len;
+ siov_base = siov[sidx].iov_base;
+ } else {
+ memcpy(diov_base, siov_base, diov_len);
+ total_sz += diov_len;
+
+ /* Advance only the destination to the next element */
+ didx++;
+ if (didx == diovcnt) {
+ break;
+ }
+
+ siov_base += diov_len;
+ siov_len -= diov_len;
+ diov_len = diov[didx].iov_len;
+ diov_base = diov[didx].iov_base;
+ }
+ }
+
+ return total_sz;
+}
diff --git a/src/spdk/lib/util/math.c b/src/spdk/lib/util/math.c
new file mode 100644
index 000000000..7d1852421
--- /dev/null
+++ b/src/spdk/lib/util/math.c
@@ -0,0 +1,69 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+#include "spdk/util.h"
+
+/* The following will automatically generate several version of
+ * this function, targeted at different architectures. This
+ * is only supported by GCC 6 or newer. */
+#if defined(__GNUC__) && __GNUC__ >= 6 && !defined(__clang__) \
+ && (defined(__i386__) || defined(__x86_64__))
+__attribute__((target_clones("bmi", "arch=core2", "arch=atom", "default")))
+#endif
+uint32_t
+spdk_u32log2(uint32_t x)
+{
+ if (x == 0) {
+ /* log(0) is undefined */
+ return 0;
+ }
+ return 31u - __builtin_clz(x);
+}
+
+/* The following will automatically generate several version of
+ * this function, targeted at different architectures. This
+ * is only supported by GCC 6 or newer. */
+#if defined(__GNUC__) && __GNUC__ >= 6 && !defined(__clang__) \
+ && (defined(__i386__) || defined(__x86_64__))
+__attribute__((target_clones("bmi", "arch=core2", "arch=atom", "default")))
+#endif
+uint64_t
+spdk_u64log2(uint64_t x)
+{
+ if (x == 0) {
+ /* log(0) is undefined */
+ return 0;
+ }
+ return 63u - __builtin_clzl(x);
+}
diff --git a/src/spdk/lib/util/pipe.c b/src/spdk/lib/util/pipe.c
new file mode 100644
index 000000000..1c640dd2e
--- /dev/null
+++ b/src/spdk/lib/util/pipe.c
@@ -0,0 +1,246 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/pipe.h"
+#include "spdk/util.h"
+
+struct spdk_pipe {
+ uint8_t *buf;
+ uint32_t sz;
+
+ uint32_t write;
+ uint32_t read;
+};
+
+struct spdk_pipe *
+spdk_pipe_create(void *buf, uint32_t sz)
+{
+ struct spdk_pipe *pipe;
+
+ pipe = calloc(1, sizeof(*pipe));
+ if (pipe == NULL) {
+ return NULL;
+ }
+
+ pipe->buf = buf;
+ pipe->sz = sz;
+
+ return pipe;
+}
+
+void
+spdk_pipe_destroy(struct spdk_pipe *pipe)
+{
+ free(pipe);
+}
+
+int
+spdk_pipe_writer_get_buffer(struct spdk_pipe *pipe, uint32_t requested_sz, struct iovec *iovs)
+{
+ uint32_t sz;
+ uint32_t read;
+ uint32_t write;
+
+ read = pipe->read;
+ write = pipe->write;
+
+ if (read <= write) {
+ requested_sz = spdk_min(requested_sz, ((read + pipe->sz) - write - 1));
+
+ sz = spdk_min(requested_sz, pipe->sz - write);
+
+ iovs[0].iov_base = (sz == 0) ? NULL : (pipe->buf + write);
+ iovs[0].iov_len = sz;
+
+ requested_sz -= sz;
+
+ if (requested_sz > 0) {
+ sz = spdk_min(requested_sz, read);
+
+ iovs[1].iov_base = (sz == 0) ? NULL : pipe->buf;
+ iovs[1].iov_len = sz;
+ } else {
+ iovs[1].iov_base = NULL;
+ iovs[1].iov_len = 0;
+ }
+ } else {
+ sz = spdk_min(requested_sz, read - write - 1);
+
+ iovs[0].iov_base = (sz == 0) ? NULL : (pipe->buf + write);
+ iovs[0].iov_len = sz;
+ iovs[1].iov_base = NULL;
+ iovs[1].iov_len = 0;
+ }
+
+ return iovs[0].iov_len + iovs[1].iov_len;
+}
+
+int
+spdk_pipe_writer_advance(struct spdk_pipe *pipe, uint32_t requested_sz)
+{
+ uint32_t sz;
+ uint32_t read;
+ uint32_t write;
+
+ read = pipe->read;
+ write = pipe->write;
+
+ if (requested_sz > pipe->sz - 1) {
+ return -EINVAL;
+ }
+
+ if (read <= write) {
+ if (requested_sz > (read + pipe->sz) - write) {
+ return -EINVAL;
+ }
+
+ sz = spdk_min(requested_sz, pipe->sz - write);
+
+ write += sz;
+ if (write > pipe->sz - 1) {
+ write = 0;
+ }
+ requested_sz -= sz;
+
+ if (requested_sz > 0) {
+ if (requested_sz >= read) {
+ return -EINVAL;
+ }
+
+ write = requested_sz;
+ }
+ } else {
+ if (requested_sz > (read - write - 1)) {
+ return -EINVAL;
+ }
+
+ write += requested_sz;
+ }
+
+ pipe->write = write;
+
+ return 0;
+}
+
+uint32_t
+spdk_pipe_reader_bytes_available(struct spdk_pipe *pipe)
+{
+ uint32_t read;
+ uint32_t write;
+
+ read = pipe->read;
+ write = pipe->write;
+
+ if (read <= write) {
+ return write - read;
+ }
+
+ return (write + pipe->sz) - read;
+}
+
+int
+spdk_pipe_reader_get_buffer(struct spdk_pipe *pipe, uint32_t requested_sz, struct iovec *iovs)
+{
+ uint32_t sz;
+ uint32_t read;
+ uint32_t write;
+
+ read = pipe->read;
+ write = pipe->write;
+
+ if (read <= write) {
+ sz = spdk_min(requested_sz, write - read);
+
+ iovs[0].iov_base = (sz == 0) ? NULL : (pipe->buf + read);
+ iovs[0].iov_len = sz;
+ iovs[1].iov_base = NULL;
+ iovs[1].iov_len = 0;
+ } else {
+ sz = spdk_min(requested_sz, pipe->sz - read);
+
+ iovs[0].iov_base = (sz == 0) ? NULL : (pipe->buf + read);
+ iovs[0].iov_len = sz;
+
+ requested_sz -= sz;
+
+ if (requested_sz > 0) {
+ sz = spdk_min(requested_sz, write);
+ iovs[1].iov_base = (sz == 0) ? NULL : pipe->buf;
+ iovs[1].iov_len = sz;
+ } else {
+ iovs[1].iov_base = NULL;
+ iovs[1].iov_len = 0;
+ }
+ }
+
+ return iovs[0].iov_len + iovs[1].iov_len;
+}
+
+int
+spdk_pipe_reader_advance(struct spdk_pipe *pipe, uint32_t requested_sz)
+{
+ uint32_t sz;
+ uint32_t read;
+ uint32_t write;
+
+ read = pipe->read;
+ write = pipe->write;
+
+ if (read <= write) {
+ if (requested_sz > (write - read)) {
+ return -EINVAL;
+ }
+
+ read += requested_sz;
+ } else {
+ sz = spdk_min(requested_sz, pipe->sz - read);
+
+ read += sz;
+ if (read > pipe->sz - 1) {
+ read = 0;
+ }
+ requested_sz -= sz;
+
+ if (requested_sz > 0) {
+ if (requested_sz > write) {
+ return -EINVAL;
+ }
+
+ read = requested_sz;
+ }
+ }
+
+ pipe->read = read;
+
+ return 0;
+}
diff --git a/src/spdk/lib/util/spdk_util.map b/src/spdk/lib/util/spdk_util.map
new file mode 100644
index 000000000..07e067faa
--- /dev/null
+++ b/src/spdk/lib/util/spdk_util.map
@@ -0,0 +1,128 @@
+{
+ global:
+
+ # public functions in base64.h
+ spdk_base64_encode;
+ spdk_base64_urlsafe_encode;
+ spdk_base64_decode;
+ spdk_base64_urlsafe_decode;
+
+ # public functions in bit_array.h
+ spdk_bit_array_capacity;
+ spdk_bit_array_create;
+ spdk_bit_array_free;
+ spdk_bit_array_resize;
+ spdk_bit_array_get;
+ spdk_bit_array_set;
+ spdk_bit_array_clear;
+ spdk_bit_array_find_first_set;
+ spdk_bit_array_find_first_clear;
+ spdk_bit_array_count_set;
+ spdk_bit_array_count_clear;
+ spdk_bit_array_store_mask;
+ spdk_bit_array_load_mask;
+ spdk_bit_array_clear_mask;
+
+ # public functions in cpuset.h
+ spdk_cpuset_alloc;
+ spdk_cpuset_free;
+ spdk_cpuset_equal;
+ spdk_cpuset_copy;
+ spdk_cpuset_and;
+ spdk_cpuset_or;
+ spdk_cpuset_xor;
+ spdk_cpuset_negate;
+ spdk_cpuset_zero;
+ spdk_cpuset_set_cpu;
+ spdk_cpuset_get_cpu;
+ spdk_cpuset_count;
+ spdk_cpuset_fmt;
+ spdk_cpuset_parse;
+
+ # public functions in crc16.h
+ spdk_crc16_t10dif;
+ spdk_crc16_t10dif_copy;
+
+ # public functions in crc32.h
+ spdk_crc32_ieee_update;
+ spdk_crc32c_update;
+
+ # public functions in dif.h
+ spdk_dif_ctx_init;
+ spdk_dif_ctx_set_data_offset;
+ spdk_dif_ctx_set_remapped_init_ref_tag;
+ spdk_dif_generate;
+ spdk_dif_verify;
+ spdk_dif_update_crc32c;
+ spdk_dif_generate_copy;
+ spdk_dif_verify_copy;
+ spdk_dif_inject_error;
+ spdk_dix_generate;
+ spdk_dix_verify;
+ spdk_dix_inject_error;
+ spdk_dif_set_md_interleave_iovs;
+ spdk_dif_generate_stream;
+ spdk_dif_verify_stream;
+ spdk_dif_update_crc32c_stream;
+ spdk_dif_get_range_with_md;
+ spdk_dif_get_length_with_md;
+ spdk_dif_remap_ref_tag;
+ spdk_dix_remap_ref_tag;
+
+ # public functions in fd.h
+ spdk_fd_get_size;
+ spdk_fd_get_blocklen;
+
+ # public functions in file.h
+ spdk_posix_file_load;
+
+ # public functions in pipe.h
+ spdk_pipe_create;
+ spdk_pipe_destroy;
+ spdk_pipe_writer_get_buffer;
+ spdk_pipe_writer_advance;
+ spdk_pipe_reader_bytes_available;
+ spdk_pipe_reader_get_buffer;
+ spdk_pipe_reader_advance;
+
+ # public functions in string.h
+ spdk_sprintf_alloc;
+ spdk_vsprintf_alloc;
+ spdk_sprintf_append_realloc;
+ spdk_vsprintf_append_realloc;
+ spdk_strlwr;
+ spdk_strsepq;
+ spdk_str_trim;
+ spdk_strerror_r;
+ spdk_strerror;
+ spdk_str_chomp;
+ spdk_strcpy_pad;
+ spdk_strlen_pad;
+ spdk_parse_ip_addr;
+ spdk_parse_capacity;
+ spdk_mem_all_zero;
+ spdk_strtol;
+ spdk_strtoll;
+
+ # public functions in util.h
+ spdk_u32log2;
+ spdk_u64log2;
+ spdk_iovcpy;
+
+ # resolvers for functions in util.h
+ spdk_u32log2.resolver;
+ spdk_u64log2.resolver;
+
+ # public functions in uuid.h
+ spdk_uuid_parse;
+ spdk_uuid_fmt_lower;
+ spdk_uuid_compare;
+ spdk_uuid_generate;
+ spdk_uuid_copy;
+
+
+
+
+
+ local: *;
+};
diff --git a/src/spdk/lib/util/strerror_tls.c b/src/spdk/lib/util/strerror_tls.c
new file mode 100644
index 000000000..c9dc8f13f
--- /dev/null
+++ b/src/spdk/lib/util/strerror_tls.c
@@ -0,0 +1,43 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/string.h"
+
+static __thread char strerror_message[64];
+
+const char *
+spdk_strerror(int errnum)
+{
+ spdk_strerror_r(errnum, strerror_message, sizeof(strerror_message));
+ return strerror_message;
+}
diff --git a/src/spdk/lib/util/string.c b/src/spdk/lib/util/string.c
new file mode 100644
index 000000000..30ac1628a
--- /dev/null
+++ b/src/spdk/lib/util/string.c
@@ -0,0 +1,476 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+
+#include "spdk/string.h"
+
+char *
+spdk_vsprintf_append_realloc(char *buffer, const char *format, va_list args)
+{
+ va_list args_copy;
+ char *new_buffer;
+ int orig_size = 0, new_size;
+
+ /* Original buffer size */
+ if (buffer) {
+ orig_size = strlen(buffer);
+ }
+
+ /* Necessary buffer size */
+ va_copy(args_copy, args);
+ new_size = vsnprintf(NULL, 0, format, args_copy);
+ va_end(args_copy);
+
+ if (new_size < 0) {
+ return NULL;
+ }
+ new_size += orig_size + 1;
+
+ new_buffer = realloc(buffer, new_size);
+ if (new_buffer == NULL) {
+ return NULL;
+ }
+
+ vsnprintf(new_buffer + orig_size, new_size - orig_size, format, args);
+
+ return new_buffer;
+}
+
+char *
+spdk_sprintf_append_realloc(char *buffer, const char *format, ...)
+{
+ va_list args;
+ char *ret;
+
+ va_start(args, format);
+ ret = spdk_vsprintf_append_realloc(buffer, format, args);
+ va_end(args);
+
+ return ret;
+}
+
+char *
+spdk_vsprintf_alloc(const char *format, va_list args)
+{
+ return spdk_vsprintf_append_realloc(NULL, format, args);
+}
+
+char *
+spdk_sprintf_alloc(const char *format, ...)
+{
+ va_list args;
+ char *ret;
+
+ va_start(args, format);
+ ret = spdk_vsprintf_alloc(format, args);
+ va_end(args);
+
+ return ret;
+}
+
+char *
+spdk_strlwr(char *s)
+{
+ char *p;
+
+ if (s == NULL) {
+ return NULL;
+ }
+
+ p = s;
+ while (*p != '\0') {
+ *p = tolower(*p);
+ p++;
+ }
+
+ return s;
+}
+
+char *
+spdk_strsepq(char **stringp, const char *delim)
+{
+ char *p, *q, *r;
+ int quoted = 0, bslash = 0;
+
+ p = *stringp;
+ if (p == NULL) {
+ return NULL;
+ }
+
+ r = q = p;
+ while (*q != '\0' && *q != '\n') {
+ /* eat quoted characters */
+ if (bslash) {
+ bslash = 0;
+ *r++ = *q++;
+ continue;
+ } else if (quoted) {
+ if (quoted == '"' && *q == '\\') {
+ bslash = 1;
+ q++;
+ continue;
+ } else if (*q == quoted) {
+ quoted = 0;
+ q++;
+ continue;
+ }
+ *r++ = *q++;
+ continue;
+ } else if (*q == '\\') {
+ bslash = 1;
+ q++;
+ continue;
+ } else if (*q == '"' || *q == '\'') {
+ quoted = *q;
+ q++;
+ continue;
+ }
+
+ /* separator? */
+ if (strchr(delim, *q) == NULL) {
+ *r++ = *q++;
+ continue;
+ }
+
+ /* new string */
+ q++;
+ break;
+ }
+ *r = '\0';
+
+ /* skip tailer */
+ while (*q != '\0' && strchr(delim, *q) != NULL) {
+ q++;
+ }
+ if (*q != '\0') {
+ *stringp = q;
+ } else {
+ *stringp = NULL;
+ }
+
+ return p;
+}
+
+char *
+spdk_str_trim(char *s)
+{
+ char *p, *q;
+
+ if (s == NULL) {
+ return NULL;
+ }
+
+ /* remove header */
+ p = s;
+ while (*p != '\0' && isspace(*p)) {
+ p++;
+ }
+
+ /* remove tailer */
+ q = p + strlen(p);
+ while (q - 1 >= p && isspace(*(q - 1))) {
+ q--;
+ *q = '\0';
+ }
+
+ /* if remove header, move */
+ if (p != s) {
+ q = s;
+ while (*p != '\0') {
+ *q++ = *p++;
+ }
+ *q = '\0';
+ }
+
+ return s;
+}
+
+void
+spdk_strcpy_pad(void *dst, const char *src, size_t size, int pad)
+{
+ size_t len;
+
+ len = strlen(src);
+ if (len < size) {
+ memcpy(dst, src, len);
+ memset((char *)dst + len, pad, size - len);
+ } else {
+ memcpy(dst, src, size);
+ }
+}
+
+size_t
+spdk_strlen_pad(const void *str, size_t size, int pad)
+{
+ const uint8_t *start;
+ const uint8_t *iter;
+ uint8_t pad_byte;
+
+ pad_byte = (uint8_t)pad;
+ start = (const uint8_t *)str;
+
+ if (size == 0) {
+ return 0;
+ }
+
+ iter = start + size - 1;
+ while (1) {
+ if (*iter != pad_byte) {
+ return iter - start + 1;
+ }
+
+ if (iter == start) {
+ /* Hit the start of the string finding only pad_byte. */
+ return 0;
+ }
+ iter--;
+ }
+}
+
+int
+spdk_parse_ip_addr(char *ip, char **host, char **port)
+{
+ char *p;
+
+ if (ip == NULL) {
+ return -EINVAL;
+ }
+
+ *host = NULL;
+ *port = NULL;
+
+ if (ip[0] == '[') {
+ /* IPv6 */
+ p = strchr(ip, ']');
+ if (p == NULL) {
+ return -EINVAL;
+ }
+ *host = &ip[1];
+ *p = '\0';
+
+ p++;
+ if (*p == '\0') {
+ return 0;
+ } else if (*p != ':') {
+ return -EINVAL;
+ }
+
+ p++;
+ if (*p == '\0') {
+ return 0;
+ }
+
+ *port = p;
+ } else {
+ /* IPv4 */
+ p = strchr(ip, ':');
+ if (p == NULL) {
+ *host = ip;
+ return 0;
+ }
+
+ *host = ip;
+ *p = '\0';
+
+ p++;
+ if (*p == '\0') {
+ return 0;
+ }
+
+ *port = p;
+ }
+
+ return 0;
+}
+
+size_t
+spdk_str_chomp(char *s)
+{
+ size_t len = strlen(s);
+ size_t removed = 0;
+
+ while (len > 0) {
+ if (s[len - 1] != '\r' && s[len - 1] != '\n') {
+ break;
+ }
+
+ s[len - 1] = '\0';
+ len--;
+ removed++;
+ }
+
+ return removed;
+}
+
+void
+spdk_strerror_r(int errnum, char *buf, size_t buflen)
+{
+ int rc;
+
+#if defined(__USE_GNU)
+ char *new_buffer;
+ new_buffer = strerror_r(errnum, buf, buflen);
+ if (new_buffer == buf) {
+ rc = 0;
+ } else if (new_buffer != NULL) {
+ snprintf(buf, buflen, "%s", new_buffer);
+ rc = 0;
+ } else {
+ rc = 1;
+ }
+#else
+ rc = strerror_r(errnum, buf, buflen);
+#endif
+
+ if (rc != 0) {
+ snprintf(buf, buflen, "Unknown error %d", errnum);
+ }
+}
+
+int
+spdk_parse_capacity(const char *cap_str, uint64_t *cap, bool *has_prefix)
+{
+ int rc;
+ char bin_prefix;
+
+ rc = sscanf(cap_str, "%"SCNu64"%c", cap, &bin_prefix);
+ if (rc == 1) {
+ *has_prefix = false;
+ return 0;
+ } else if (rc == 0) {
+ if (errno == 0) {
+ /* No scanf matches - the string does not start with a digit */
+ return -EINVAL;
+ } else {
+ /* Parsing error */
+ return -errno;
+ }
+ }
+
+ *has_prefix = true;
+ switch (bin_prefix) {
+ case 'k':
+ case 'K':
+ *cap *= 1024;
+ break;
+ case 'm':
+ case 'M':
+ *cap *= 1024 * 1024;
+ break;
+ case 'g':
+ case 'G':
+ *cap *= 1024 * 1024 * 1024;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+bool
+spdk_mem_all_zero(const void *data, size_t size)
+{
+ const uint8_t *buf = data;
+
+ while (size--) {
+ if (*buf++ != 0) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+long int
+spdk_strtol(const char *nptr, int base)
+{
+ long val;
+ char *endptr;
+
+ /* Since strtoll() can legitimately return 0, LONG_MAX, or LONG_MIN
+ * on both success and failure, the calling program should set errno
+ * to 0 before the call.
+ */
+ errno = 0;
+
+ val = strtol(nptr, &endptr, base);
+
+ if (!errno && *endptr != '\0') {
+ /* Non integer character was found. */
+ return -EINVAL;
+ } else if (errno == ERANGE && (val == LONG_MAX || val == LONG_MIN)) {
+ /* Overflow occurred. */
+ return -ERANGE;
+ } else if (errno != 0 && val == 0) {
+ /* Other error occurred. */
+ return -errno;
+ } else if (val < 0) {
+ /* Input string was negative number. */
+ return -ERANGE;
+ }
+
+ return val;
+}
+
+long long int
+spdk_strtoll(const char *nptr, int base)
+{
+ long long val;
+ char *endptr;
+
+ /* Since strtoll() can legitimately return 0, LLONG_MAX, or LLONG_MIN
+ * on both success and failure, the calling program should set errno
+ * to 0 before the call.
+ */
+ errno = 0;
+
+ val = strtoll(nptr, &endptr, base);
+
+ if (!errno && *endptr != '\0') {
+ /* Non integer character was found. */
+ return -EINVAL;
+ } else if (errno == ERANGE && (val == LLONG_MAX || val == LLONG_MIN)) {
+ /* Overflow occurred. */
+ return -ERANGE;
+ } else if (errno != 0 && val == 0) {
+ /* Other error occurred. */
+ return -errno;
+ } else if (val < 0) {
+ /* Input string was negative number. */
+ return -ERANGE;
+ }
+
+ return val;
+}
diff --git a/src/spdk/lib/util/util_internal.h b/src/spdk/lib/util/util_internal.h
new file mode 100644
index 000000000..655ef513d
--- /dev/null
+++ b/src/spdk/lib/util/util_internal.h
@@ -0,0 +1,77 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (C) 2008-2012 Daisuke Aoyama <aoyama@peach.ne.jp>.
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef SPDK_UTIL_INTERNAL_H
+#define SPDK_UTIL_INTERNAL_H
+
+#include "spdk/stdinc.h"
+
+/**
+ * IEEE CRC-32 polynomial (bit reflected)
+ */
+#define SPDK_CRC32_POLYNOMIAL_REFLECT 0xedb88320UL
+
+/**
+ * CRC-32C (Castagnoli) polynomial (bit reflected)
+ */
+#define SPDK_CRC32C_POLYNOMIAL_REFLECT 0x82f63b78UL
+
+struct spdk_crc32_table {
+ uint32_t table[256];
+};
+
+/**
+ * Initialize a CRC32 lookup table for a given polynomial.
+ *
+ * \param table Table to fill with precalculated CRC-32 data.
+ * \param polynomial_reflect Bit-reflected CRC-32 polynomial.
+ */
+void crc32_table_init(struct spdk_crc32_table *table,
+ uint32_t polynomial_reflect);
+
+
+/**
+ * Calculate a partial CRC-32 checksum.
+ *
+ * \param table CRC-32 table initialized with crc32_table_init().
+ * \param buf Data buffer to checksum.
+ * \param len Length of buf in bytes.
+ * \param crc Previous CRC-32 value.
+ * \return Updated CRC-32 value.
+ */
+uint32_t crc32_update(const struct spdk_crc32_table *table,
+ const void *buf, size_t len,
+ uint32_t crc);
+
+#endif /* SPDK_UTIL_INTERNAL_H */
diff --git a/src/spdk/lib/util/uuid.c b/src/spdk/lib/util/uuid.c
new file mode 100644
index 000000000..176f65880
--- /dev/null
+++ b/src/spdk/lib/util/uuid.c
@@ -0,0 +1,73 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) Intel Corporation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/uuid.h"
+
+#include <uuid/uuid.h>
+
+SPDK_STATIC_ASSERT(sizeof(struct spdk_uuid) == sizeof(uuid_t), "Size mismatch");
+
+int
+spdk_uuid_parse(struct spdk_uuid *uuid, const char *uuid_str)
+{
+ return uuid_parse(uuid_str, (void *)uuid) == 0 ? 0 : -EINVAL;
+}
+
+int
+spdk_uuid_fmt_lower(char *uuid_str, size_t uuid_str_size, const struct spdk_uuid *uuid)
+{
+ if (uuid_str_size < SPDK_UUID_STRING_LEN) {
+ return -EINVAL;
+ }
+
+ uuid_unparse_lower((void *)uuid, uuid_str);
+ return 0;
+}
+
+int
+spdk_uuid_compare(const struct spdk_uuid *u1, const struct spdk_uuid *u2)
+{
+ return uuid_compare((void *)u1, (void *)u2);
+}
+
+void
+spdk_uuid_generate(struct spdk_uuid *uuid)
+{
+ uuid_generate((void *)uuid);
+}
+
+void
+spdk_uuid_copy(struct spdk_uuid *dst, const struct spdk_uuid *src)
+{
+ uuid_copy((void *)dst, (void *)src);
+}
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-05 09:55:15 +0000