From e6918187568dbd01842d8d1d2c808ce16a894239 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 21 Apr 2024 13:54:28 +0200 Subject: Adding upstream version 18.2.2. Signed-off-by: Daniel Baumann --- src/spdk/lib/util/Makefile | 47 + src/spdk/lib/util/base64.c | 262 +++++ src/spdk/lib/util/base64_neon.c | 225 +++++ src/spdk/lib/util/bit_array.c | 363 +++++++ src/spdk/lib/util/cpuset.c | 336 +++++++ src/spdk/lib/util/crc16.c | 668 +++++++++++++ src/spdk/lib/util/crc32.c | 95 ++ src/spdk/lib/util/crc32_ieee.c | 49 + src/spdk/lib/util/crc32c.c | 133 +++ src/spdk/lib/util/dif.c | 1999 +++++++++++++++++++++++++++++++++++++ src/spdk/lib/util/fd.c | 103 ++ src/spdk/lib/util/file.c | 71 ++ src/spdk/lib/util/iov.c | 111 ++ src/spdk/lib/util/math.c | 69 ++ src/spdk/lib/util/pipe.c | 246 +++++ src/spdk/lib/util/spdk_util.map | 128 +++ src/spdk/lib/util/strerror_tls.c | 43 + src/spdk/lib/util/string.c | 476 +++++++++ src/spdk/lib/util/util_internal.h | 77 ++ src/spdk/lib/util/uuid.c | 73 ++ 20 files changed, 5574 insertions(+) create mode 100644 src/spdk/lib/util/Makefile create mode 100644 src/spdk/lib/util/base64.c create mode 100644 src/spdk/lib/util/base64_neon.c create mode 100644 src/spdk/lib/util/bit_array.c create mode 100644 src/spdk/lib/util/cpuset.c create mode 100644 src/spdk/lib/util/crc16.c create mode 100644 src/spdk/lib/util/crc32.c create mode 100644 src/spdk/lib/util/crc32_ieee.c create mode 100644 src/spdk/lib/util/crc32c.c create mode 100644 src/spdk/lib/util/dif.c create mode 100644 src/spdk/lib/util/fd.c create mode 100644 src/spdk/lib/util/file.c create mode 100644 src/spdk/lib/util/iov.c create mode 100644 src/spdk/lib/util/math.c create mode 100644 src/spdk/lib/util/pipe.c create mode 100644 src/spdk/lib/util/spdk_util.map create mode 100644 src/spdk/lib/util/strerror_tls.c create mode 100644 src/spdk/lib/util/string.c create mode 100644 src/spdk/lib/util/util_internal.h create mode 100644 src/spdk/lib/util/uuid.c (limited to 'src/spdk/lib/util') diff --git a/src/spdk/lib/util/Makefile b/src/spdk/lib/util/Makefile new file mode 100644 index 000000000..23f8db6d0 --- /dev/null +++ b/src/spdk/lib/util/Makefile @@ -0,0 +1,47 @@ +# +# BSD LICENSE +# +# Copyright (c) Intel Corporation. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..) +include $(SPDK_ROOT_DIR)/mk/spdk.common.mk + +SO_VER := 2 +SO_MINOR := 0 + +C_SRCS = base64.c bit_array.c cpuset.c crc16.c crc32.c crc32c.c crc32_ieee.c \ + dif.c fd.c file.c iov.c math.c pipe.c strerror_tls.c string.c uuid.c +LIBNAME = util +LOCAL_SYS_LIBS = -luuid + +SPDK_MAP_FILE = $(abspath $(CURDIR)/spdk_util.map) + +include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk diff --git a/src/spdk/lib/util/base64.c b/src/spdk/lib/util/base64.c new file mode 100644 index 000000000..adc5e15da --- /dev/null +++ b/src/spdk/lib/util/base64.c @@ -0,0 +1,262 @@ +/*- + * BSD LICENSE + * + * Copyright(c) Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "spdk/stdinc.h" +#include "spdk/endian.h" +#include "spdk/base64.h" + +#ifdef __aarch64__ +#include "base64_neon.c" +#endif + +#define BASE64_ENC_BITMASK 0x3FUL +#define BASE64_PADDING_CHAR '=' + +static const char base64_enc_table[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789+/"; + +static const char base64_urfsafe_enc_table[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789-_"; + +static const uint8_t +base64_dec_table[] = { + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255, 255, 255, 63, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, 255, 255, 255, 255, + 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 255, 255, 255, 255, 255, + 255, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, +}; + +static const uint8_t +base64_urlsafe_dec_table[] = { + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255, 255, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, 255, 255, 255, 255, + 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 255, 255, 255, 255, 63, + 255, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, +}; + +static int +base64_encode(char *dst, const char *enc_table, const void *src, size_t src_len) +{ + uint32_t raw_u32; + + if (!dst || !src || src_len <= 0) { + return -EINVAL; + } + +#ifdef __aarch64__ + base64_encode_neon64(&dst, enc_table, &src, &src_len); +#endif + + while (src_len >= 4) { + raw_u32 = from_be32(src); + + *dst++ = enc_table[(raw_u32 >> 26) & BASE64_ENC_BITMASK]; + *dst++ = enc_table[(raw_u32 >> 20) & BASE64_ENC_BITMASK]; + *dst++ = enc_table[(raw_u32 >> 14) & BASE64_ENC_BITMASK]; + *dst++ = enc_table[(raw_u32 >> 8) & BASE64_ENC_BITMASK]; + + src_len -= 3; + src += 3; + } + + if (src_len == 0) { + goto out; + } + + raw_u32 = 0; + memcpy(&raw_u32, src, src_len); + raw_u32 = from_be32(&raw_u32); + + *dst++ = enc_table[(raw_u32 >> 26) & BASE64_ENC_BITMASK]; + *dst++ = enc_table[(raw_u32 >> 20) & BASE64_ENC_BITMASK]; + *dst++ = (src_len >= 2) ? enc_table[(raw_u32 >> 14) & BASE64_ENC_BITMASK] : BASE64_PADDING_CHAR; + *dst++ = (src_len == 3) ? enc_table[(raw_u32 >> 8) & BASE64_ENC_BITMASK] : BASE64_PADDING_CHAR; + +out: + *dst = '\0'; + + return 0; +} + +int +spdk_base64_encode(char *dst, const void *src, size_t src_len) +{ + return base64_encode(dst, base64_enc_table, src, src_len); +} + +int +spdk_base64_urlsafe_encode(char *dst, const void *src, size_t src_len) +{ + return base64_encode(dst, base64_urfsafe_enc_table, src, src_len); +} + +#ifdef __aarch64__ +static int +base64_decode(void *dst, size_t *_dst_len, const uint8_t *dec_table, + const uint8_t *dec_table_opt, const char *src) +#else +static int +base64_decode(void *dst, size_t *_dst_len, const uint8_t *dec_table, const char *src) +#endif +{ + size_t src_strlen; + size_t tail_len = 0; + const uint8_t *src_in; + uint32_t tmp[4]; + int i; + + if (!src) { + return -EINVAL; + } + + src_strlen = strlen(src); + + /* strlen of src should be 4n */ + if (src_strlen == 0 || src_strlen % 4 != 0) { + return -EINVAL; + } + + /* Consider Base64 padding, it at most has 2 padding characters. */ + for (i = 0; i < 2; i++) { + if (src[src_strlen - 1] != BASE64_PADDING_CHAR) { + break; + } + src_strlen--; + } + + /* strlen of src without padding shouldn't be 4n+1 */ + if (src_strlen == 0 || src_strlen % 4 == 1) { + return -EINVAL; + } + + if (_dst_len) { + *_dst_len = spdk_base64_get_decoded_len(src_strlen); + } + + /* If dst is NULL, the client is only concerned w/ _dst_len, return */ + if (!dst) { + return 0; + } + + src_in = (const uint8_t *) src; + +#ifdef __aarch64__ + base64_decode_neon64(&dst, dec_table_opt, &src_in, &src_strlen); + + if (src_strlen == 0) { + return 0; + } +#endif + + /* space of dst can be used by to_be32 */ + while (src_strlen > 4) { + tmp[0] = dec_table[*src_in++]; + tmp[1] = dec_table[*src_in++]; + tmp[2] = dec_table[*src_in++]; + tmp[3] = dec_table[*src_in++]; + + if (tmp[0] == 255 || tmp[1] == 255 || tmp[2] == 255 || tmp[3] == 255) { + return -EINVAL; + } + + to_be32(dst, tmp[3] << 8 | tmp[2] << 14 | tmp[1] << 20 | tmp[0] << 26); + + dst += 3; + src_strlen -= 4; + } + + /* space of dst is not enough to be used by to_be32 */ + tmp[0] = dec_table[src_in[0]]; + tmp[1] = dec_table[src_in[1]]; + tmp[2] = (src_strlen >= 3) ? dec_table[src_in[2]] : 0; + tmp[3] = (src_strlen == 4) ? dec_table[src_in[3]] : 0; + tail_len = src_strlen - 1; + + if (tmp[0] == 255 || tmp[1] == 255 || tmp[2] == 255 || tmp[3] == 255) { + return -EINVAL; + } + + to_be32(&tmp[3], tmp[3] << 8 | tmp[2] << 14 | tmp[1] << 20 | tmp[0] << 26); + memcpy(dst, (uint8_t *)&tmp[3], tail_len); + + return 0; +} + +int +spdk_base64_decode(void *dst, size_t *dst_len, const char *src) +{ +#ifdef __aarch64__ + return base64_decode(dst, dst_len, base64_dec_table, base64_dec_table_neon64, src); +#else + return base64_decode(dst, dst_len, base64_dec_table, src); +#endif +} + +int +spdk_base64_urlsafe_decode(void *dst, size_t *dst_len, const char *src) +{ +#ifdef __aarch64__ + return base64_decode(dst, dst_len, base64_urlsafe_dec_table, base64_urlsafe_dec_table_neon64, + src); +#else + return base64_decode(dst, dst_len, base64_urlsafe_dec_table, src); +#endif +} diff --git a/src/spdk/lib/util/base64_neon.c b/src/spdk/lib/util/base64_neon.c new file mode 100644 index 000000000..971cff06c --- /dev/null +++ b/src/spdk/lib/util/base64_neon.c @@ -0,0 +1,225 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2005-2007, Nick Galbreath + * Copyright (c) 2013-2017, Alfred Klomp + * Copyright (c) 2015-2017, Wojciech Mula + * Copyright (c) 2016-2017, Matthieu Darbois + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __aarch64__ +#error Unsupported hardware +#endif + +#include "spdk/stdinc.h" +/* + * Encoding + * Use a 64-byte lookup to do the encoding. + * Reuse existing base64_dec_table and base64_dec_table. + + * Decoding + * The input consists of five valid character sets in the Base64 alphabet, + * which we need to map back to the 6-bit values they represent. + * There are three ranges, two singles, and then there's the rest. + * + * LUT1[0-63] = base64_dec_table_neon64[0-63] + * LUT2[0-63] = base64_dec_table_neon64[64-127] + * # From To LUT Characters + * 1 [0..42] [255] #1 invalid input + * 2 [43] [62] #1 + + * 3 [44..46] [255] #1 invalid input + * 4 [47] [63] #1 / + * 5 [48..57] [52..61] #1 0..9 + * 6 [58..63] [255] #1 invalid input + * 7 [64] [255] #2 invalid input + * 8 [65..90] [0..25] #2 A..Z + * 9 [91..96] [255] #2 invalid input + * 10 [97..122] [26..51] #2 a..z + * 11 [123..126] [255] #2 invalid input + * (12) Everything else => invalid input + */ +static const uint8_t base64_dec_table_neon64[] = { + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255, 255, 255, 63, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, 255, 255, 255, 255, + 0, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, + 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 255, 255, 255, 255, + 255, 255, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, + 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 255, 255, 255, 255 +}; + +/* + * LUT1[0-63] = base64_urlsafe_dec_table_neon64[0-63] + * LUT2[0-63] = base64_urlsafe_dec_table_neon64[64-127] + * # From To LUT Characters + * 1 [0..44] [255] #1 invalid input + * 2 [45] [62] #1 - + * 3 [46..47] [255] #1 invalid input + * 5 [48..57] [52..61] #1 0..9 + * 6 [58..63] [255] #1 invalid input + * 7 [64] [255] #2 invalid input + * 8 [65..90] [0..25] #2 A..Z + * 9 [91..94] [255] #2 invalid input + * 10 [95] [63] #2 _ + * 11 [96] [255] #2 invalid input + * 12 [97..122] [26..51] #2 a..z + * 13 [123..126] [255] #2 invalid input + * (14) Everything else => invalid input + */ +static const uint8_t base64_urlsafe_dec_table_neon64[] = { + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255, 255, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, 255, 255, 255, 255, + 0, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, + 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 255, 255, 255, 255, + 63, 255, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, + 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 255, 255, 255, 255 +}; + +#include +#define CMPGT(s,n) vcgtq_u8((s), vdupq_n_u8(n)) + +static inline uint8x16x4_t +load_64byte_table(const uint8_t *p) +{ + uint8x16x4_t ret; + ret.val[0] = vld1q_u8(p + 0); + ret.val[1] = vld1q_u8(p + 16); + ret.val[2] = vld1q_u8(p + 32); + ret.val[3] = vld1q_u8(p + 48); + return ret; +} + +static void +base64_encode_neon64(char **dst, const char *enc_table, const void **src, size_t *src_len) +{ + const uint8x16x4_t tbl_enc = load_64byte_table(enc_table); + + while (*src_len >= 48) { + uint8x16x3_t str; + uint8x16x4_t res; + + /* Load 48 bytes and deinterleave */ + str = vld3q_u8((uint8_t *)*src); + + /* Divide bits of three input bytes over four output bytes and clear top two bits */ + res.val[0] = vshrq_n_u8(str.val[0], 2); + res.val[1] = vandq_u8(vorrq_u8(vshrq_n_u8(str.val[1], 4), vshlq_n_u8(str.val[0], 4)), + vdupq_n_u8(0x3F)); + res.val[2] = vandq_u8(vorrq_u8(vshrq_n_u8(str.val[2], 6), vshlq_n_u8(str.val[1], 2)), + vdupq_n_u8(0x3F)); + res.val[3] = vandq_u8(str.val[2], vdupq_n_u8(0x3F)); + + /* + * The bits have now been shifted to the right locations; + * translate their values 0..63 to the Base64 alphabet. + * Use a 64-byte table lookup: + */ + res.val[0] = vqtbl4q_u8(tbl_enc, res.val[0]); + res.val[1] = vqtbl4q_u8(tbl_enc, res.val[1]); + res.val[2] = vqtbl4q_u8(tbl_enc, res.val[2]); + res.val[3] = vqtbl4q_u8(tbl_enc, res.val[3]); + + /* Interleave and store result */ + vst4q_u8((uint8_t *)*dst, res); + + *src += 48; /* 3 * 16 bytes of input */ + *dst += 64; /* 4 * 16 bytes of output */ + *src_len -= 48; + } +} + +static void +base64_decode_neon64(void **dst, const uint8_t *dec_table_neon64, const uint8_t **src, + size_t *src_len) +{ + /* + * First LUT tbl_dec1 will use VTBL instruction (out of range indices are set to 0 in destination). + * Second LUT tbl_dec2 will use VTBX instruction (out of range indices will be unchanged in destination). + * Input [64..126] will be mapped to index [1..63] in tb1_dec2. Index 0 means that value comes from tb1_dec1. + */ + const uint8x16x4_t tbl_dec1 = load_64byte_table(dec_table_neon64); + const uint8x16x4_t tbl_dec2 = load_64byte_table(dec_table_neon64 + 64); + const uint8x16_t offset = vdupq_n_u8(63U); + + while (*src_len >= 64) { + + uint8x16x4_t dec1, dec2; + uint8x16x3_t dec; + + /* Load 64 bytes and deinterleave */ + uint8x16x4_t str = vld4q_u8((uint8_t *)*src); + + /* Get indices for 2nd LUT */ + dec2.val[0] = vqsubq_u8(str.val[0], offset); + dec2.val[1] = vqsubq_u8(str.val[1], offset); + dec2.val[2] = vqsubq_u8(str.val[2], offset); + dec2.val[3] = vqsubq_u8(str.val[3], offset); + + /* Get values from 1st LUT */ + dec1.val[0] = vqtbl4q_u8(tbl_dec1, str.val[0]); + dec1.val[1] = vqtbl4q_u8(tbl_dec1, str.val[1]); + dec1.val[2] = vqtbl4q_u8(tbl_dec1, str.val[2]); + dec1.val[3] = vqtbl4q_u8(tbl_dec1, str.val[3]); + + /* Get values from 2nd LUT */ + dec2.val[0] = vqtbx4q_u8(dec2.val[0], tbl_dec2, dec2.val[0]); + dec2.val[1] = vqtbx4q_u8(dec2.val[1], tbl_dec2, dec2.val[1]); + dec2.val[2] = vqtbx4q_u8(dec2.val[2], tbl_dec2, dec2.val[2]); + dec2.val[3] = vqtbx4q_u8(dec2.val[3], tbl_dec2, dec2.val[3]); + + /* Get final values */ + str.val[0] = vorrq_u8(dec1.val[0], dec2.val[0]); + str.val[1] = vorrq_u8(dec1.val[1], dec2.val[1]); + str.val[2] = vorrq_u8(dec1.val[2], dec2.val[2]); + str.val[3] = vorrq_u8(dec1.val[3], dec2.val[3]); + + /* Check for invalid input, any value larger than 63 */ + uint8x16_t classified = CMPGT(str.val[0], 63); + classified = vorrq_u8(classified, CMPGT(str.val[1], 63)); + classified = vorrq_u8(classified, CMPGT(str.val[2], 63)); + classified = vorrq_u8(classified, CMPGT(str.val[3], 63)); + + /* check that all bits are zero */ + if (vmaxvq_u8(classified) != 0U) { + break; + } + + /* Compress four bytes into three */ + dec.val[0] = vorrq_u8(vshlq_n_u8(str.val[0], 2), vshrq_n_u8(str.val[1], 4)); + dec.val[1] = vorrq_u8(vshlq_n_u8(str.val[1], 4), vshrq_n_u8(str.val[2], 2)); + dec.val[2] = vorrq_u8(vshlq_n_u8(str.val[2], 6), str.val[3]); + + /* Interleave and store decoded result */ + vst3q_u8((uint8_t *)*dst, dec); + + *src += 64; + *dst += 48; + *src_len -= 64; + } +} diff --git a/src/spdk/lib/util/bit_array.c b/src/spdk/lib/util/bit_array.c new file mode 100644 index 000000000..43c1a4d9b --- /dev/null +++ b/src/spdk/lib/util/bit_array.c @@ -0,0 +1,363 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "spdk/stdinc.h" + +#include "spdk/bit_array.h" +#include "spdk/env.h" + +#include "spdk/likely.h" +#include "spdk/util.h" + +typedef uint64_t spdk_bit_array_word; +#define SPDK_BIT_ARRAY_WORD_TZCNT(x) (__builtin_ctzll(x)) +#define SPDK_BIT_ARRAY_WORD_POPCNT(x) (__builtin_popcountll(x)) +#define SPDK_BIT_ARRAY_WORD_C(x) ((spdk_bit_array_word)(x)) +#define SPDK_BIT_ARRAY_WORD_BYTES sizeof(spdk_bit_array_word) +#define SPDK_BIT_ARRAY_WORD_BITS (SPDK_BIT_ARRAY_WORD_BYTES * 8) +#define SPDK_BIT_ARRAY_WORD_INDEX_SHIFT spdk_u32log2(SPDK_BIT_ARRAY_WORD_BITS) +#define SPDK_BIT_ARRAY_WORD_INDEX_MASK ((1u << SPDK_BIT_ARRAY_WORD_INDEX_SHIFT) - 1) + +struct spdk_bit_array { + uint32_t bit_count; + spdk_bit_array_word words[]; +}; + +struct spdk_bit_array * +spdk_bit_array_create(uint32_t num_bits) +{ + struct spdk_bit_array *ba = NULL; + + spdk_bit_array_resize(&ba, num_bits); + + return ba; +} + +void +spdk_bit_array_free(struct spdk_bit_array **bap) +{ + struct spdk_bit_array *ba; + + if (!bap) { + return; + } + + ba = *bap; + *bap = NULL; + spdk_free(ba); +} + +static inline uint32_t +bit_array_word_count(uint32_t num_bits) +{ + return (num_bits + SPDK_BIT_ARRAY_WORD_BITS - 1) >> SPDK_BIT_ARRAY_WORD_INDEX_SHIFT; +} + +static inline spdk_bit_array_word +bit_array_word_mask(uint32_t num_bits) +{ + assert(num_bits < SPDK_BIT_ARRAY_WORD_BITS); + return (SPDK_BIT_ARRAY_WORD_C(1) << num_bits) - 1; +} + +int +spdk_bit_array_resize(struct spdk_bit_array **bap, uint32_t num_bits) +{ + struct spdk_bit_array *new_ba; + uint32_t old_word_count, new_word_count; + size_t new_size; + + /* + * Max number of bits allowed is UINT32_MAX - 1, because we use UINT32_MAX to denote + * when a set or cleared bit cannot be found. + */ + if (!bap || num_bits == UINT32_MAX) { + return -EINVAL; + } + + new_word_count = bit_array_word_count(num_bits); + new_size = offsetof(struct spdk_bit_array, words) + new_word_count * SPDK_BIT_ARRAY_WORD_BYTES; + + /* + * Always keep one extra word with a 0 and a 1 past the actual required size so that the + * find_first functions can just keep going until they match. + */ + new_size += SPDK_BIT_ARRAY_WORD_BYTES; + + new_ba = (struct spdk_bit_array *)spdk_realloc(*bap, new_size, 64); + if (!new_ba) { + return -ENOMEM; + } + + /* + * Set up special extra word (see above comment about find_first_clear). + * + * This is set to 0b10 so that find_first_clear will find a 0 at the very first + * bit past the end of the buffer, and find_first_set will find a 1 at the next bit + * past that. + */ + new_ba->words[new_word_count] = 0x2; + + if (*bap == NULL) { + old_word_count = 0; + new_ba->bit_count = 0; + } else { + old_word_count = bit_array_word_count(new_ba->bit_count); + } + + if (new_word_count > old_word_count) { + /* Zero out new entries */ + memset(&new_ba->words[old_word_count], 0, + (new_word_count - old_word_count) * SPDK_BIT_ARRAY_WORD_BYTES); + } else if (new_word_count == old_word_count && num_bits < new_ba->bit_count) { + /* Make sure any existing partial last word is cleared beyond the new num_bits. */ + uint32_t last_word_bits; + spdk_bit_array_word mask; + + last_word_bits = num_bits & SPDK_BIT_ARRAY_WORD_INDEX_MASK; + mask = bit_array_word_mask(last_word_bits); + new_ba->words[old_word_count - 1] &= mask; + } + + new_ba->bit_count = num_bits; + *bap = new_ba; + return 0; +} + +uint32_t +spdk_bit_array_capacity(const struct spdk_bit_array *ba) +{ + return ba->bit_count; +} + +static inline int +bit_array_get_word(const struct spdk_bit_array *ba, uint32_t bit_index, + uint32_t *word_index, uint32_t *word_bit_index) +{ + if (spdk_unlikely(bit_index >= ba->bit_count)) { + return -EINVAL; + } + + *word_index = bit_index >> SPDK_BIT_ARRAY_WORD_INDEX_SHIFT; + *word_bit_index = bit_index & SPDK_BIT_ARRAY_WORD_INDEX_MASK; + + return 0; +} + +bool +spdk_bit_array_get(const struct spdk_bit_array *ba, uint32_t bit_index) +{ + uint32_t word_index, word_bit_index; + + if (bit_array_get_word(ba, bit_index, &word_index, &word_bit_index)) { + return false; + } + + return (ba->words[word_index] >> word_bit_index) & 1U; +} + +int +spdk_bit_array_set(struct spdk_bit_array *ba, uint32_t bit_index) +{ + uint32_t word_index, word_bit_index; + + if (bit_array_get_word(ba, bit_index, &word_index, &word_bit_index)) { + return -EINVAL; + } + + ba->words[word_index] |= (SPDK_BIT_ARRAY_WORD_C(1) << word_bit_index); + return 0; +} + +void +spdk_bit_array_clear(struct spdk_bit_array *ba, uint32_t bit_index) +{ + uint32_t word_index, word_bit_index; + + if (bit_array_get_word(ba, bit_index, &word_index, &word_bit_index)) { + /* + * Clearing past the end of the bit array is a no-op, since bit past the end + * are implicitly 0. + */ + return; + } + + ba->words[word_index] &= ~(SPDK_BIT_ARRAY_WORD_C(1) << word_bit_index); +} + +static inline uint32_t +bit_array_find_first(const struct spdk_bit_array *ba, uint32_t start_bit_index, + spdk_bit_array_word xor_mask) +{ + uint32_t word_index, first_word_bit_index; + spdk_bit_array_word word, first_word_mask; + const spdk_bit_array_word *words, *cur_word; + + if (spdk_unlikely(start_bit_index >= ba->bit_count)) { + return ba->bit_count; + } + + word_index = start_bit_index >> SPDK_BIT_ARRAY_WORD_INDEX_SHIFT; + words = ba->words; + cur_word = &words[word_index]; + + /* + * Special case for first word: skip start_bit_index % SPDK_BIT_ARRAY_WORD_BITS bits + * within the first word. + */ + first_word_bit_index = start_bit_index & SPDK_BIT_ARRAY_WORD_INDEX_MASK; + first_word_mask = bit_array_word_mask(first_word_bit_index); + + word = (*cur_word ^ xor_mask) & ~first_word_mask; + + /* + * spdk_bit_array_resize() guarantees that an extra word with a 1 and a 0 will always be + * at the end of the words[] array, so just keep going until a word matches. + */ + while (word == 0) { + word = *++cur_word ^ xor_mask; + } + + return ((uintptr_t)cur_word - (uintptr_t)words) * 8 + SPDK_BIT_ARRAY_WORD_TZCNT(word); +} + + +uint32_t +spdk_bit_array_find_first_set(const struct spdk_bit_array *ba, uint32_t start_bit_index) +{ + uint32_t bit_index; + + bit_index = bit_array_find_first(ba, start_bit_index, 0); + + /* + * If we ran off the end of the array and found the 1 bit in the extra word, + * return UINT32_MAX to indicate no actual 1 bits were found. + */ + if (bit_index >= ba->bit_count) { + bit_index = UINT32_MAX; + } + + return bit_index; +} + +uint32_t +spdk_bit_array_find_first_clear(const struct spdk_bit_array *ba, uint32_t start_bit_index) +{ + uint32_t bit_index; + + bit_index = bit_array_find_first(ba, start_bit_index, SPDK_BIT_ARRAY_WORD_C(-1)); + + /* + * If we ran off the end of the array and found the 0 bit in the extra word, + * return UINT32_MAX to indicate no actual 0 bits were found. + */ + if (bit_index >= ba->bit_count) { + bit_index = UINT32_MAX; + } + + return bit_index; +} + +uint32_t +spdk_bit_array_count_set(const struct spdk_bit_array *ba) +{ + const spdk_bit_array_word *cur_word = ba->words; + uint32_t word_count = bit_array_word_count(ba->bit_count); + uint32_t set_count = 0; + + while (word_count--) { + /* + * No special treatment is needed for the last (potentially partial) word, since + * spdk_bit_array_resize() makes sure the bits past bit_count are cleared. + */ + set_count += SPDK_BIT_ARRAY_WORD_POPCNT(*cur_word++); + } + + return set_count; +} + +uint32_t +spdk_bit_array_count_clear(const struct spdk_bit_array *ba) +{ + return ba->bit_count - spdk_bit_array_count_set(ba); +} + +void +spdk_bit_array_store_mask(const struct spdk_bit_array *ba, void *mask) +{ + uint32_t size, i; + uint32_t num_bits = spdk_bit_array_capacity(ba); + + size = num_bits / CHAR_BIT; + memcpy(mask, ba->words, size); + + for (i = 0; i < num_bits % CHAR_BIT; i++) { + if (spdk_bit_array_get(ba, i + size * CHAR_BIT)) { + ((uint8_t *)mask)[size] |= (1U << i); + } else { + ((uint8_t *)mask)[size] &= ~(1U << i); + } + } +} + +void +spdk_bit_array_load_mask(struct spdk_bit_array *ba, const void *mask) +{ + uint32_t size, i; + uint32_t num_bits = spdk_bit_array_capacity(ba); + + size = num_bits / CHAR_BIT; + memcpy(ba->words, mask, size); + + for (i = 0; i < num_bits % CHAR_BIT; i++) { + if (((uint8_t *)mask)[size] & (1U << i)) { + spdk_bit_array_set(ba, i + size * CHAR_BIT); + } else { + spdk_bit_array_clear(ba, i + size * CHAR_BIT); + } + } +} + +void +spdk_bit_array_clear_mask(struct spdk_bit_array *ba) +{ + uint32_t size, i; + uint32_t num_bits = spdk_bit_array_capacity(ba); + + size = num_bits / CHAR_BIT; + memset(ba->words, 0, size); + + for (i = 0; i < num_bits % CHAR_BIT; i++) { + spdk_bit_array_clear(ba, i + size * CHAR_BIT); + } +} diff --git a/src/spdk/lib/util/cpuset.c b/src/spdk/lib/util/cpuset.c new file mode 100644 index 000000000..8d7c8dc89 --- /dev/null +++ b/src/spdk/lib/util/cpuset.c @@ -0,0 +1,336 @@ +/*- + * BSD LICENSE + * + * Copyright(c) Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "spdk/cpuset.h" +#include "spdk/log.h" + +struct spdk_cpuset * +spdk_cpuset_alloc(void) +{ + return (struct spdk_cpuset *)calloc(sizeof(struct spdk_cpuset), 1); +} + +void +spdk_cpuset_free(struct spdk_cpuset *set) +{ + free(set); +} + +bool +spdk_cpuset_equal(const struct spdk_cpuset *set1, const struct spdk_cpuset *set2) +{ + assert(set1 != NULL); + assert(set2 != NULL); + return memcmp(set1->cpus, set2->cpus, sizeof(set2->cpus)) == 0; +} + +void +spdk_cpuset_copy(struct spdk_cpuset *dst, const struct spdk_cpuset *src) +{ + assert(dst != NULL); + assert(src != NULL); + memcpy(&dst->cpus, &src->cpus, sizeof(src->cpus)); +} + +void +spdk_cpuset_negate(struct spdk_cpuset *set) +{ + unsigned int i; + assert(set != NULL); + for (i = 0; i < sizeof(set->cpus); i++) { + set->cpus[i] = ~set->cpus[i]; + } +} + +void +spdk_cpuset_and(struct spdk_cpuset *dst, const struct spdk_cpuset *src) +{ + unsigned int i; + assert(dst != NULL); + assert(src != NULL); + for (i = 0; i < sizeof(src->cpus); i++) { + dst->cpus[i] &= src->cpus[i]; + } +} + +void +spdk_cpuset_or(struct spdk_cpuset *dst, const struct spdk_cpuset *src) +{ + unsigned int i; + assert(dst != NULL); + assert(src != NULL); + for (i = 0; i < sizeof(src->cpus); i++) { + dst->cpus[i] |= src->cpus[i]; + } +} + +void +spdk_cpuset_xor(struct spdk_cpuset *dst, const struct spdk_cpuset *src) +{ + unsigned int i; + assert(dst != NULL); + assert(src != NULL); + for (i = 0; i < sizeof(src->cpus); i++) { + dst->cpus[i] ^= src->cpus[i]; + } +} + +void +spdk_cpuset_zero(struct spdk_cpuset *set) +{ + assert(set != NULL); + memset(set->cpus, 0, sizeof(set->cpus)); +} + +void +spdk_cpuset_set_cpu(struct spdk_cpuset *set, uint32_t cpu, bool state) +{ + assert(set != NULL); + assert(cpu < sizeof(set->cpus) * 8); + if (state) { + set->cpus[cpu / 8] |= (1U << (cpu % 8)); + } else { + set->cpus[cpu / 8] &= ~(1U << (cpu % 8)); + } +} + +bool +spdk_cpuset_get_cpu(const struct spdk_cpuset *set, uint32_t cpu) +{ + assert(set != NULL); + assert(cpu < sizeof(set->cpus) * 8); + return (set->cpus[cpu / 8] >> (cpu % 8)) & 1U; +} + +uint32_t +spdk_cpuset_count(const struct spdk_cpuset *set) +{ + uint32_t count = 0; + uint8_t n; + unsigned int i; + for (i = 0; i < sizeof(set->cpus); i++) { + n = set->cpus[i]; + while (n) { + n &= (n - 1); + count++; + } + } + return count; +} + +const char * +spdk_cpuset_fmt(struct spdk_cpuset *set) +{ + uint32_t lcore, lcore_max = 0; + int val, i, n; + char *ptr; + static const char *hex = "0123456789abcdef"; + + assert(set != NULL); + + for (lcore = 0; lcore < sizeof(set->cpus) * 8; lcore++) { + if (spdk_cpuset_get_cpu(set, lcore)) { + lcore_max = lcore; + } + } + + ptr = set->str; + n = lcore_max / 8; + val = set->cpus[n]; + + /* Store first number only if it is not leading zero */ + if ((val & 0xf0) != 0) { + *(ptr++) = hex[(val & 0xf0) >> 4]; + } + *(ptr++) = hex[val & 0x0f]; + + for (i = n - 1; i >= 0; i--) { + val = set->cpus[i]; + *(ptr++) = hex[(val & 0xf0) >> 4]; + *(ptr++) = hex[val & 0x0f]; + } + *ptr = '\0'; + + return set->str; +} + +static int +hex_value(uint8_t c) +{ +#define V(x, y) [x] = y + 1 + static const int8_t val[256] = { + V('0', 0), V('1', 1), V('2', 2), V('3', 3), V('4', 4), + V('5', 5), V('6', 6), V('7', 7), V('8', 8), V('9', 9), + V('A', 0xA), V('B', 0xB), V('C', 0xC), V('D', 0xD), V('E', 0xE), V('F', 0xF), + V('a', 0xA), V('b', 0xB), V('c', 0xC), V('d', 0xD), V('e', 0xE), V('f', 0xF), + }; +#undef V + + return val[c] - 1; +} + +static int +parse_list(const char *mask, struct spdk_cpuset *set) +{ + char *end; + const char *ptr = mask; + uint32_t lcore; + uint32_t lcore_min, lcore_max; + + spdk_cpuset_zero(set); + lcore_min = UINT32_MAX; + + ptr++; + end = (char *)ptr; + do { + while (isblank(*ptr)) { + ptr++; + } + if (*ptr == '\0' || *ptr == ']' || *ptr == '-' || *ptr == ',') { + goto invalid_character; + } + + errno = 0; + lcore = strtoul(ptr, &end, 10); + if (errno) { + SPDK_ERRLOG("Conversion of core mask in '%s' failed\n", mask); + return -1; + } + + if (lcore >= sizeof(set->cpus) * 8) { + SPDK_ERRLOG("Core number %" PRIu32 " is out of range in '%s'\n", lcore, mask); + return -1; + } + + while (isblank(*end)) { + end++; + } + + if (*end == '-') { + lcore_min = lcore; + } else if (*end == ',' || *end == ']') { + lcore_max = lcore; + if (lcore_min == UINT32_MAX) { + lcore_min = lcore; + } + if (lcore_min > lcore_max) { + SPDK_ERRLOG("Invalid range of CPUs (%" PRIu32 " > %" PRIu32 ")\n", + lcore_min, lcore_max); + return -1; + } + for (lcore = lcore_min; lcore <= lcore_max; lcore++) { + spdk_cpuset_set_cpu(set, lcore, true); + } + lcore_min = UINT32_MAX; + } else { + goto invalid_character; + } + + ptr = end + 1; + + } while (*end != ']'); + + return 0; + +invalid_character: + if (*end == '\0') { + SPDK_ERRLOG("Unexpected end of core list '%s'\n", mask); + } else { + SPDK_ERRLOG("Parsing of core list '%s' failed on character '%c'\n", mask, *end); + } + return -1; +} + +static int +parse_mask(const char *mask, struct spdk_cpuset *set, size_t len) +{ + int i, j; + char c; + int val; + uint32_t lcore = 0; + + if (mask[0] == '0' && (mask[1] == 'x' || mask[1] == 'X')) { + mask += 2; + len -= 2; + } + + spdk_cpuset_zero(set); + for (i = len - 1; i >= 0; i--) { + c = mask[i]; + val = hex_value(c); + if (val < 0) { + /* Invalid character */ + SPDK_ERRLOG("Invalid character in core mask '%s' (%c)\n", mask, c); + return -1; + } + for (j = 0; j < 4 && lcore < sizeof(set->cpus); j++, lcore++) { + if ((1 << j) & val) { + spdk_cpuset_set_cpu(set, lcore, true); + } + } + } + + return 0; +} + +int +spdk_cpuset_parse(struct spdk_cpuset *set, const char *mask) +{ + int ret; + size_t len; + + if (mask == NULL || set == NULL) { + return -1; + } + + while (isblank(*mask)) { + mask++; + } + + len = strlen(mask); + while (len > 0 && isblank(mask[len - 1])) { + len--; + } + + if (len == 0) { + return -1; + } + + if (mask[0] == '[') { + ret = parse_list(mask, set); + } else { + ret = parse_mask(mask, set, len); + } + + return ret; +} diff --git a/src/spdk/lib/util/crc16.c b/src/spdk/lib/util/crc16.c new file mode 100644 index 000000000..2ba168c4b --- /dev/null +++ b/src/spdk/lib/util/crc16.c @@ -0,0 +1,668 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "spdk/crc16.h" +#include "spdk/config.h" + +/* + * Use Intelligent Storage Acceleration Library for line speed CRC + */ + +#ifdef SPDK_CONFIG_ISAL +#include "isa-l/include/crc.h" + +uint16_t +spdk_crc16_t10dif(uint16_t init_crc, const void *buf, size_t len) +{ + return (crc16_t10dif(init_crc, buf, len)); +} + +uint16_t +spdk_crc16_t10dif_copy(uint16_t init_crc, uint8_t *dst, uint8_t *src, + size_t len) +{ + return (crc16_t10dif_copy(init_crc, dst, src, len)); +} + +#else +/* + * Use table-driven (somewhat faster) CRC + */ + +/* + * Static tables used for the table_driven implementation. + */ + +static const uint16_t crc_table_fast[16][256] = { + { + 0x0000u, 0x8BB7u, 0x9CD9u, 0x176Eu, 0xB205u, 0x39B2u, 0x2EDCu, 0xA56Bu, + 0xEFBDu, 0x640Au, 0x7364u, 0xF8D3u, 0x5DB8u, 0xD60Fu, 0xC161u, 0x4AD6u, + 0x54CDu, 0xDF7Au, 0xC814u, 0x43A3u, 0xE6C8u, 0x6D7Fu, 0x7A11u, 0xF1A6u, + 0xBB70u, 0x30C7u, 0x27A9u, 0xAC1Eu, 0x0975u, 0x82C2u, 0x95ACu, 0x1E1Bu, + 0xA99Au, 0x222Du, 0x3543u, 0xBEF4u, 0x1B9Fu, 0x9028u, 0x8746u, 0x0CF1u, + 0x4627u, 0xCD90u, 0xDAFEu, 0x5149u, 0xF422u, 0x7F95u, 0x68FBu, 0xE34Cu, + 0xFD57u, 0x76E0u, 0x618Eu, 0xEA39u, 0x4F52u, 0xC4E5u, 0xD38Bu, 0x583Cu, + 0x12EAu, 0x995Du, 0x8E33u, 0x0584u, 0xA0EFu, 0x2B58u, 0x3C36u, 0xB781u, + 0xD883u, 0x5334u, 0x445Au, 0xCFEDu, 0x6A86u, 0xE131u, 0xF65Fu, 0x7DE8u, + 0x373Eu, 0xBC89u, 0xABE7u, 0x2050u, 0x853Bu, 0x0E8Cu, 0x19E2u, 0x9255u, + 0x8C4Eu, 0x07F9u, 0x1097u, 0x9B20u, 0x3E4Bu, 0xB5FCu, 0xA292u, 0x2925u, + 0x63F3u, 0xE844u, 0xFF2Au, 0x749Du, 0xD1F6u, 0x5A41u, 0x4D2Fu, 0xC698u, + 0x7119u, 0xFAAEu, 0xEDC0u, 0x6677u, 0xC31Cu, 0x48ABu, 0x5FC5u, 0xD472u, + 0x9EA4u, 0x1513u, 0x027Du, 0x89CAu, 0x2CA1u, 0xA716u, 0xB078u, 0x3BCFu, + 0x25D4u, 0xAE63u, 0xB90Du, 0x32BAu, 0x97D1u, 0x1C66u, 0x0B08u, 0x80BFu, + 0xCA69u, 0x41DEu, 0x56B0u, 0xDD07u, 0x786Cu, 0xF3DBu, 0xE4B5u, 0x6F02u, + 0x3AB1u, 0xB106u, 0xA668u, 0x2DDFu, 0x88B4u, 0x0303u, 0x146Du, 0x9FDAu, + 0xD50Cu, 0x5EBBu, 0x49D5u, 0xC262u, 0x6709u, 0xECBEu, 0xFBD0u, 0x7067u, + 0x6E7Cu, 0xE5CBu, 0xF2A5u, 0x7912u, 0xDC79u, 0x57CEu, 0x40A0u, 0xCB17u, + 0x81C1u, 0x0A76u, 0x1D18u, 0x96AFu, 0x33C4u, 0xB873u, 0xAF1Du, 0x24AAu, + 0x932Bu, 0x189Cu, 0x0FF2u, 0x8445u, 0x212Eu, 0xAA99u, 0xBDF7u, 0x3640u, + 0x7C96u, 0xF721u, 0xE04Fu, 0x6BF8u, 0xCE93u, 0x4524u, 0x524Au, 0xD9FDu, + 0xC7E6u, 0x4C51u, 0x5B3Fu, 0xD088u, 0x75E3u, 0xFE54u, 0xE93Au, 0x628Du, + 0x285Bu, 0xA3ECu, 0xB482u, 0x3F35u, 0x9A5Eu, 0x11E9u, 0x0687u, 0x8D30u, + 0xE232u, 0x6985u, 0x7EEBu, 0xF55Cu, 0x5037u, 0xDB80u, 0xCCEEu, 0x4759u, + 0x0D8Fu, 0x8638u, 0x9156u, 0x1AE1u, 0xBF8Au, 0x343Du, 0x2353u, 0xA8E4u, + 0xB6FFu, 0x3D48u, 0x2A26u, 0xA191u, 0x04FAu, 0x8F4Du, 0x9823u, 0x1394u, + 0x5942u, 0xD2F5u, 0xC59Bu, 0x4E2Cu, 0xEB47u, 0x60F0u, 0x779Eu, 0xFC29u, + 0x4BA8u, 0xC01Fu, 0xD771u, 0x5CC6u, 0xF9ADu, 0x721Au, 0x6574u, 0xEEC3u, + 0xA415u, 0x2FA2u, 0x38CCu, 0xB37Bu, 0x1610u, 0x9DA7u, 0x8AC9u, 0x017Eu, + 0x1F65u, 0x94D2u, 0x83BCu, 0x080Bu, 0xAD60u, 0x26D7u, 0x31B9u, 0xBA0Eu, + 0xF0D8u, 0x7B6Fu, 0x6C01u, 0xE7B6u, 0x42DDu, 0xC96Au, 0xDE04u, 0x55B3u + }, + { + 0x0000u, 0x7562u, 0xEAC4u, 0x9FA6u, 0x5E3Fu, 0x2B5Du, 0xB4FBu, 0xC199u, + 0xBC7Eu, 0xC91Cu, 0x56BAu, 0x23D8u, 0xE241u, 0x9723u, 0x0885u, 0x7DE7u, + 0xF34Bu, 0x8629u, 0x198Fu, 0x6CEDu, 0xAD74u, 0xD816u, 0x47B0u, 0x32D2u, + 0x4F35u, 0x3A57u, 0xA5F1u, 0xD093u, 0x110Au, 0x6468u, 0xFBCEu, 0x8EACu, + 0x6D21u, 0x1843u, 0x87E5u, 0xF287u, 0x331Eu, 0x467Cu, 0xD9DAu, 0xACB8u, + 0xD15Fu, 0xA43Du, 0x3B9Bu, 0x4EF9u, 0x8F60u, 0xFA02u, 0x65A4u, 0x10C6u, + 0x9E6Au, 0xEB08u, 0x74AEu, 0x01CCu, 0xC055u, 0xB537u, 0x2A91u, 0x5FF3u, + 0x2214u, 0x5776u, 0xC8D0u, 0xBDB2u, 0x7C2Bu, 0x0949u, 0x96EFu, 0xE38Du, + 0xDA42u, 0xAF20u, 0x3086u, 0x45E4u, 0x847Du, 0xF11Fu, 0x6EB9u, 0x1BDBu, + 0x663Cu, 0x135Eu, 0x8CF8u, 0xF99Au, 0x3803u, 0x4D61u, 0xD2C7u, 0xA7A5u, + 0x2909u, 0x5C6Bu, 0xC3CDu, 0xB6AFu, 0x7736u, 0x0254u, 0x9DF2u, 0xE890u, + 0x9577u, 0xE015u, 0x7FB3u, 0x0AD1u, 0xCB48u, 0xBE2Au, 0x218Cu, 0x54EEu, + 0xB763u, 0xC201u, 0x5DA7u, 0x28C5u, 0xE95Cu, 0x9C3Eu, 0x0398u, 0x76FAu, + 0x0B1Du, 0x7E7Fu, 0xE1D9u, 0x94BBu, 0x5522u, 0x2040u, 0xBFE6u, 0xCA84u, + 0x4428u, 0x314Au, 0xAEECu, 0xDB8Eu, 0x1A17u, 0x6F75u, 0xF0D3u, 0x85B1u, + 0xF856u, 0x8D34u, 0x1292u, 0x67F0u, 0xA669u, 0xD30Bu, 0x4CADu, 0x39CFu, + 0x3F33u, 0x4A51u, 0xD5F7u, 0xA095u, 0x610Cu, 0x146Eu, 0x8BC8u, 0xFEAAu, + 0x834Du, 0xF62Fu, 0x6989u, 0x1CEBu, 0xDD72u, 0xA810u, 0x37B6u, 0x42D4u, + 0xCC78u, 0xB91Au, 0x26BCu, 0x53DEu, 0x9247u, 0xE725u, 0x7883u, 0x0DE1u, + 0x7006u, 0x0564u, 0x9AC2u, 0xEFA0u, 0x2E39u, 0x5B5Bu, 0xC4FDu, 0xB19Fu, + 0x5212u, 0x2770u, 0xB8D6u, 0xCDB4u, 0x0C2Du, 0x794Fu, 0xE6E9u, 0x938Bu, + 0xEE6Cu, 0x9B0Eu, 0x04A8u, 0x71CAu, 0xB053u, 0xC531u, 0x5A97u, 0x2FF5u, + 0xA159u, 0xD43Bu, 0x4B9Du, 0x3EFFu, 0xFF66u, 0x8A04u, 0x15A2u, 0x60C0u, + 0x1D27u, 0x6845u, 0xF7E3u, 0x8281u, 0x4318u, 0x367Au, 0xA9DCu, 0xDCBEu, + 0xE571u, 0x9013u, 0x0FB5u, 0x7AD7u, 0xBB4Eu, 0xCE2Cu, 0x518Au, 0x24E8u, + 0x590Fu, 0x2C6Du, 0xB3CBu, 0xC6A9u, 0x0730u, 0x7252u, 0xEDF4u, 0x9896u, + 0x163Au, 0x6358u, 0xFCFEu, 0x899Cu, 0x4805u, 0x3D67u, 0xA2C1u, 0xD7A3u, + 0xAA44u, 0xDF26u, 0x4080u, 0x35E2u, 0xF47Bu, 0x8119u, 0x1EBFu, 0x6BDDu, + 0x8850u, 0xFD32u, 0x6294u, 0x17F6u, 0xD66Fu, 0xA30Du, 0x3CABu, 0x49C9u, + 0x342Eu, 0x414Cu, 0xDEEAu, 0xAB88u, 0x6A11u, 0x1F73u, 0x80D5u, 0xF5B7u, + 0x7B1Bu, 0x0E79u, 0x91DFu, 0xE4BDu, 0x2524u, 0x5046u, 0xCFE0u, 0xBA82u, + 0xC765u, 0xB207u, 0x2DA1u, 0x58C3u, 0x995Au, 0xEC38u, 0x739Eu, 0x06FCu + }, + { + 0x0000u, 0x7E66u, 0xFCCCu, 0x82AAu, 0x722Fu, 0x0C49u, 0x8EE3u, 0xF085u, + 0xE45Eu, 0x9A38u, 0x1892u, 0x66F4u, 0x9671u, 0xE817u, 0x6ABDu, 0x14DBu, + 0x430Bu, 0x3D6Du, 0xBFC7u, 0xC1A1u, 0x3124u, 0x4F42u, 0xCDE8u, 0xB38Eu, + 0xA755u, 0xD933u, 0x5B99u, 0x25FFu, 0xD57Au, 0xAB1Cu, 0x29B6u, 0x57D0u, + 0x8616u, 0xF870u, 0x7ADAu, 0x04BCu, 0xF439u, 0x8A5Fu, 0x08F5u, 0x7693u, + 0x6248u, 0x1C2Eu, 0x9E84u, 0xE0E2u, 0x1067u, 0x6E01u, 0xECABu, 0x92CDu, + 0xC51Du, 0xBB7Bu, 0x39D1u, 0x47B7u, 0xB732u, 0xC954u, 0x4BFEu, 0x3598u, + 0x2143u, 0x5F25u, 0xDD8Fu, 0xA3E9u, 0x536Cu, 0x2D0Au, 0xAFA0u, 0xD1C6u, + 0x879Bu, 0xF9FDu, 0x7B57u, 0x0531u, 0xF5B4u, 0x8BD2u, 0x0978u, 0x771Eu, + 0x63C5u, 0x1DA3u, 0x9F09u, 0xE16Fu, 0x11EAu, 0x6F8Cu, 0xED26u, 0x9340u, + 0xC490u, 0xBAF6u, 0x385Cu, 0x463Au, 0xB6BFu, 0xC8D9u, 0x4A73u, 0x3415u, + 0x20CEu, 0x5EA8u, 0xDC02u, 0xA264u, 0x52E1u, 0x2C87u, 0xAE2Du, 0xD04Bu, + 0x018Du, 0x7FEBu, 0xFD41u, 0x8327u, 0x73A2u, 0x0DC4u, 0x8F6Eu, 0xF108u, + 0xE5D3u, 0x9BB5u, 0x191Fu, 0x6779u, 0x97FCu, 0xE99Au, 0x6B30u, 0x1556u, + 0x4286u, 0x3CE0u, 0xBE4Au, 0xC02Cu, 0x30A9u, 0x4ECFu, 0xCC65u, 0xB203u, + 0xA6D8u, 0xD8BEu, 0x5A14u, 0x2472u, 0xD4F7u, 0xAA91u, 0x283Bu, 0x565Du, + 0x8481u, 0xFAE7u, 0x784Du, 0x062Bu, 0xF6AEu, 0x88C8u, 0x0A62u, 0x7404u, + 0x60DFu, 0x1EB9u, 0x9C13u, 0xE275u, 0x12F0u, 0x6C96u, 0xEE3Cu, 0x905Au, + 0xC78Au, 0xB9ECu, 0x3B46u, 0x4520u, 0xB5A5u, 0xCBC3u, 0x4969u, 0x370Fu, + 0x23D4u, 0x5DB2u, 0xDF18u, 0xA17Eu, 0x51FBu, 0x2F9Du, 0xAD37u, 0xD351u, + 0x0297u, 0x7CF1u, 0xFE5Bu, 0x803Du, 0x70B8u, 0x0EDEu, 0x8C74u, 0xF212u, + 0xE6C9u, 0x98AFu, 0x1A05u, 0x6463u, 0x94E6u, 0xEA80u, 0x682Au, 0x164Cu, + 0x419Cu, 0x3FFAu, 0xBD50u, 0xC336u, 0x33B3u, 0x4DD5u, 0xCF7Fu, 0xB119u, + 0xA5C2u, 0xDBA4u, 0x590Eu, 0x2768u, 0xD7EDu, 0xA98Bu, 0x2B21u, 0x5547u, + 0x031Au, 0x7D7Cu, 0xFFD6u, 0x81B0u, 0x7135u, 0x0F53u, 0x8DF9u, 0xF39Fu, + 0xE744u, 0x9922u, 0x1B88u, 0x65EEu, 0x956Bu, 0xEB0Du, 0x69A7u, 0x17C1u, + 0x4011u, 0x3E77u, 0xBCDDu, 0xC2BBu, 0x323Eu, 0x4C58u, 0xCEF2u, 0xB094u, + 0xA44Fu, 0xDA29u, 0x5883u, 0x26E5u, 0xD660u, 0xA806u, 0x2AACu, 0x54CAu, + 0x850Cu, 0xFB6Au, 0x79C0u, 0x07A6u, 0xF723u, 0x8945u, 0x0BEFu, 0x7589u, + 0x6152u, 0x1F34u, 0x9D9Eu, 0xE3F8u, 0x137Du, 0x6D1Bu, 0xEFB1u, 0x91D7u, + 0xC607u, 0xB861u, 0x3ACBu, 0x44ADu, 0xB428u, 0xCA4Eu, 0x48E4u, 0x3682u, + 0x2259u, 0x5C3Fu, 0xDE95u, 0xA0F3u, 0x5076u, 0x2E10u, 0xACBAu, 0xD2DCu + }, + { + 0x0000u, 0x82B5u, 0x8EDDu, 0x0C68u, 0x960Du, 0x14B8u, 0x18D0u, 0x9A65u, + 0xA7ADu, 0x2518u, 0x2970u, 0xABC5u, 0x31A0u, 0xB315u, 0xBF7Du, 0x3DC8u, + 0xC4EDu, 0x4658u, 0x4A30u, 0xC885u, 0x52E0u, 0xD055u, 0xDC3Du, 0x5E88u, + 0x6340u, 0xE1F5u, 0xED9Du, 0x6F28u, 0xF54Du, 0x77F8u, 0x7B90u, 0xF925u, + 0x026Du, 0x80D8u, 0x8CB0u, 0x0E05u, 0x9460u, 0x16D5u, 0x1ABDu, 0x9808u, + 0xA5C0u, 0x2775u, 0x2B1Du, 0xA9A8u, 0x33CDu, 0xB178u, 0xBD10u, 0x3FA5u, + 0xC680u, 0x4435u, 0x485Du, 0xCAE8u, 0x508Du, 0xD238u, 0xDE50u, 0x5CE5u, + 0x612Du, 0xE398u, 0xEFF0u, 0x6D45u, 0xF720u, 0x7595u, 0x79FDu, 0xFB48u, + 0x04DAu, 0x866Fu, 0x8A07u, 0x08B2u, 0x92D7u, 0x1062u, 0x1C0Au, 0x9EBFu, + 0xA377u, 0x21C2u, 0x2DAAu, 0xAF1Fu, 0x357Au, 0xB7CFu, 0xBBA7u, 0x3912u, + 0xC037u, 0x4282u, 0x4EEAu, 0xCC5Fu, 0x563Au, 0xD48Fu, 0xD8E7u, 0x5A52u, + 0x679Au, 0xE52Fu, 0xE947u, 0x6BF2u, 0xF197u, 0x7322u, 0x7F4Au, 0xFDFFu, + 0x06B7u, 0x8402u, 0x886Au, 0x0ADFu, 0x90BAu, 0x120Fu, 0x1E67u, 0x9CD2u, + 0xA11Au, 0x23AFu, 0x2FC7u, 0xAD72u, 0x3717u, 0xB5A2u, 0xB9CAu, 0x3B7Fu, + 0xC25Au, 0x40EFu, 0x4C87u, 0xCE32u, 0x5457u, 0xD6E2u, 0xDA8Au, 0x583Fu, + 0x65F7u, 0xE742u, 0xEB2Au, 0x699Fu, 0xF3FAu, 0x714Fu, 0x7D27u, 0xFF92u, + 0x09B4u, 0x8B01u, 0x8769u, 0x05DCu, 0x9FB9u, 0x1D0Cu, 0x1164u, 0x93D1u, + 0xAE19u, 0x2CACu, 0x20C4u, 0xA271u, 0x3814u, 0xBAA1u, 0xB6C9u, 0x347Cu, + 0xCD59u, 0x4FECu, 0x4384u, 0xC131u, 0x5B54u, 0xD9E1u, 0xD589u, 0x573Cu, + 0x6AF4u, 0xE841u, 0xE429u, 0x669Cu, 0xFCF9u, 0x7E4Cu, 0x7224u, 0xF091u, + 0x0BD9u, 0x896Cu, 0x8504u, 0x07B1u, 0x9DD4u, 0x1F61u, 0x1309u, 0x91BCu, + 0xAC74u, 0x2EC1u, 0x22A9u, 0xA01Cu, 0x3A79u, 0xB8CCu, 0xB4A4u, 0x3611u, + 0xCF34u, 0x4D81u, 0x41E9u, 0xC35Cu, 0x5939u, 0xDB8Cu, 0xD7E4u, 0x5551u, + 0x6899u, 0xEA2Cu, 0xE644u, 0x64F1u, 0xFE94u, 0x7C21u, 0x7049u, 0xF2FCu, + 0x0D6Eu, 0x8FDBu, 0x83B3u, 0x0106u, 0x9B63u, 0x19D6u, 0x15BEu, 0x970Bu, + 0xAAC3u, 0x2876u, 0x241Eu, 0xA6ABu, 0x3CCEu, 0xBE7Bu, 0xB213u, 0x30A6u, + 0xC983u, 0x4B36u, 0x475Eu, 0xC5EBu, 0x5F8Eu, 0xDD3Bu, 0xD153u, 0x53E6u, + 0x6E2Eu, 0xEC9Bu, 0xE0F3u, 0x6246u, 0xF823u, 0x7A96u, 0x76FEu, 0xF44Bu, + 0x0F03u, 0x8DB6u, 0x81DEu, 0x036Bu, 0x990Eu, 0x1BBBu, 0x17D3u, 0x9566u, + 0xA8AEu, 0x2A1Bu, 0x2673u, 0xA4C6u, 0x3EA3u, 0xBC16u, 0xB07Eu, 0x32CBu, + 0xCBEEu, 0x495Bu, 0x4533u, 0xC786u, 0x5DE3u, 0xDF56u, 0xD33Eu, 0x518Bu, + 0x6C43u, 0xEEF6u, 0xE29Eu, 0x602Bu, 0xFA4Eu, 0x78FBu, 0x7493u, 0xF626u + }, + { + 0x0000u, 0x1368u, 0x26D0u, 0x35B8u, 0x4DA0u, 0x5EC8u, 0x6B70u, 0x7818u, + 0x9B40u, 0x8828u, 0xBD90u, 0xAEF8u, 0xD6E0u, 0xC588u, 0xF030u, 0xE358u, + 0xBD37u, 0xAE5Fu, 0x9BE7u, 0x888Fu, 0xF097u, 0xE3FFu, 0xD647u, 0xC52Fu, + 0x2677u, 0x351Fu, 0x00A7u, 0x13CFu, 0x6BD7u, 0x78BFu, 0x4D07u, 0x5E6Fu, + 0xF1D9u, 0xE2B1u, 0xD709u, 0xC461u, 0xBC79u, 0xAF11u, 0x9AA9u, 0x89C1u, + 0x6A99u, 0x79F1u, 0x4C49u, 0x5F21u, 0x2739u, 0x3451u, 0x01E9u, 0x1281u, + 0x4CEEu, 0x5F86u, 0x6A3Eu, 0x7956u, 0x014Eu, 0x1226u, 0x279Eu, 0x34F6u, + 0xD7AEu, 0xC4C6u, 0xF17Eu, 0xE216u, 0x9A0Eu, 0x8966u, 0xBCDEu, 0xAFB6u, + 0x6805u, 0x7B6Du, 0x4ED5u, 0x5DBDu, 0x25A5u, 0x36CDu, 0x0375u, 0x101Du, + 0xF345u, 0xE02Du, 0xD595u, 0xC6FDu, 0xBEE5u, 0xAD8Du, 0x9835u, 0x8B5Du, + 0xD532u, 0xC65Au, 0xF3E2u, 0xE08Au, 0x9892u, 0x8BFAu, 0xBE42u, 0xAD2Au, + 0x4E72u, 0x5D1Au, 0x68A2u, 0x7BCAu, 0x03D2u, 0x10BAu, 0x2502u, 0x366Au, + 0x99DCu, 0x8AB4u, 0xBF0Cu, 0xAC64u, 0xD47Cu, 0xC714u, 0xF2ACu, 0xE1C4u, + 0x029Cu, 0x11F4u, 0x244Cu, 0x3724u, 0x4F3Cu, 0x5C54u, 0x69ECu, 0x7A84u, + 0x24EBu, 0x3783u, 0x023Bu, 0x1153u, 0x694Bu, 0x7A23u, 0x4F9Bu, 0x5CF3u, + 0xBFABu, 0xACC3u, 0x997Bu, 0x8A13u, 0xF20Bu, 0xE163u, 0xD4DBu, 0xC7B3u, + 0xD00Au, 0xC362u, 0xF6DAu, 0xE5B2u, 0x9DAAu, 0x8EC2u, 0xBB7Au, 0xA812u, + 0x4B4Au, 0x5822u, 0x6D9Au, 0x7EF2u, 0x06EAu, 0x1582u, 0x203Au, 0x3352u, + 0x6D3Du, 0x7E55u, 0x4BEDu, 0x5885u, 0x209Du, 0x33F5u, 0x064Du, 0x1525u, + 0xF67Du, 0xE515u, 0xD0ADu, 0xC3C5u, 0xBBDDu, 0xA8B5u, 0x9D0Du, 0x8E65u, + 0x21D3u, 0x32BBu, 0x0703u, 0x146Bu, 0x6C73u, 0x7F1Bu, 0x4AA3u, 0x59CBu, + 0xBA93u, 0xA9FBu, 0x9C43u, 0x8F2Bu, 0xF733u, 0xE45Bu, 0xD1E3u, 0xC28Bu, + 0x9CE4u, 0x8F8Cu, 0xBA34u, 0xA95Cu, 0xD144u, 0xC22Cu, 0xF794u, 0xE4FCu, + 0x07A4u, 0x14CCu, 0x2174u, 0x321Cu, 0x4A04u, 0x596Cu, 0x6CD4u, 0x7FBCu, + 0xB80Fu, 0xAB67u, 0x9EDFu, 0x8DB7u, 0xF5AFu, 0xE6C7u, 0xD37Fu, 0xC017u, + 0x234Fu, 0x3027u, 0x059Fu, 0x16F7u, 0x6EEFu, 0x7D87u, 0x483Fu, 0x5B57u, + 0x0538u, 0x1650u, 0x23E8u, 0x3080u, 0x4898u, 0x5BF0u, 0x6E48u, 0x7D20u, + 0x9E78u, 0x8D10u, 0xB8A8u, 0xABC0u, 0xD3D8u, 0xC0B0u, 0xF508u, 0xE660u, + 0x49D6u, 0x5ABEu, 0x6F06u, 0x7C6Eu, 0x0476u, 0x171Eu, 0x22A6u, 0x31CEu, + 0xD296u, 0xC1FEu, 0xF446u, 0xE72Eu, 0x9F36u, 0x8C5Eu, 0xB9E6u, 0xAA8Eu, + 0xF4E1u, 0xE789u, 0xD231u, 0xC159u, 0xB941u, 0xAA29u, 0x9F91u, 0x8CF9u, + 0x6FA1u, 0x7CC9u, 0x4971u, 0x5A19u, 0x2201u, 0x3169u, 0x04D1u, 0x17B9u + }, + { + 0x0000u, 0x2BA3u, 0x5746u, 0x7CE5u, 0xAE8Cu, 0x852Fu, 0xF9CAu, 0xD269u, + 0xD6AFu, 0xFD0Cu, 0x81E9u, 0xAA4Au, 0x7823u, 0x5380u, 0x2F65u, 0x04C6u, + 0x26E9u, 0x0D4Au, 0x71AFu, 0x5A0Cu, 0x8865u, 0xA3C6u, 0xDF23u, 0xF480u, + 0xF046u, 0xDBE5u, 0xA700u, 0x8CA3u, 0x5ECAu, 0x7569u, 0x098Cu, 0x222Fu, + 0x4DD2u, 0x6671u, 0x1A94u, 0x3137u, 0xE35Eu, 0xC8FDu, 0xB418u, 0x9FBBu, + 0x9B7Du, 0xB0DEu, 0xCC3Bu, 0xE798u, 0x35F1u, 0x1E52u, 0x62B7u, 0x4914u, + 0x6B3Bu, 0x4098u, 0x3C7Du, 0x17DEu, 0xC5B7u, 0xEE14u, 0x92F1u, 0xB952u, + 0xBD94u, 0x9637u, 0xEAD2u, 0xC171u, 0x1318u, 0x38BBu, 0x445Eu, 0x6FFDu, + 0x9BA4u, 0xB007u, 0xCCE2u, 0xE741u, 0x3528u, 0x1E8Bu, 0x626Eu, 0x49CDu, + 0x4D0Bu, 0x66A8u, 0x1A4Du, 0x31EEu, 0xE387u, 0xC824u, 0xB4C1u, 0x9F62u, + 0xBD4Du, 0x96EEu, 0xEA0Bu, 0xC1A8u, 0x13C1u, 0x3862u, 0x4487u, 0x6F24u, + 0x6BE2u, 0x4041u, 0x3CA4u, 0x1707u, 0xC56Eu, 0xEECDu, 0x9228u, 0xB98Bu, + 0xD676u, 0xFDD5u, 0x8130u, 0xAA93u, 0x78FAu, 0x5359u, 0x2FBCu, 0x041Fu, + 0x00D9u, 0x2B7Au, 0x579Fu, 0x7C3Cu, 0xAE55u, 0x85F6u, 0xF913u, 0xD2B0u, + 0xF09Fu, 0xDB3Cu, 0xA7D9u, 0x8C7Au, 0x5E13u, 0x75B0u, 0x0955u, 0x22F6u, + 0x2630u, 0x0D93u, 0x7176u, 0x5AD5u, 0x88BCu, 0xA31Fu, 0xDFFAu, 0xF459u, + 0xBCFFu, 0x975Cu, 0xEBB9u, 0xC01Au, 0x1273u, 0x39D0u, 0x4535u, 0x6E96u, + 0x6A50u, 0x41F3u, 0x3D16u, 0x16B5u, 0xC4DCu, 0xEF7Fu, 0x939Au, 0xB839u, + 0x9A16u, 0xB1B5u, 0xCD50u, 0xE6F3u, 0x349Au, 0x1F39u, 0x63DCu, 0x487Fu, + 0x4CB9u, 0x671Au, 0x1BFFu, 0x305Cu, 0xE235u, 0xC996u, 0xB573u, 0x9ED0u, + 0xF12Du, 0xDA8Eu, 0xA66Bu, 0x8DC8u, 0x5FA1u, 0x7402u, 0x08E7u, 0x2344u, + 0x2782u, 0x0C21u, 0x70C4u, 0x5B67u, 0x890Eu, 0xA2ADu, 0xDE48u, 0xF5EBu, + 0xD7C4u, 0xFC67u, 0x8082u, 0xAB21u, 0x7948u, 0x52EBu, 0x2E0Eu, 0x05ADu, + 0x016Bu, 0x2AC8u, 0x562Du, 0x7D8Eu, 0xAFE7u, 0x8444u, 0xF8A1u, 0xD302u, + 0x275Bu, 0x0CF8u, 0x701Du, 0x5BBEu, 0x89D7u, 0xA274u, 0xDE91u, 0xF532u, + 0xF1F4u, 0xDA57u, 0xA6B2u, 0x8D11u, 0x5F78u, 0x74DBu, 0x083Eu, 0x239Du, + 0x01B2u, 0x2A11u, 0x56F4u, 0x7D57u, 0xAF3Eu, 0x849Du, 0xF878u, 0xD3DBu, + 0xD71Du, 0xFCBEu, 0x805Bu, 0xABF8u, 0x7991u, 0x5232u, 0x2ED7u, 0x0574u, + 0x6A89u, 0x412Au, 0x3DCFu, 0x166Cu, 0xC405u, 0xEFA6u, 0x9343u, 0xB8E0u, + 0xBC26u, 0x9785u, 0xEB60u, 0xC0C3u, 0x12AAu, 0x3909u, 0x45ECu, 0x6E4Fu, + 0x4C60u, 0x67C3u, 0x1B26u, 0x3085u, 0xE2ECu, 0xC94Fu, 0xB5AAu, 0x9E09u, + 0x9ACFu, 0xB16Cu, 0xCD89u, 0xE62Au, 0x3443u, 0x1FE0u, 0x6305u, 0x48A6u + }, + { + 0x0000u, 0xF249u, 0x6F25u, 0x9D6Cu, 0xDE4Au, 0x2C03u, 0xB16Fu, 0x4326u, + 0x3723u, 0xC56Au, 0x5806u, 0xAA4Fu, 0xE969u, 0x1B20u, 0x864Cu, 0x7405u, + 0x6E46u, 0x9C0Fu, 0x0163u, 0xF32Au, 0xB00Cu, 0x4245u, 0xDF29u, 0x2D60u, + 0x5965u, 0xAB2Cu, 0x3640u, 0xC409u, 0x872Fu, 0x7566u, 0xE80Au, 0x1A43u, + 0xDC8Cu, 0x2EC5u, 0xB3A9u, 0x41E0u, 0x02C6u, 0xF08Fu, 0x6DE3u, 0x9FAAu, + 0xEBAFu, 0x19E6u, 0x848Au, 0x76C3u, 0x35E5u, 0xC7ACu, 0x5AC0u, 0xA889u, + 0xB2CAu, 0x4083u, 0xDDEFu, 0x2FA6u, 0x6C80u, 0x9EC9u, 0x03A5u, 0xF1ECu, + 0x85E9u, 0x77A0u, 0xEACCu, 0x1885u, 0x5BA3u, 0xA9EAu, 0x3486u, 0xC6CFu, + 0x32AFu, 0xC0E6u, 0x5D8Au, 0xAFC3u, 0xECE5u, 0x1EACu, 0x83C0u, 0x7189u, + 0x058Cu, 0xF7C5u, 0x6AA9u, 0x98E0u, 0xDBC6u, 0x298Fu, 0xB4E3u, 0x46AAu, + 0x5CE9u, 0xAEA0u, 0x33CCu, 0xC185u, 0x82A3u, 0x70EAu, 0xED86u, 0x1FCFu, + 0x6BCAu, 0x9983u, 0x04EFu, 0xF6A6u, 0xB580u, 0x47C9u, 0xDAA5u, 0x28ECu, + 0xEE23u, 0x1C6Au, 0x8106u, 0x734Fu, 0x3069u, 0xC220u, 0x5F4Cu, 0xAD05u, + 0xD900u, 0x2B49u, 0xB625u, 0x446Cu, 0x074Au, 0xF503u, 0x686Fu, 0x9A26u, + 0x8065u, 0x722Cu, 0xEF40u, 0x1D09u, 0x5E2Fu, 0xAC66u, 0x310Au, 0xC343u, + 0xB746u, 0x450Fu, 0xD863u, 0x2A2Au, 0x690Cu, 0x9B45u, 0x0629u, 0xF460u, + 0x655Eu, 0x9717u, 0x0A7Bu, 0xF832u, 0xBB14u, 0x495Du, 0xD431u, 0x2678u, + 0x527Du, 0xA034u, 0x3D58u, 0xCF11u, 0x8C37u, 0x7E7Eu, 0xE312u, 0x115Bu, + 0x0B18u, 0xF951u, 0x643Du, 0x9674u, 0xD552u, 0x271Bu, 0xBA77u, 0x483Eu, + 0x3C3Bu, 0xCE72u, 0x531Eu, 0xA157u, 0xE271u, 0x1038u, 0x8D54u, 0x7F1Du, + 0xB9D2u, 0x4B9Bu, 0xD6F7u, 0x24BEu, 0x6798u, 0x95D1u, 0x08BDu, 0xFAF4u, + 0x8EF1u, 0x7CB8u, 0xE1D4u, 0x139Du, 0x50BBu, 0xA2F2u, 0x3F9Eu, 0xCDD7u, + 0xD794u, 0x25DDu, 0xB8B1u, 0x4AF8u, 0x09DEu, 0xFB97u, 0x66FBu, 0x94B2u, + 0xE0B7u, 0x12FEu, 0x8F92u, 0x7DDBu, 0x3EFDu, 0xCCB4u, 0x51D8u, 0xA391u, + 0x57F1u, 0xA5B8u, 0x38D4u, 0xCA9Du, 0x89BBu, 0x7BF2u, 0xE69Eu, 0x14D7u, + 0x60D2u, 0x929Bu, 0x0FF7u, 0xFDBEu, 0xBE98u, 0x4CD1u, 0xD1BDu, 0x23F4u, + 0x39B7u, 0xCBFEu, 0x5692u, 0xA4DBu, 0xE7FDu, 0x15B4u, 0x88D8u, 0x7A91u, + 0x0E94u, 0xFCDDu, 0x61B1u, 0x93F8u, 0xD0DEu, 0x2297u, 0xBFFBu, 0x4DB2u, + 0x8B7Du, 0x7934u, 0xE458u, 0x1611u, 0x5537u, 0xA77Eu, 0x3A12u, 0xC85Bu, + 0xBC5Eu, 0x4E17u, 0xD37Bu, 0x2132u, 0x6214u, 0x905Du, 0x0D31u, 0xFF78u, + 0xE53Bu, 0x1772u, 0x8A1Eu, 0x7857u, 0x3B71u, 0xC938u, 0x5454u, 0xA61Du, + 0xD218u, 0x2051u, 0xBD3Du, 0x4F74u, 0x0C52u, 0xFE1Bu, 0x6377u, 0x913Eu + }, + { + 0x0000u, 0xCABCu, 0x1ECFu, 0xD473u, 0x3D9Eu, 0xF722u, 0x2351u, 0xE9EDu, + 0x7B3Cu, 0xB180u, 0x65F3u, 0xAF4Fu, 0x46A2u, 0x8C1Eu, 0x586Du, 0x92D1u, + 0xF678u, 0x3CC4u, 0xE8B7u, 0x220Bu, 0xCBE6u, 0x015Au, 0xD529u, 0x1F95u, + 0x8D44u, 0x47F8u, 0x938Bu, 0x5937u, 0xB0DAu, 0x7A66u, 0xAE15u, 0x64A9u, + 0x6747u, 0xADFBu, 0x7988u, 0xB334u, 0x5AD9u, 0x9065u, 0x4416u, 0x8EAAu, + 0x1C7Bu, 0xD6C7u, 0x02B4u, 0xC808u, 0x21E5u, 0xEB59u, 0x3F2Au, 0xF596u, + 0x913Fu, 0x5B83u, 0x8FF0u, 0x454Cu, 0xACA1u, 0x661Du, 0xB26Eu, 0x78D2u, + 0xEA03u, 0x20BFu, 0xF4CCu, 0x3E70u, 0xD79Du, 0x1D21u, 0xC952u, 0x03EEu, + 0xCE8Eu, 0x0432u, 0xD041u, 0x1AFDu, 0xF310u, 0x39ACu, 0xEDDFu, 0x2763u, + 0xB5B2u, 0x7F0Eu, 0xAB7Du, 0x61C1u, 0x882Cu, 0x4290u, 0x96E3u, 0x5C5Fu, + 0x38F6u, 0xF24Au, 0x2639u, 0xEC85u, 0x0568u, 0xCFD4u, 0x1BA7u, 0xD11Bu, + 0x43CAu, 0x8976u, 0x5D05u, 0x97B9u, 0x7E54u, 0xB4E8u, 0x609Bu, 0xAA27u, + 0xA9C9u, 0x6375u, 0xB706u, 0x7DBAu, 0x9457u, 0x5EEBu, 0x8A98u, 0x4024u, + 0xD2F5u, 0x1849u, 0xCC3Au, 0x0686u, 0xEF6Bu, 0x25D7u, 0xF1A4u, 0x3B18u, + 0x5FB1u, 0x950Du, 0x417Eu, 0x8BC2u, 0x622Fu, 0xA893u, 0x7CE0u, 0xB65Cu, + 0x248Du, 0xEE31u, 0x3A42u, 0xF0FEu, 0x1913u, 0xD3AFu, 0x07DCu, 0xCD60u, + 0x16ABu, 0xDC17u, 0x0864u, 0xC2D8u, 0x2B35u, 0xE189u, 0x35FAu, 0xFF46u, + 0x6D97u, 0xA72Bu, 0x7358u, 0xB9E4u, 0x5009u, 0x9AB5u, 0x4EC6u, 0x847Au, + 0xE0D3u, 0x2A6Fu, 0xFE1Cu, 0x34A0u, 0xDD4Du, 0x17F1u, 0xC382u, 0x093Eu, + 0x9BEFu, 0x5153u, 0x8520u, 0x4F9Cu, 0xA671u, 0x6CCDu, 0xB8BEu, 0x7202u, + 0x71ECu, 0xBB50u, 0x6F23u, 0xA59Fu, 0x4C72u, 0x86CEu, 0x52BDu, 0x9801u, + 0x0AD0u, 0xC06Cu, 0x141Fu, 0xDEA3u, 0x374Eu, 0xFDF2u, 0x2981u, 0xE33Du, + 0x8794u, 0x4D28u, 0x995Bu, 0x53E7u, 0xBA0Au, 0x70B6u, 0xA4C5u, 0x6E79u, + 0xFCA8u, 0x3614u, 0xE267u, 0x28DBu, 0xC136u, 0x0B8Au, 0xDFF9u, 0x1545u, + 0xD825u, 0x1299u, 0xC6EAu, 0x0C56u, 0xE5BBu, 0x2F07u, 0xFB74u, 0x31C8u, + 0xA319u, 0x69A5u, 0xBDD6u, 0x776Au, 0x9E87u, 0x543Bu, 0x8048u, 0x4AF4u, + 0x2E5Du, 0xE4E1u, 0x3092u, 0xFA2Eu, 0x13C3u, 0xD97Fu, 0x0D0Cu, 0xC7B0u, + 0x5561u, 0x9FDDu, 0x4BAEu, 0x8112u, 0x68FFu, 0xA243u, 0x7630u, 0xBC8Cu, + 0xBF62u, 0x75DEu, 0xA1ADu, 0x6B11u, 0x82FCu, 0x4840u, 0x9C33u, 0x568Fu, + 0xC45Eu, 0x0EE2u, 0xDA91u, 0x102Du, 0xF9C0u, 0x337Cu, 0xE70Fu, 0x2DB3u, + 0x491Au, 0x83A6u, 0x57D5u, 0x9D69u, 0x7484u, 0xBE38u, 0x6A4Bu, 0xA0F7u, + 0x3226u, 0xF89Au, 0x2CE9u, 0xE655u, 0x0FB8u, 0xC504u, 0x1177u, 0xDBCBu + }, + { + 0x0000u, 0x2D56u, 0x5AACu, 0x77FAu, 0xB558u, 0x980Eu, 0xEFF4u, 0xC2A2u, + 0xE107u, 0xCC51u, 0xBBABu, 0x96FDu, 0x545Fu, 0x7909u, 0x0EF3u, 0x23A5u, + 0x49B9u, 0x64EFu, 0x1315u, 0x3E43u, 0xFCE1u, 0xD1B7u, 0xA64Du, 0x8B1Bu, + 0xA8BEu, 0x85E8u, 0xF212u, 0xDF44u, 0x1DE6u, 0x30B0u, 0x474Au, 0x6A1Cu, + 0x9372u, 0xBE24u, 0xC9DEu, 0xE488u, 0x262Au, 0x0B7Cu, 0x7C86u, 0x51D0u, + 0x7275u, 0x5F23u, 0x28D9u, 0x058Fu, 0xC72Du, 0xEA7Bu, 0x9D81u, 0xB0D7u, + 0xDACBu, 0xF79Du, 0x8067u, 0xAD31u, 0x6F93u, 0x42C5u, 0x353Fu, 0x1869u, + 0x3BCCu, 0x169Au, 0x6160u, 0x4C36u, 0x8E94u, 0xA3C2u, 0xD438u, 0xF96Eu, + 0xAD53u, 0x8005u, 0xF7FFu, 0xDAA9u, 0x180Bu, 0x355Du, 0x42A7u, 0x6FF1u, + 0x4C54u, 0x6102u, 0x16F8u, 0x3BAEu, 0xF90Cu, 0xD45Au, 0xA3A0u, 0x8EF6u, + 0xE4EAu, 0xC9BCu, 0xBE46u, 0x9310u, 0x51B2u, 0x7CE4u, 0x0B1Eu, 0x2648u, + 0x05EDu, 0x28BBu, 0x5F41u, 0x7217u, 0xB0B5u, 0x9DE3u, 0xEA19u, 0xC74Fu, + 0x3E21u, 0x1377u, 0x648Du, 0x49DBu, 0x8B79u, 0xA62Fu, 0xD1D5u, 0xFC83u, + 0xDF26u, 0xF270u, 0x858Au, 0xA8DCu, 0x6A7Eu, 0x4728u, 0x30D2u, 0x1D84u, + 0x7798u, 0x5ACEu, 0x2D34u, 0x0062u, 0xC2C0u, 0xEF96u, 0x986Cu, 0xB53Au, + 0x969Fu, 0xBBC9u, 0xCC33u, 0xE165u, 0x23C7u, 0x0E91u, 0x796Bu, 0x543Du, + 0xD111u, 0xFC47u, 0x8BBDu, 0xA6EBu, 0x6449u, 0x491Fu, 0x3EE5u, 0x13B3u, + 0x3016u, 0x1D40u, 0x6ABAu, 0x47ECu, 0x854Eu, 0xA818u, 0xDFE2u, 0xF2B4u, + 0x98A8u, 0xB5FEu, 0xC204u, 0xEF52u, 0x2DF0u, 0x00A6u, 0x775Cu, 0x5A0Au, + 0x79AFu, 0x54F9u, 0x2303u, 0x0E55u, 0xCCF7u, 0xE1A1u, 0x965Bu, 0xBB0Du, + 0x4263u, 0x6F35u, 0x18CFu, 0x3599u, 0xF73Bu, 0xDA6Du, 0xAD97u, 0x80C1u, + 0xA364u, 0x8E32u, 0xF9C8u, 0xD49Eu, 0x163Cu, 0x3B6Au, 0x4C90u, 0x61C6u, + 0x0BDAu, 0x268Cu, 0x5176u, 0x7C20u, 0xBE82u, 0x93D4u, 0xE42Eu, 0xC978u, + 0xEADDu, 0xC78Bu, 0xB071u, 0x9D27u, 0x5F85u, 0x72D3u, 0x0529u, 0x287Fu, + 0x7C42u, 0x5114u, 0x26EEu, 0x0BB8u, 0xC91Au, 0xE44Cu, 0x93B6u, 0xBEE0u, + 0x9D45u, 0xB013u, 0xC7E9u, 0xEABFu, 0x281Du, 0x054Bu, 0x72B1u, 0x5FE7u, + 0x35FBu, 0x18ADu, 0x6F57u, 0x4201u, 0x80A3u, 0xADF5u, 0xDA0Fu, 0xF759u, + 0xD4FCu, 0xF9AAu, 0x8E50u, 0xA306u, 0x61A4u, 0x4CF2u, 0x3B08u, 0x165Eu, + 0xEF30u, 0xC266u, 0xB59Cu, 0x98CAu, 0x5A68u, 0x773Eu, 0x00C4u, 0x2D92u, + 0x0E37u, 0x2361u, 0x549Bu, 0x79CDu, 0xBB6Fu, 0x9639u, 0xE1C3u, 0xCC95u, + 0xA689u, 0x8BDFu, 0xFC25u, 0xD173u, 0x13D1u, 0x3E87u, 0x497Du, 0x642Bu, + 0x478Eu, 0x6AD8u, 0x1D22u, 0x3074u, 0xF2D6u, 0xDF80u, 0xA87Au, 0x852Cu + }, + { + 0x0000u, 0x2995u, 0x532Au, 0x7ABFu, 0xA654u, 0x8FC1u, 0xF57Eu, 0xDCEBu, + 0xC71Fu, 0xEE8Au, 0x9435u, 0xBDA0u, 0x614Bu, 0x48DEu, 0x3261u, 0x1BF4u, + 0x0589u, 0x2C1Cu, 0x56A3u, 0x7F36u, 0xA3DDu, 0x8A48u, 0xF0F7u, 0xD962u, + 0xC296u, 0xEB03u, 0x91BCu, 0xB829u, 0x64C2u, 0x4D57u, 0x37E8u, 0x1E7Du, + 0x0B12u, 0x2287u, 0x5838u, 0x71ADu, 0xAD46u, 0x84D3u, 0xFE6Cu, 0xD7F9u, + 0xCC0Du, 0xE598u, 0x9F27u, 0xB6B2u, 0x6A59u, 0x43CCu, 0x3973u, 0x10E6u, + 0x0E9Bu, 0x270Eu, 0x5DB1u, 0x7424u, 0xA8CFu, 0x815Au, 0xFBE5u, 0xD270u, + 0xC984u, 0xE011u, 0x9AAEu, 0xB33Bu, 0x6FD0u, 0x4645u, 0x3CFAu, 0x156Fu, + 0x1624u, 0x3FB1u, 0x450Eu, 0x6C9Bu, 0xB070u, 0x99E5u, 0xE35Au, 0xCACFu, + 0xD13Bu, 0xF8AEu, 0x8211u, 0xAB84u, 0x776Fu, 0x5EFAu, 0x2445u, 0x0DD0u, + 0x13ADu, 0x3A38u, 0x4087u, 0x6912u, 0xB5F9u, 0x9C6Cu, 0xE6D3u, 0xCF46u, + 0xD4B2u, 0xFD27u, 0x8798u, 0xAE0Du, 0x72E6u, 0x5B73u, 0x21CCu, 0x0859u, + 0x1D36u, 0x34A3u, 0x4E1Cu, 0x6789u, 0xBB62u, 0x92F7u, 0xE848u, 0xC1DDu, + 0xDA29u, 0xF3BCu, 0x8903u, 0xA096u, 0x7C7Du, 0x55E8u, 0x2F57u, 0x06C2u, + 0x18BFu, 0x312Au, 0x4B95u, 0x6200u, 0xBEEBu, 0x977Eu, 0xEDC1u, 0xC454u, + 0xDFA0u, 0xF635u, 0x8C8Au, 0xA51Fu, 0x79F4u, 0x5061u, 0x2ADEu, 0x034Bu, + 0x2C48u, 0x05DDu, 0x7F62u, 0x56F7u, 0x8A1Cu, 0xA389u, 0xD936u, 0xF0A3u, + 0xEB57u, 0xC2C2u, 0xB87Du, 0x91E8u, 0x4D03u, 0x6496u, 0x1E29u, 0x37BCu, + 0x29C1u, 0x0054u, 0x7AEBu, 0x537Eu, 0x8F95u, 0xA600u, 0xDCBFu, 0xF52Au, + 0xEEDEu, 0xC74Bu, 0xBDF4u, 0x9461u, 0x488Au, 0x611Fu, 0x1BA0u, 0x3235u, + 0x275Au, 0x0ECFu, 0x7470u, 0x5DE5u, 0x810Eu, 0xA89Bu, 0xD224u, 0xFBB1u, + 0xE045u, 0xC9D0u, 0xB36Fu, 0x9AFAu, 0x4611u, 0x6F84u, 0x153Bu, 0x3CAEu, + 0x22D3u, 0x0B46u, 0x71F9u, 0x586Cu, 0x8487u, 0xAD12u, 0xD7ADu, 0xFE38u, + 0xE5CCu, 0xCC59u, 0xB6E6u, 0x9F73u, 0x4398u, 0x6A0Du, 0x10B2u, 0x3927u, + 0x3A6Cu, 0x13F9u, 0x6946u, 0x40D3u, 0x9C38u, 0xB5ADu, 0xCF12u, 0xE687u, + 0xFD73u, 0xD4E6u, 0xAE59u, 0x87CCu, 0x5B27u, 0x72B2u, 0x080Du, 0x2198u, + 0x3FE5u, 0x1670u, 0x6CCFu, 0x455Au, 0x99B1u, 0xB024u, 0xCA9Bu, 0xE30Eu, + 0xF8FAu, 0xD16Fu, 0xABD0u, 0x8245u, 0x5EAEu, 0x773Bu, 0x0D84u, 0x2411u, + 0x317Eu, 0x18EBu, 0x6254u, 0x4BC1u, 0x972Au, 0xBEBFu, 0xC400u, 0xED95u, + 0xF661u, 0xDFF4u, 0xA54Bu, 0x8CDEu, 0x5035u, 0x79A0u, 0x031Fu, 0x2A8Au, + 0x34F7u, 0x1D62u, 0x67DDu, 0x4E48u, 0x92A3u, 0xBB36u, 0xC189u, 0xE81Cu, + 0xF3E8u, 0xDA7Du, 0xA0C2u, 0x8957u, 0x55BCu, 0x7C29u, 0x0696u, 0x2F03u + }, + { + 0x0000u, 0x5890u, 0xB120u, 0xE9B0u, 0xE9F7u, 0xB167u, 0x58D7u, 0x0047u, + 0x5859u, 0x00C9u, 0xE979u, 0xB1E9u, 0xB1AEu, 0xE93Eu, 0x008Eu, 0x581Eu, + 0xB0B2u, 0xE822u, 0x0192u, 0x5902u, 0x5945u, 0x01D5u, 0xE865u, 0xB0F5u, + 0xE8EBu, 0xB07Bu, 0x59CBu, 0x015Bu, 0x011Cu, 0x598Cu, 0xB03Cu, 0xE8ACu, + 0xEAD3u, 0xB243u, 0x5BF3u, 0x0363u, 0x0324u, 0x5BB4u, 0xB204u, 0xEA94u, + 0xB28Au, 0xEA1Au, 0x03AAu, 0x5B3Au, 0x5B7Du, 0x03EDu, 0xEA5Du, 0xB2CDu, + 0x5A61u, 0x02F1u, 0xEB41u, 0xB3D1u, 0xB396u, 0xEB06u, 0x02B6u, 0x5A26u, + 0x0238u, 0x5AA8u, 0xB318u, 0xEB88u, 0xEBCFu, 0xB35Fu, 0x5AEFu, 0x027Fu, + 0x5E11u, 0x0681u, 0xEF31u, 0xB7A1u, 0xB7E6u, 0xEF76u, 0x06C6u, 0x5E56u, + 0x0648u, 0x5ED8u, 0xB768u, 0xEFF8u, 0xEFBFu, 0xB72Fu, 0x5E9Fu, 0x060Fu, + 0xEEA3u, 0xB633u, 0x5F83u, 0x0713u, 0x0754u, 0x5FC4u, 0xB674u, 0xEEE4u, + 0xB6FAu, 0xEE6Au, 0x07DAu, 0x5F4Au, 0x5F0Du, 0x079Du, 0xEE2Du, 0xB6BDu, + 0xB4C2u, 0xEC52u, 0x05E2u, 0x5D72u, 0x5D35u, 0x05A5u, 0xEC15u, 0xB485u, + 0xEC9Bu, 0xB40Bu, 0x5DBBu, 0x052Bu, 0x056Cu, 0x5DFCu, 0xB44Cu, 0xECDCu, + 0x0470u, 0x5CE0u, 0xB550u, 0xEDC0u, 0xED87u, 0xB517u, 0x5CA7u, 0x0437u, + 0x5C29u, 0x04B9u, 0xED09u, 0xB599u, 0xB5DEu, 0xED4Eu, 0x04FEu, 0x5C6Eu, + 0xBC22u, 0xE4B2u, 0x0D02u, 0x5592u, 0x55D5u, 0x0D45u, 0xE4F5u, 0xBC65u, + 0xE47Bu, 0xBCEBu, 0x555Bu, 0x0DCBu, 0x0D8Cu, 0x551Cu, 0xBCACu, 0xE43Cu, + 0x0C90u, 0x5400u, 0xBDB0u, 0xE520u, 0xE567u, 0xBDF7u, 0x5447u, 0x0CD7u, + 0x54C9u, 0x0C59u, 0xE5E9u, 0xBD79u, 0xBD3Eu, 0xE5AEu, 0x0C1Eu, 0x548Eu, + 0x56F1u, 0x0E61u, 0xE7D1u, 0xBF41u, 0xBF06u, 0xE796u, 0x0E26u, 0x56B6u, + 0x0EA8u, 0x5638u, 0xBF88u, 0xE718u, 0xE75Fu, 0xBFCFu, 0x567Fu, 0x0EEFu, + 0xE643u, 0xBED3u, 0x5763u, 0x0FF3u, 0x0FB4u, 0x5724u, 0xBE94u, 0xE604u, + 0xBE1Au, 0xE68Au, 0x0F3Au, 0x57AAu, 0x57EDu, 0x0F7Du, 0xE6CDu, 0xBE5Du, + 0xE233u, 0xBAA3u, 0x5313u, 0x0B83u, 0x0BC4u, 0x5354u, 0xBAE4u, 0xE274u, + 0xBA6Au, 0xE2FAu, 0x0B4Au, 0x53DAu, 0x539Du, 0x0B0Du, 0xE2BDu, 0xBA2Du, + 0x5281u, 0x0A11u, 0xE3A1u, 0xBB31u, 0xBB76u, 0xE3E6u, 0x0A56u, 0x52C6u, + 0x0AD8u, 0x5248u, 0xBBF8u, 0xE368u, 0xE32Fu, 0xBBBFu, 0x520Fu, 0x0A9Fu, + 0x08E0u, 0x5070u, 0xB9C0u, 0xE150u, 0xE117u, 0xB987u, 0x5037u, 0x08A7u, + 0x50B9u, 0x0829u, 0xE199u, 0xB909u, 0xB94Eu, 0xE1DEu, 0x086Eu, 0x50FEu, + 0xB852u, 0xE0C2u, 0x0972u, 0x51E2u, 0x51A5u, 0x0935u, 0xE085u, 0xB815u, + 0xE00Bu, 0xB89Bu, 0x512Bu, 0x09BBu, 0x09FCu, 0x516Cu, 0xB8DCu, 0xE04Cu + }, + { + 0x0000u, 0xF3F3u, 0x6C51u, 0x9FA2u, 0xD8A2u, 0x2B51u, 0xB4F3u, 0x4700u, + 0x3AF3u, 0xC900u, 0x56A2u, 0xA551u, 0xE251u, 0x11A2u, 0x8E00u, 0x7DF3u, + 0x75E6u, 0x8615u, 0x19B7u, 0xEA44u, 0xAD44u, 0x5EB7u, 0xC115u, 0x32E6u, + 0x4F15u, 0xBCE6u, 0x2344u, 0xD0B7u, 0x97B7u, 0x6444u, 0xFBE6u, 0x0815u, + 0xEBCCu, 0x183Fu, 0x879Du, 0x746Eu, 0x336Eu, 0xC09Du, 0x5F3Fu, 0xACCCu, + 0xD13Fu, 0x22CCu, 0xBD6Eu, 0x4E9Du, 0x099Du, 0xFA6Eu, 0x65CCu, 0x963Fu, + 0x9E2Au, 0x6DD9u, 0xF27Bu, 0x0188u, 0x4688u, 0xB57Bu, 0x2AD9u, 0xD92Au, + 0xA4D9u, 0x572Au, 0xC888u, 0x3B7Bu, 0x7C7Bu, 0x8F88u, 0x102Au, 0xE3D9u, + 0x5C2Fu, 0xAFDCu, 0x307Eu, 0xC38Du, 0x848Du, 0x777Eu, 0xE8DCu, 0x1B2Fu, + 0x66DCu, 0x952Fu, 0x0A8Du, 0xF97Eu, 0xBE7Eu, 0x4D8Du, 0xD22Fu, 0x21DCu, + 0x29C9u, 0xDA3Au, 0x4598u, 0xB66Bu, 0xF16Bu, 0x0298u, 0x9D3Au, 0x6EC9u, + 0x133Au, 0xE0C9u, 0x7F6Bu, 0x8C98u, 0xCB98u, 0x386Bu, 0xA7C9u, 0x543Au, + 0xB7E3u, 0x4410u, 0xDBB2u, 0x2841u, 0x6F41u, 0x9CB2u, 0x0310u, 0xF0E3u, + 0x8D10u, 0x7EE3u, 0xE141u, 0x12B2u, 0x55B2u, 0xA641u, 0x39E3u, 0xCA10u, + 0xC205u, 0x31F6u, 0xAE54u, 0x5DA7u, 0x1AA7u, 0xE954u, 0x76F6u, 0x8505u, + 0xF8F6u, 0x0B05u, 0x94A7u, 0x6754u, 0x2054u, 0xD3A7u, 0x4C05u, 0xBFF6u, + 0xB85Eu, 0x4BADu, 0xD40Fu, 0x27FCu, 0x60FCu, 0x930Fu, 0x0CADu, 0xFF5Eu, + 0x82ADu, 0x715Eu, 0xEEFCu, 0x1D0Fu, 0x5A0Fu, 0xA9FCu, 0x365Eu, 0xC5ADu, + 0xCDB8u, 0x3E4Bu, 0xA1E9u, 0x521Au, 0x151Au, 0xE6E9u, 0x794Bu, 0x8AB8u, + 0xF74Bu, 0x04B8u, 0x9B1Au, 0x68E9u, 0x2FE9u, 0xDC1Au, 0x43B8u, 0xB04Bu, + 0x5392u, 0xA061u, 0x3FC3u, 0xCC30u, 0x8B30u, 0x78C3u, 0xE761u, 0x1492u, + 0x6961u, 0x9A92u, 0x0530u, 0xF6C3u, 0xB1C3u, 0x4230u, 0xDD92u, 0x2E61u, + 0x2674u, 0xD587u, 0x4A25u, 0xB9D6u, 0xFED6u, 0x0D25u, 0x9287u, 0x6174u, + 0x1C87u, 0xEF74u, 0x70D6u, 0x8325u, 0xC425u, 0x37D6u, 0xA874u, 0x5B87u, + 0xE471u, 0x1782u, 0x8820u, 0x7BD3u, 0x3CD3u, 0xCF20u, 0x5082u, 0xA371u, + 0xDE82u, 0x2D71u, 0xB2D3u, 0x4120u, 0x0620u, 0xF5D3u, 0x6A71u, 0x9982u, + 0x9197u, 0x6264u, 0xFDC6u, 0x0E35u, 0x4935u, 0xBAC6u, 0x2564u, 0xD697u, + 0xAB64u, 0x5897u, 0xC735u, 0x34C6u, 0x73C6u, 0x8035u, 0x1F97u, 0xEC64u, + 0x0FBDu, 0xFC4Eu, 0x63ECu, 0x901Fu, 0xD71Fu, 0x24ECu, 0xBB4Eu, 0x48BDu, + 0x354Eu, 0xC6BDu, 0x591Fu, 0xAAECu, 0xEDECu, 0x1E1Fu, 0x81BDu, 0x724Eu, + 0x7A5Bu, 0x89A8u, 0x160Au, 0xE5F9u, 0xA2F9u, 0x510Au, 0xCEA8u, 0x3D5Bu, + 0x40A8u, 0xB35Bu, 0x2CF9u, 0xDF0Au, 0x980Au, 0x6BF9u, 0xF45Bu, 0x07A8u + }, + { + 0x0000u, 0xFB0Bu, 0x7DA1u, 0x86AAu, 0xFB42u, 0x0049u, 0x86E3u, 0x7DE8u, + 0x7D33u, 0x8638u, 0x0092u, 0xFB99u, 0x8671u, 0x7D7Au, 0xFBD0u, 0x00DBu, + 0xFA66u, 0x016Du, 0x87C7u, 0x7CCCu, 0x0124u, 0xFA2Fu, 0x7C85u, 0x878Eu, + 0x8755u, 0x7C5Eu, 0xFAF4u, 0x01FFu, 0x7C17u, 0x871Cu, 0x01B6u, 0xFABDu, + 0x7F7Bu, 0x8470u, 0x02DAu, 0xF9D1u, 0x8439u, 0x7F32u, 0xF998u, 0x0293u, + 0x0248u, 0xF943u, 0x7FE9u, 0x84E2u, 0xF90Au, 0x0201u, 0x84ABu, 0x7FA0u, + 0x851Du, 0x7E16u, 0xF8BCu, 0x03B7u, 0x7E5Fu, 0x8554u, 0x03FEu, 0xF8F5u, + 0xF82Eu, 0x0325u, 0x858Fu, 0x7E84u, 0x036Cu, 0xF867u, 0x7ECDu, 0x85C6u, + 0xFEF6u, 0x05FDu, 0x8357u, 0x785Cu, 0x05B4u, 0xFEBFu, 0x7815u, 0x831Eu, + 0x83C5u, 0x78CEu, 0xFE64u, 0x056Fu, 0x7887u, 0x838Cu, 0x0526u, 0xFE2Du, + 0x0490u, 0xFF9Bu, 0x7931u, 0x823Au, 0xFFD2u, 0x04D9u, 0x8273u, 0x7978u, + 0x79A3u, 0x82A8u, 0x0402u, 0xFF09u, 0x82E1u, 0x79EAu, 0xFF40u, 0x044Bu, + 0x818Du, 0x7A86u, 0xFC2Cu, 0x0727u, 0x7ACFu, 0x81C4u, 0x076Eu, 0xFC65u, + 0xFCBEu, 0x07B5u, 0x811Fu, 0x7A14u, 0x07FCu, 0xFCF7u, 0x7A5Du, 0x8156u, + 0x7BEBu, 0x80E0u, 0x064Au, 0xFD41u, 0x80A9u, 0x7BA2u, 0xFD08u, 0x0603u, + 0x06D8u, 0xFDD3u, 0x7B79u, 0x8072u, 0xFD9Au, 0x0691u, 0x803Bu, 0x7B30u, + 0x765Bu, 0x8D50u, 0x0BFAu, 0xF0F1u, 0x8D19u, 0x7612u, 0xF0B8u, 0x0BB3u, + 0x0B68u, 0xF063u, 0x76C9u, 0x8DC2u, 0xF02Au, 0x0B21u, 0x8D8Bu, 0x7680u, + 0x8C3Du, 0x7736u, 0xF19Cu, 0x0A97u, 0x777Fu, 0x8C74u, 0x0ADEu, 0xF1D5u, + 0xF10Eu, 0x0A05u, 0x8CAFu, 0x77A4u, 0x0A4Cu, 0xF147u, 0x77EDu, 0x8CE6u, + 0x0920u, 0xF22Bu, 0x7481u, 0x8F8Au, 0xF262u, 0x0969u, 0x8FC3u, 0x74C8u, + 0x7413u, 0x8F18u, 0x09B2u, 0xF2B9u, 0x8F51u, 0x745Au, 0xF2F0u, 0x09FBu, + 0xF346u, 0x084Du, 0x8EE7u, 0x75ECu, 0x0804u, 0xF30Fu, 0x75A5u, 0x8EAEu, + 0x8E75u, 0x757Eu, 0xF3D4u, 0x08DFu, 0x7537u, 0x8E3Cu, 0x0896u, 0xF39Du, + 0x88ADu, 0x73A6u, 0xF50Cu, 0x0E07u, 0x73EFu, 0x88E4u, 0x0E4Eu, 0xF545u, + 0xF59Eu, 0x0E95u, 0x883Fu, 0x7334u, 0x0EDCu, 0xF5D7u, 0x737Du, 0x8876u, + 0x72CBu, 0x89C0u, 0x0F6Au, 0xF461u, 0x8989u, 0x7282u, 0xF428u, 0x0F23u, + 0x0FF8u, 0xF4F3u, 0x7259u, 0x8952u, 0xF4BAu, 0x0FB1u, 0x891Bu, 0x7210u, + 0xF7D6u, 0x0CDDu, 0x8A77u, 0x717Cu, 0x0C94u, 0xF79Fu, 0x7135u, 0x8A3Eu, + 0x8AE5u, 0x71EEu, 0xF744u, 0x0C4Fu, 0x71A7u, 0x8AACu, 0x0C06u, 0xF70Du, + 0x0DB0u, 0xF6BBu, 0x7011u, 0x8B1Au, 0xF6F2u, 0x0DF9u, 0x8B53u, 0x7058u, + 0x7083u, 0x8B88u, 0x0D22u, 0xF629u, 0x8BC1u, 0x70CAu, 0xF660u, 0x0D6Bu + }, + { + 0x0000u, 0xECB6u, 0x52DBu, 0xBE6Du, 0xA5B6u, 0x4900u, 0xF76Du, 0x1BDBu, + 0xC0DBu, 0x2C6Du, 0x9200u, 0x7EB6u, 0x656Du, 0x89DBu, 0x37B6u, 0xDB00u, + 0x0A01u, 0xE6B7u, 0x58DAu, 0xB46Cu, 0xAFB7u, 0x4301u, 0xFD6Cu, 0x11DAu, + 0xCADAu, 0x266Cu, 0x9801u, 0x74B7u, 0x6F6Cu, 0x83DAu, 0x3DB7u, 0xD101u, + 0x1402u, 0xF8B4u, 0x46D9u, 0xAA6Fu, 0xB1B4u, 0x5D02u, 0xE36Fu, 0x0FD9u, + 0xD4D9u, 0x386Fu, 0x8602u, 0x6AB4u, 0x716Fu, 0x9DD9u, 0x23B4u, 0xCF02u, + 0x1E03u, 0xF2B5u, 0x4CD8u, 0xA06Eu, 0xBBB5u, 0x5703u, 0xE96Eu, 0x05D8u, + 0xDED8u, 0x326Eu, 0x8C03u, 0x60B5u, 0x7B6Eu, 0x97D8u, 0x29B5u, 0xC503u, + 0x2804u, 0xC4B2u, 0x7ADFu, 0x9669u, 0x8DB2u, 0x6104u, 0xDF69u, 0x33DFu, + 0xE8DFu, 0x0469u, 0xBA04u, 0x56B2u, 0x4D69u, 0xA1DFu, 0x1FB2u, 0xF304u, + 0x2205u, 0xCEB3u, 0x70DEu, 0x9C68u, 0x87B3u, 0x6B05u, 0xD568u, 0x39DEu, + 0xE2DEu, 0x0E68u, 0xB005u, 0x5CB3u, 0x4768u, 0xABDEu, 0x15B3u, 0xF905u, + 0x3C06u, 0xD0B0u, 0x6EDDu, 0x826Bu, 0x99B0u, 0x7506u, 0xCB6Bu, 0x27DDu, + 0xFCDDu, 0x106Bu, 0xAE06u, 0x42B0u, 0x596Bu, 0xB5DDu, 0x0BB0u, 0xE706u, + 0x3607u, 0xDAB1u, 0x64DCu, 0x886Au, 0x93B1u, 0x7F07u, 0xC16Au, 0x2DDCu, + 0xF6DCu, 0x1A6Au, 0xA407u, 0x48B1u, 0x536Au, 0xBFDCu, 0x01B1u, 0xED07u, + 0x5008u, 0xBCBEu, 0x02D3u, 0xEE65u, 0xF5BEu, 0x1908u, 0xA765u, 0x4BD3u, + 0x90D3u, 0x7C65u, 0xC208u, 0x2EBEu, 0x3565u, 0xD9D3u, 0x67BEu, 0x8B08u, + 0x5A09u, 0xB6BFu, 0x08D2u, 0xE464u, 0xFFBFu, 0x1309u, 0xAD64u, 0x41D2u, + 0x9AD2u, 0x7664u, 0xC809u, 0x24BFu, 0x3F64u, 0xD3D2u, 0x6DBFu, 0x8109u, + 0x440Au, 0xA8BCu, 0x16D1u, 0xFA67u, 0xE1BCu, 0x0D0Au, 0xB367u, 0x5FD1u, + 0x84D1u, 0x6867u, 0xD60Au, 0x3ABCu, 0x2167u, 0xCDD1u, 0x73BCu, 0x9F0Au, + 0x4E0Bu, 0xA2BDu, 0x1CD0u, 0xF066u, 0xEBBDu, 0x070Bu, 0xB966u, 0x55D0u, + 0x8ED0u, 0x6266u, 0xDC0Bu, 0x30BDu, 0x2B66u, 0xC7D0u, 0x79BDu, 0x950Bu, + 0x780Cu, 0x94BAu, 0x2AD7u, 0xC661u, 0xDDBAu, 0x310Cu, 0x8F61u, 0x63D7u, + 0xB8D7u, 0x5461u, 0xEA0Cu, 0x06BAu, 0x1D61u, 0xF1D7u, 0x4FBAu, 0xA30Cu, + 0x720Du, 0x9EBBu, 0x20D6u, 0xCC60u, 0xD7BBu, 0x3B0Du, 0x8560u, 0x69D6u, + 0xB2D6u, 0x5E60u, 0xE00Du, 0x0CBBu, 0x1760u, 0xFBD6u, 0x45BBu, 0xA90Du, + 0x6C0Eu, 0x80B8u, 0x3ED5u, 0xD263u, 0xC9B8u, 0x250Eu, 0x9B63u, 0x77D5u, + 0xACD5u, 0x4063u, 0xFE0Eu, 0x12B8u, 0x0963u, 0xE5D5u, 0x5BB8u, 0xB70Eu, + 0x660Fu, 0x8AB9u, 0x34D4u, 0xD862u, 0xC3B9u, 0x2F0Fu, 0x9162u, 0x7DD4u, + 0xA6D4u, 0x4A62u, 0xF40Fu, 0x18B9u, 0x0362u, 0xEFD4u, 0x51B9u, 0xBD0Fu + }, + { + 0x0000u, 0xA010u, 0xCB97u, 0x6B87u, 0x1C99u, 0xBC89u, 0xD70Eu, 0x771Eu, + 0x3932u, 0x9922u, 0xF2A5u, 0x52B5u, 0x25ABu, 0x85BBu, 0xEE3Cu, 0x4E2Cu, + 0x7264u, 0xD274u, 0xB9F3u, 0x19E3u, 0x6EFDu, 0xCEEDu, 0xA56Au, 0x057Au, + 0x4B56u, 0xEB46u, 0x80C1u, 0x20D1u, 0x57CFu, 0xF7DFu, 0x9C58u, 0x3C48u, + 0xE4C8u, 0x44D8u, 0x2F5Fu, 0x8F4Fu, 0xF851u, 0x5841u, 0x33C6u, 0x93D6u, + 0xDDFAu, 0x7DEAu, 0x166Du, 0xB67Du, 0xC163u, 0x6173u, 0x0AF4u, 0xAAE4u, + 0x96ACu, 0x36BCu, 0x5D3Bu, 0xFD2Bu, 0x8A35u, 0x2A25u, 0x41A2u, 0xE1B2u, + 0xAF9Eu, 0x0F8Eu, 0x6409u, 0xC419u, 0xB307u, 0x1317u, 0x7890u, 0xD880u, + 0x4227u, 0xE237u, 0x89B0u, 0x29A0u, 0x5EBEu, 0xFEAEu, 0x9529u, 0x3539u, + 0x7B15u, 0xDB05u, 0xB082u, 0x1092u, 0x678Cu, 0xC79Cu, 0xAC1Bu, 0x0C0Bu, + 0x3043u, 0x9053u, 0xFBD4u, 0x5BC4u, 0x2CDAu, 0x8CCAu, 0xE74Du, 0x475Du, + 0x0971u, 0xA961u, 0xC2E6u, 0x62F6u, 0x15E8u, 0xB5F8u, 0xDE7Fu, 0x7E6Fu, + 0xA6EFu, 0x06FFu, 0x6D78u, 0xCD68u, 0xBA76u, 0x1A66u, 0x71E1u, 0xD1F1u, + 0x9FDDu, 0x3FCDu, 0x544Au, 0xF45Au, 0x8344u, 0x2354u, 0x48D3u, 0xE8C3u, + 0xD48Bu, 0x749Bu, 0x1F1Cu, 0xBF0Cu, 0xC812u, 0x6802u, 0x0385u, 0xA395u, + 0xEDB9u, 0x4DA9u, 0x262Eu, 0x863Eu, 0xF120u, 0x5130u, 0x3AB7u, 0x9AA7u, + 0x844Eu, 0x245Eu, 0x4FD9u, 0xEFC9u, 0x98D7u, 0x38C7u, 0x5340u, 0xF350u, + 0xBD7Cu, 0x1D6Cu, 0x76EBu, 0xD6FBu, 0xA1E5u, 0x01F5u, 0x6A72u, 0xCA62u, + 0xF62Au, 0x563Au, 0x3DBDu, 0x9DADu, 0xEAB3u, 0x4AA3u, 0x2124u, 0x8134u, + 0xCF18u, 0x6F08u, 0x048Fu, 0xA49Fu, 0xD381u, 0x7391u, 0x1816u, 0xB806u, + 0x6086u, 0xC096u, 0xAB11u, 0x0B01u, 0x7C1Fu, 0xDC0Fu, 0xB788u, 0x1798u, + 0x59B4u, 0xF9A4u, 0x9223u, 0x3233u, 0x452Du, 0xE53Du, 0x8EBAu, 0x2EAAu, + 0x12E2u, 0xB2F2u, 0xD975u, 0x7965u, 0x0E7Bu, 0xAE6Bu, 0xC5ECu, 0x65FCu, + 0x2BD0u, 0x8BC0u, 0xE047u, 0x4057u, 0x3749u, 0x9759u, 0xFCDEu, 0x5CCEu, + 0xC669u, 0x6679u, 0x0DFEu, 0xADEEu, 0xDAF0u, 0x7AE0u, 0x1167u, 0xB177u, + 0xFF5Bu, 0x5F4Bu, 0x34CCu, 0x94DCu, 0xE3C2u, 0x43D2u, 0x2855u, 0x8845u, + 0xB40Du, 0x141Du, 0x7F9Au, 0xDF8Au, 0xA894u, 0x0884u, 0x6303u, 0xC313u, + 0x8D3Fu, 0x2D2Fu, 0x46A8u, 0xE6B8u, 0x91A6u, 0x31B6u, 0x5A31u, 0xFA21u, + 0x22A1u, 0x82B1u, 0xE936u, 0x4926u, 0x3E38u, 0x9E28u, 0xF5AFu, 0x55BFu, + 0x1B93u, 0xBB83u, 0xD004u, 0x7014u, 0x070Au, 0xA71Au, 0xCC9Du, 0x6C8Du, + 0x50C5u, 0xF0D5u, 0x9B52u, 0x3B42u, 0x4C5Cu, 0xEC4Cu, 0x87CBu, 0x27DBu, + 0x69F7u, 0xC9E7u, 0xA260u, 0x0270u, 0x756Eu, 0xD57Eu, 0xBEF9u, 0x1EE9u + }, + { + 0x0000u, 0x832Bu, 0x8DE1u, 0x0ECAu, 0x9075u, 0x135Eu, 0x1D94u, 0x9EBFu, + 0xAB5Du, 0x2876u, 0x26BCu, 0xA597u, 0x3B28u, 0xB803u, 0xB6C9u, 0x35E2u, + 0xDD0Du, 0x5E26u, 0x50ECu, 0xD3C7u, 0x4D78u, 0xCE53u, 0xC099u, 0x43B2u, + 0x7650u, 0xF57Bu, 0xFBB1u, 0x789Au, 0xE625u, 0x650Eu, 0x6BC4u, 0xE8EFu, + 0x31ADu, 0xB286u, 0xBC4Cu, 0x3F67u, 0xA1D8u, 0x22F3u, 0x2C39u, 0xAF12u, + 0x9AF0u, 0x19DBu, 0x1711u, 0x943Au, 0x0A85u, 0x89AEu, 0x8764u, 0x044Fu, + 0xECA0u, 0x6F8Bu, 0x6141u, 0xE26Au, 0x7CD5u, 0xFFFEu, 0xF134u, 0x721Fu, + 0x47FDu, 0xC4D6u, 0xCA1Cu, 0x4937u, 0xD788u, 0x54A3u, 0x5A69u, 0xD942u, + 0x635Au, 0xE071u, 0xEEBBu, 0x6D90u, 0xF32Fu, 0x7004u, 0x7ECEu, 0xFDE5u, + 0xC807u, 0x4B2Cu, 0x45E6u, 0xC6CDu, 0x5872u, 0xDB59u, 0xD593u, 0x56B8u, + 0xBE57u, 0x3D7Cu, 0x33B6u, 0xB09Du, 0x2E22u, 0xAD09u, 0xA3C3u, 0x20E8u, + 0x150Au, 0x9621u, 0x98EBu, 0x1BC0u, 0x857Fu, 0x0654u, 0x089Eu, 0x8BB5u, + 0x52F7u, 0xD1DCu, 0xDF16u, 0x5C3Du, 0xC282u, 0x41A9u, 0x4F63u, 0xCC48u, + 0xF9AAu, 0x7A81u, 0x744Bu, 0xF760u, 0x69DFu, 0xEAF4u, 0xE43Eu, 0x6715u, + 0x8FFAu, 0x0CD1u, 0x021Bu, 0x8130u, 0x1F8Fu, 0x9CA4u, 0x926Eu, 0x1145u, + 0x24A7u, 0xA78Cu, 0xA946u, 0x2A6Du, 0xB4D2u, 0x37F9u, 0x3933u, 0xBA18u, + 0xC6B4u, 0x459Fu, 0x4B55u, 0xC87Eu, 0x56C1u, 0xD5EAu, 0xDB20u, 0x580Bu, + 0x6DE9u, 0xEEC2u, 0xE008u, 0x6323u, 0xFD9Cu, 0x7EB7u, 0x707Du, 0xF356u, + 0x1BB9u, 0x9892u, 0x9658u, 0x1573u, 0x8BCCu, 0x08E7u, 0x062Du, 0x8506u, + 0xB0E4u, 0x33CFu, 0x3D05u, 0xBE2Eu, 0x2091u, 0xA3BAu, 0xAD70u, 0x2E5Bu, + 0xF719u, 0x7432u, 0x7AF8u, 0xF9D3u, 0x676Cu, 0xE447u, 0xEA8Du, 0x69A6u, + 0x5C44u, 0xDF6Fu, 0xD1A5u, 0x528Eu, 0xCC31u, 0x4F1Au, 0x41D0u, 0xC2FBu, + 0x2A14u, 0xA93Fu, 0xA7F5u, 0x24DEu, 0xBA61u, 0x394Au, 0x3780u, 0xB4ABu, + 0x8149u, 0x0262u, 0x0CA8u, 0x8F83u, 0x113Cu, 0x9217u, 0x9CDDu, 0x1FF6u, + 0xA5EEu, 0x26C5u, 0x280Fu, 0xAB24u, 0x359Bu, 0xB6B0u, 0xB87Au, 0x3B51u, + 0x0EB3u, 0x8D98u, 0x8352u, 0x0079u, 0x9EC6u, 0x1DEDu, 0x1327u, 0x900Cu, + 0x78E3u, 0xFBC8u, 0xF502u, 0x7629u, 0xE896u, 0x6BBDu, 0x6577u, 0xE65Cu, + 0xD3BEu, 0x5095u, 0x5E5Fu, 0xDD74u, 0x43CBu, 0xC0E0u, 0xCE2Au, 0x4D01u, + 0x9443u, 0x1768u, 0x19A2u, 0x9A89u, 0x0436u, 0x871Du, 0x89D7u, 0x0AFCu, + 0x3F1Eu, 0xBC35u, 0xB2FFu, 0x31D4u, 0xAF6Bu, 0x2C40u, 0x228Au, 0xA1A1u, + 0x494Eu, 0xCA65u, 0xC4AFu, 0x4784u, 0xD93Bu, 0x5A10u, 0x54DAu, 0xD7F1u, + 0xE213u, 0x6138u, 0x6FF2u, 0xECD9u, 0x7266u, 0xF14Du, 0xFF87u, 0x7CACu + } +}; + +static inline uint16_t +crc_update_fast(uint16_t crc, const void *data, size_t data_len) +{ + const unsigned char *d = (const unsigned char *)data; + const unsigned char *d_end = d + data_len; + const unsigned char *d_last16 = d + (data_len & ~0x0F); + + for (; d < d_last16 ; d += 16) { + crc = crc_table_fast[15][d[0] ^ (uint8_t)(crc >> 8)] ^ + crc_table_fast[14][d[1] ^ (uint8_t)(crc >> 0)] ^ + crc_table_fast[13][d[2]] ^ + crc_table_fast[12][d[3]] ^ + crc_table_fast[11][d[4]] ^ + crc_table_fast[10][d[5]] ^ + crc_table_fast[9][d[6]] ^ + crc_table_fast[8][d[7]] ^ + crc_table_fast[7][d[8]] ^ + crc_table_fast[6][d[9]] ^ + crc_table_fast[5][d[10]] ^ + crc_table_fast[4][d[11]] ^ + crc_table_fast[3][d[12]] ^ + crc_table_fast[2][d[13]] ^ + crc_table_fast[1][d[14]] ^ + crc_table_fast[0][d[15]]; + } + for (; d < d_end ; d++) { + crc = (crc << 8) ^ crc_table_fast[0][((uint8_t)(crc >> 8) ^ *d)]; + } + return crc & 0xffff; +} + +static inline uint16_t +crc16_table_t10dif(uint16_t init_crc, const void *buf, size_t len) +{ + uint16_t crc; + const uint8_t *data = (const uint8_t *)buf; + + crc = init_crc; + crc = crc_update_fast(crc, data, len); + return crc; +} + +uint16_t +spdk_crc16_t10dif(uint16_t init_crc, const void *buf, size_t len) +{ + return (crc16_table_t10dif(init_crc, buf, len)); +} + +uint16_t +spdk_crc16_t10dif_copy(uint16_t init_crc, uint8_t *dst, uint8_t *src, size_t len) +{ + memcpy(dst, src, len); + return (crc16_table_t10dif(init_crc, src, len)); +} + +#endif diff --git a/src/spdk/lib/util/crc32.c b/src/spdk/lib/util/crc32.c new file mode 100644 index 000000000..34bb60b78 --- /dev/null +++ b/src/spdk/lib/util/crc32.c @@ -0,0 +1,95 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "util_internal.h" +#include "spdk/crc32.h" + +void +crc32_table_init(struct spdk_crc32_table *table, uint32_t polynomial_reflect) +{ + int i, j; + uint32_t val; + + for (i = 0; i < 256; i++) { + val = i; + for (j = 0; j < 8; j++) { + if (val & 1) { + val = (val >> 1) ^ polynomial_reflect; + } else { + val = (val >> 1); + } + } + table->table[i] = val; + } +} + +#ifdef SPDK_HAVE_ARM_CRC + +uint32_t +crc32_update(const struct spdk_crc32_table *table, const void *buf, size_t len, uint32_t crc) +{ + size_t count; + const uint64_t *dword_buf; + + count = len & 7; + while (count--) { + crc = __crc32b(crc, *(const uint8_t *)buf); + buf++; + } + dword_buf = (const uint64_t *)buf; + + count = len / 8; + while (count--) { + crc = __crc32d(crc, *dword_buf); + dword_buf++; + } + + return crc; +} + +#else + +uint32_t +crc32_update(const struct spdk_crc32_table *table, const void *buf, size_t len, uint32_t crc) +{ + const uint8_t *buf_u8 = buf; + size_t i; + + for (i = 0; i < len; i++) { + crc = (crc >> 8) ^ table->table[(crc ^ buf_u8[i]) & 0xff]; + } + + return crc; +} + +#endif diff --git a/src/spdk/lib/util/crc32_ieee.c b/src/spdk/lib/util/crc32_ieee.c new file mode 100644 index 000000000..ddc3c9901 --- /dev/null +++ b/src/spdk/lib/util/crc32_ieee.c @@ -0,0 +1,49 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "util_internal.h" +#include "spdk/crc32.h" + +static struct spdk_crc32_table g_crc32_ieee_table; + +__attribute__((constructor)) static void +crc32_ieee_init(void) +{ + crc32_table_init(&g_crc32_ieee_table, SPDK_CRC32_POLYNOMIAL_REFLECT); +} + +uint32_t +spdk_crc32_ieee_update(const void *buf, size_t len, uint32_t crc) +{ + return crc32_update(&g_crc32_ieee_table, buf, len, crc); +} diff --git a/src/spdk/lib/util/crc32c.c b/src/spdk/lib/util/crc32c.c new file mode 100644 index 000000000..9acd8d80f --- /dev/null +++ b/src/spdk/lib/util/crc32c.c @@ -0,0 +1,133 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "util_internal.h" +#include "spdk/crc32.h" + +#ifdef SPDK_CONFIG_ISAL +#define SPDK_HAVE_ISAL +#include +#elif defined(__aarch64__) && defined(__ARM_FEATURE_CRC32) +#define SPDK_HAVE_ARM_CRC +#include +#elif defined(__x86_64__) && defined(__SSE4_2__) +#define SPDK_HAVE_SSE4_2 +#include +#endif + +#ifdef SPDK_HAVE_ISAL + +uint32_t +spdk_crc32c_update(const void *buf, size_t len, uint32_t crc) +{ + return crc32_iscsi((unsigned char *)buf, len, crc); +} + +#elif defined(SPDK_HAVE_SSE4_2) + +uint32_t +spdk_crc32c_update(const void *buf, size_t len, uint32_t crc) +{ + uint64_t crc_tmp64; + size_t count; + + /* _mm_crc32_u64() needs a 64-bit intermediate value */ + crc_tmp64 = crc; + + /* Process as much of the buffer as possible in 64-bit blocks. */ + count = len / 8; + while (count--) { + uint64_t block; + + /* + * Use memcpy() to avoid unaligned loads, which are undefined behavior in C. + * The compiler will optimize out the memcpy() in release builds. + */ + memcpy(&block, buf, sizeof(block)); + crc_tmp64 = _mm_crc32_u64(crc_tmp64, block); + buf += sizeof(block); + } + crc = (uint32_t)crc_tmp64; + + /* Handle any trailing bytes. */ + count = len & 7; + while (count--) { + crc = _mm_crc32_u8(crc, *(const uint8_t *)buf); + buf++; + } + + return crc; +} + +#elif defined(SPDK_HAVE_ARM_CRC) + +uint32_t +spdk_crc32c_update(const void *buf, size_t len, uint32_t crc) +{ + size_t count; + + count = len / 8; + while (count--) { + uint64_t block; + + memcpy(&block, buf, sizeof(block)); + crc = __crc32cd(crc, block); + buf += sizeof(block); + } + + count = len & 7; + while (count--) { + crc = __crc32cb(crc, *(const uint8_t *)buf); + buf++; + } + + return crc; +} + +#else /* Neither SSE 4.2 nor ARM CRC32 instructions available */ + +static struct spdk_crc32_table g_crc32c_table; + +__attribute__((constructor)) static void +crc32c_init(void) +{ + crc32_table_init(&g_crc32c_table, SPDK_CRC32C_POLYNOMIAL_REFLECT); +} + +uint32_t +spdk_crc32c_update(const void *buf, size_t len, uint32_t crc) +{ + return crc32_update(&g_crc32c_table, buf, len, crc); +} + +#endif diff --git a/src/spdk/lib/util/dif.c b/src/spdk/lib/util/dif.c new file mode 100644 index 000000000..64bce1487 --- /dev/null +++ b/src/spdk/lib/util/dif.c @@ -0,0 +1,1999 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "spdk/dif.h" +#include "spdk/crc16.h" +#include "spdk/crc32.h" +#include "spdk/endian.h" +#include "spdk/log.h" +#include "spdk/util.h" + +/* Context to iterate or create a iovec array. + * Each sgl is either iterated or created at a time. + */ +struct _dif_sgl { + /* Current iovec in the iteration or creation */ + struct iovec *iov; + + /* Remaining count of iovecs in the iteration or creation. */ + int iovcnt; + + /* Current offset in the iovec */ + uint32_t iov_offset; + + /* Size of the created iovec array in bytes */ + uint32_t total_size; +}; + +static inline void +_dif_sgl_init(struct _dif_sgl *s, struct iovec *iovs, int iovcnt) +{ + s->iov = iovs; + s->iovcnt = iovcnt; + s->iov_offset = 0; + s->total_size = 0; +} + +static void +_dif_sgl_advance(struct _dif_sgl *s, uint32_t step) +{ + s->iov_offset += step; + while (s->iovcnt != 0) { + if (s->iov_offset < s->iov->iov_len) { + break; + } + + s->iov_offset -= s->iov->iov_len; + s->iov++; + s->iovcnt--; + } +} + +static inline void +_dif_sgl_get_buf(struct _dif_sgl *s, void **_buf, uint32_t *_buf_len) +{ + if (_buf != NULL) { + *_buf = s->iov->iov_base + s->iov_offset; + } + if (_buf_len != NULL) { + *_buf_len = s->iov->iov_len - s->iov_offset; + } +} + +static inline bool +_dif_sgl_append(struct _dif_sgl *s, uint8_t *data, uint32_t data_len) +{ + assert(s->iovcnt > 0); + s->iov->iov_base = data; + s->iov->iov_len = data_len; + s->total_size += data_len; + s->iov++; + s->iovcnt--; + + if (s->iovcnt > 0) { + return true; + } else { + return false; + } +} + +static inline bool +_dif_sgl_append_split(struct _dif_sgl *dst, struct _dif_sgl *src, uint32_t data_len) +{ + uint8_t *buf; + uint32_t buf_len; + + while (data_len != 0) { + _dif_sgl_get_buf(src, (void *)&buf, &buf_len); + buf_len = spdk_min(buf_len, data_len); + + if (!_dif_sgl_append(dst, buf, buf_len)) { + return false; + } + + _dif_sgl_advance(src, buf_len); + data_len -= buf_len; + } + + return true; +} + +/* This function must be used before starting iteration. */ +static bool +_dif_sgl_is_bytes_multiple(struct _dif_sgl *s, uint32_t bytes) +{ + int i; + + for (i = 0; i < s->iovcnt; i++) { + if (s->iov[i].iov_len % bytes) { + return false; + } + } + + return true; +} + +/* This function must be used before starting iteration. */ +static bool +_dif_sgl_is_valid(struct _dif_sgl *s, uint32_t bytes) +{ + uint64_t total = 0; + int i; + + for (i = 0; i < s->iovcnt; i++) { + total += s->iov[i].iov_len; + } + + return total >= bytes; +} + +static void +_dif_sgl_copy(struct _dif_sgl *to, struct _dif_sgl *from) +{ + memcpy(to, from, sizeof(struct _dif_sgl)); +} + +static bool +_dif_type_is_valid(enum spdk_dif_type dif_type, uint32_t dif_flags) +{ + switch (dif_type) { + case SPDK_DIF_TYPE1: + case SPDK_DIF_TYPE2: + case SPDK_DIF_DISABLE: + break; + case SPDK_DIF_TYPE3: + if (dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK) { + SPDK_ERRLOG("Reference Tag should not be checked for Type 3\n"); + return false; + } + break; + default: + SPDK_ERRLOG("Unknown DIF Type: %d\n", dif_type); + return false; + } + + return true; +} + +static bool +_dif_is_disabled(enum spdk_dif_type dif_type) +{ + if (dif_type == SPDK_DIF_DISABLE) { + return true; + } else { + return false; + } +} + + +static uint32_t +_get_guard_interval(uint32_t block_size, uint32_t md_size, bool dif_loc, bool md_interleave) +{ + if (!dif_loc) { + /* For metadata formats with more than 8 bytes, if the DIF is + * contained in the last 8 bytes of metadata, then the CRC + * covers all metadata up to but excluding these last 8 bytes. + */ + if (md_interleave) { + return block_size - sizeof(struct spdk_dif); + } else { + return md_size - sizeof(struct spdk_dif); + } + } else { + /* For metadata formats with more than 8 bytes, if the DIF is + * contained in the first 8 bytes of metadata, then the CRC + * does not cover any metadata. + */ + if (md_interleave) { + return block_size - md_size; + } else { + return 0; + } + } +} + +int +spdk_dif_ctx_init(struct spdk_dif_ctx *ctx, uint32_t block_size, uint32_t md_size, + bool md_interleave, bool dif_loc, enum spdk_dif_type dif_type, uint32_t dif_flags, + uint32_t init_ref_tag, uint16_t apptag_mask, uint16_t app_tag, + uint32_t data_offset, uint16_t guard_seed) +{ + uint32_t data_block_size; + + if (md_size < sizeof(struct spdk_dif)) { + SPDK_ERRLOG("Metadata size is smaller than DIF size.\n"); + return -EINVAL; + } + + if (md_interleave) { + if (block_size < md_size) { + SPDK_ERRLOG("Block size is smaller than DIF size.\n"); + return -EINVAL; + } + data_block_size = block_size - md_size; + } else { + if (block_size == 0 || (block_size % 512) != 0) { + SPDK_ERRLOG("Zero block size is not allowed\n"); + return -EINVAL; + } + data_block_size = block_size; + } + + if (!_dif_type_is_valid(dif_type, dif_flags)) { + SPDK_ERRLOG("DIF type is invalid.\n"); + return -EINVAL; + } + + ctx->block_size = block_size; + ctx->md_size = md_size; + ctx->md_interleave = md_interleave; + ctx->guard_interval = _get_guard_interval(block_size, md_size, dif_loc, md_interleave); + ctx->dif_type = dif_type; + ctx->dif_flags = dif_flags; + ctx->init_ref_tag = init_ref_tag; + ctx->apptag_mask = apptag_mask; + ctx->app_tag = app_tag; + ctx->data_offset = data_offset; + ctx->ref_tag_offset = data_offset / data_block_size; + ctx->last_guard = guard_seed; + ctx->guard_seed = guard_seed; + ctx->remapped_init_ref_tag = 0; + + return 0; +} + +void +spdk_dif_ctx_set_data_offset(struct spdk_dif_ctx *ctx, uint32_t data_offset) +{ + uint32_t data_block_size; + + if (ctx->md_interleave) { + data_block_size = ctx->block_size - ctx->md_size; + } else { + data_block_size = ctx->block_size; + } + + ctx->data_offset = data_offset; + ctx->ref_tag_offset = data_offset / data_block_size; +} + +void +spdk_dif_ctx_set_remapped_init_ref_tag(struct spdk_dif_ctx *ctx, + uint32_t remapped_init_ref_tag) +{ + ctx->remapped_init_ref_tag = remapped_init_ref_tag; +} + +static void +_dif_generate(void *_dif, uint16_t guard, uint32_t offset_blocks, + const struct spdk_dif_ctx *ctx) +{ + struct spdk_dif *dif = _dif; + uint32_t ref_tag; + + if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { + to_be16(&dif->guard, guard); + } + + if (ctx->dif_flags & SPDK_DIF_FLAGS_APPTAG_CHECK) { + to_be16(&dif->app_tag, ctx->app_tag); + } + + if (ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK) { + /* For type 1 and 2, the reference tag is incremented for each + * subsequent logical block. For type 3, the reference tag + * remains the same as the initial reference tag. + */ + if (ctx->dif_type != SPDK_DIF_TYPE3) { + ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset + offset_blocks; + } else { + ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset; + } + + to_be32(&dif->ref_tag, ref_tag); + } +} + +static void +dif_generate(struct _dif_sgl *sgl, uint32_t num_blocks, const struct spdk_dif_ctx *ctx) +{ + uint32_t offset_blocks = 0; + void *buf; + uint16_t guard = 0; + + while (offset_blocks < num_blocks) { + _dif_sgl_get_buf(sgl, &buf, NULL); + + if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { + guard = spdk_crc16_t10dif(ctx->guard_seed, buf, ctx->guard_interval); + } + + _dif_generate(buf + ctx->guard_interval, guard, offset_blocks, ctx); + + _dif_sgl_advance(sgl, ctx->block_size); + offset_blocks++; + } +} + +static uint16_t +_dif_generate_split(struct _dif_sgl *sgl, uint32_t offset_in_block, uint32_t data_len, + uint16_t guard, uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) +{ + uint32_t offset_in_dif, buf_len; + void *buf; + struct spdk_dif dif = {}; + + assert(offset_in_block < ctx->guard_interval); + assert(offset_in_block + data_len < ctx->guard_interval || + offset_in_block + data_len == ctx->block_size); + + /* Compute CRC over split logical block data. */ + while (data_len != 0 && offset_in_block < ctx->guard_interval) { + _dif_sgl_get_buf(sgl, &buf, &buf_len); + buf_len = spdk_min(buf_len, data_len); + buf_len = spdk_min(buf_len, ctx->guard_interval - offset_in_block); + + if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { + guard = spdk_crc16_t10dif(guard, buf, buf_len); + } + + _dif_sgl_advance(sgl, buf_len); + offset_in_block += buf_len; + data_len -= buf_len; + } + + if (offset_in_block < ctx->guard_interval) { + return guard; + } + + /* If a whole logical block data is parsed, generate DIF + * and save it to the temporary DIF area. + */ + _dif_generate(&dif, guard, offset_blocks, ctx); + + /* Copy generated DIF field to the split DIF field, and then + * skip metadata field after DIF field (if any). + */ + while (offset_in_block < ctx->block_size) { + _dif_sgl_get_buf(sgl, &buf, &buf_len); + + if (offset_in_block < ctx->guard_interval + sizeof(struct spdk_dif)) { + offset_in_dif = offset_in_block - ctx->guard_interval; + buf_len = spdk_min(buf_len, sizeof(struct spdk_dif) - offset_in_dif); + + memcpy(buf, ((uint8_t *)&dif) + offset_in_dif, buf_len); + } else { + buf_len = spdk_min(buf_len, ctx->block_size - offset_in_block); + } + + _dif_sgl_advance(sgl, buf_len); + offset_in_block += buf_len; + } + + if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { + guard = ctx->guard_seed; + } + + return guard; +} + +static void +dif_generate_split(struct _dif_sgl *sgl, uint32_t num_blocks, + const struct spdk_dif_ctx *ctx) +{ + uint32_t offset_blocks; + uint16_t guard = 0; + + if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { + guard = ctx->guard_seed; + } + + for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { + _dif_generate_split(sgl, 0, ctx->block_size, guard, offset_blocks, ctx); + } +} + +int +spdk_dif_generate(struct iovec *iovs, int iovcnt, uint32_t num_blocks, + const struct spdk_dif_ctx *ctx) +{ + struct _dif_sgl sgl; + + _dif_sgl_init(&sgl, iovs, iovcnt); + + if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { + SPDK_ERRLOG("Size of iovec array is not valid.\n"); + return -EINVAL; + } + + if (_dif_is_disabled(ctx->dif_type)) { + return 0; + } + + if (_dif_sgl_is_bytes_multiple(&sgl, ctx->block_size)) { + dif_generate(&sgl, num_blocks, ctx); + } else { + dif_generate_split(&sgl, num_blocks, ctx); + } + + return 0; +} + +static void +_dif_error_set(struct spdk_dif_error *err_blk, uint8_t err_type, + uint32_t expected, uint32_t actual, uint32_t err_offset) +{ + if (err_blk) { + err_blk->err_type = err_type; + err_blk->expected = expected; + err_blk->actual = actual; + err_blk->err_offset = err_offset; + } +} + +static int +_dif_verify(void *_dif, uint16_t guard, uint32_t offset_blocks, + const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) +{ + struct spdk_dif *dif = _dif; + uint16_t _guard; + uint16_t _app_tag; + uint32_t ref_tag, _ref_tag; + + switch (ctx->dif_type) { + case SPDK_DIF_TYPE1: + case SPDK_DIF_TYPE2: + /* If Type 1 or 2 is used, then all DIF checks are disabled when + * the Application Tag is 0xFFFF. + */ + if (dif->app_tag == 0xFFFF) { + return 0; + } + break; + case SPDK_DIF_TYPE3: + /* If Type 3 is used, then all DIF checks are disabled when the + * Application Tag is 0xFFFF and the Reference Tag is 0xFFFFFFFF. + */ + if (dif->app_tag == 0xFFFF && dif->ref_tag == 0xFFFFFFFF) { + return 0; + } + break; + default: + break; + } + + /* For type 1 and 2, the reference tag is incremented for each + * subsequent logical block. For type 3, the reference tag + * remains the same as the initial reference tag. + */ + if (ctx->dif_type != SPDK_DIF_TYPE3) { + ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset + offset_blocks; + } else { + ref_tag = ctx->init_ref_tag + ctx->ref_tag_offset; + } + + if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { + /* Compare the DIF Guard field to the CRC computed over the logical + * block data. + */ + _guard = from_be16(&dif->guard); + if (_guard != guard) { + _dif_error_set(err_blk, SPDK_DIF_GUARD_ERROR, _guard, guard, + offset_blocks); + SPDK_ERRLOG("Failed to compare Guard: LBA=%" PRIu32 "," \ + " Expected=%x, Actual=%x\n", + ref_tag, _guard, guard); + return -1; + } + } + + if (ctx->dif_flags & SPDK_DIF_FLAGS_APPTAG_CHECK) { + /* Compare unmasked bits in the DIF Application Tag field to the + * passed Application Tag. + */ + _app_tag = from_be16(&dif->app_tag); + if ((_app_tag & ctx->apptag_mask) != ctx->app_tag) { + _dif_error_set(err_blk, SPDK_DIF_APPTAG_ERROR, ctx->app_tag, + (_app_tag & ctx->apptag_mask), offset_blocks); + SPDK_ERRLOG("Failed to compare App Tag: LBA=%" PRIu32 "," \ + " Expected=%x, Actual=%x\n", + ref_tag, ctx->app_tag, (_app_tag & ctx->apptag_mask)); + return -1; + } + } + + if (ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK) { + switch (ctx->dif_type) { + case SPDK_DIF_TYPE1: + case SPDK_DIF_TYPE2: + /* Compare the DIF Reference Tag field to the passed Reference Tag. + * The passed Reference Tag will be the least significant 4 bytes + * of the LBA when Type 1 is used, and application specific value + * if Type 2 is used, + */ + _ref_tag = from_be32(&dif->ref_tag); + if (_ref_tag != ref_tag) { + _dif_error_set(err_blk, SPDK_DIF_REFTAG_ERROR, ref_tag, + _ref_tag, offset_blocks); + SPDK_ERRLOG("Failed to compare Ref Tag: LBA=%" PRIu32 "," \ + " Expected=%x, Actual=%x\n", + ref_tag, ref_tag, _ref_tag); + return -1; + } + break; + case SPDK_DIF_TYPE3: + /* For Type 3, computed Reference Tag remains unchanged. + * Hence ignore the Reference Tag field. + */ + break; + default: + break; + } + } + + return 0; +} + +static int +dif_verify(struct _dif_sgl *sgl, uint32_t num_blocks, + const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) +{ + uint32_t offset_blocks = 0; + int rc; + void *buf; + uint16_t guard = 0; + + while (offset_blocks < num_blocks) { + _dif_sgl_get_buf(sgl, &buf, NULL); + + if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { + guard = spdk_crc16_t10dif(ctx->guard_seed, buf, ctx->guard_interval); + } + + rc = _dif_verify(buf + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); + if (rc != 0) { + return rc; + } + + _dif_sgl_advance(sgl, ctx->block_size); + offset_blocks++; + } + + return 0; +} + +static int +_dif_verify_split(struct _dif_sgl *sgl, uint32_t offset_in_block, uint32_t data_len, + uint16_t *_guard, uint32_t offset_blocks, + const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) +{ + uint32_t offset_in_dif, buf_len; + void *buf; + uint16_t guard; + struct spdk_dif dif = {}; + int rc; + + assert(_guard != NULL); + assert(offset_in_block < ctx->guard_interval); + assert(offset_in_block + data_len < ctx->guard_interval || + offset_in_block + data_len == ctx->block_size); + + guard = *_guard; + + /* Compute CRC over split logical block data. */ + while (data_len != 0 && offset_in_block < ctx->guard_interval) { + _dif_sgl_get_buf(sgl, &buf, &buf_len); + buf_len = spdk_min(buf_len, data_len); + buf_len = spdk_min(buf_len, ctx->guard_interval - offset_in_block); + + if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { + guard = spdk_crc16_t10dif(guard, buf, buf_len); + } + + _dif_sgl_advance(sgl, buf_len); + offset_in_block += buf_len; + data_len -= buf_len; + } + + if (offset_in_block < ctx->guard_interval) { + *_guard = guard; + return 0; + } + + /* Copy the split DIF field to the temporary DIF buffer, and then + * skip metadata field after DIF field (if any). */ + while (offset_in_block < ctx->block_size) { + _dif_sgl_get_buf(sgl, &buf, &buf_len); + + if (offset_in_block < ctx->guard_interval + sizeof(struct spdk_dif)) { + offset_in_dif = offset_in_block - ctx->guard_interval; + buf_len = spdk_min(buf_len, sizeof(struct spdk_dif) - offset_in_dif); + + memcpy((uint8_t *)&dif + offset_in_dif, buf, buf_len); + } else { + buf_len = spdk_min(buf_len, ctx->block_size - offset_in_block); + } + _dif_sgl_advance(sgl, buf_len); + offset_in_block += buf_len; + } + + rc = _dif_verify(&dif, guard, offset_blocks, ctx, err_blk); + if (rc != 0) { + return rc; + } + + if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { + guard = ctx->guard_seed; + } + + *_guard = guard; + return 0; +} + +static int +dif_verify_split(struct _dif_sgl *sgl, uint32_t num_blocks, + const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) +{ + uint32_t offset_blocks; + uint16_t guard = 0; + int rc; + + if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { + guard = ctx->guard_seed; + } + + for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { + rc = _dif_verify_split(sgl, 0, ctx->block_size, &guard, offset_blocks, + ctx, err_blk); + if (rc != 0) { + return rc; + } + } + + return 0; +} + +int +spdk_dif_verify(struct iovec *iovs, int iovcnt, uint32_t num_blocks, + const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) +{ + struct _dif_sgl sgl; + + _dif_sgl_init(&sgl, iovs, iovcnt); + + if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { + SPDK_ERRLOG("Size of iovec array is not valid.\n"); + return -EINVAL; + } + + if (_dif_is_disabled(ctx->dif_type)) { + return 0; + } + + if (_dif_sgl_is_bytes_multiple(&sgl, ctx->block_size)) { + return dif_verify(&sgl, num_blocks, ctx, err_blk); + } else { + return dif_verify_split(&sgl, num_blocks, ctx, err_blk); + } +} + +static uint32_t +dif_update_crc32c(struct _dif_sgl *sgl, uint32_t num_blocks, + uint32_t crc32c, const struct spdk_dif_ctx *ctx) +{ + uint32_t offset_blocks; + void *buf; + + for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { + _dif_sgl_get_buf(sgl, &buf, NULL); + + crc32c = spdk_crc32c_update(buf, ctx->block_size - ctx->md_size, crc32c); + + _dif_sgl_advance(sgl, ctx->block_size); + } + + return crc32c; +} + +static uint32_t +_dif_update_crc32c_split(struct _dif_sgl *sgl, uint32_t offset_in_block, uint32_t data_len, + uint32_t crc32c, const struct spdk_dif_ctx *ctx) +{ + uint32_t data_block_size, buf_len; + void *buf; + + data_block_size = ctx->block_size - ctx->md_size; + + assert(offset_in_block + data_len <= ctx->block_size); + + while (data_len != 0) { + _dif_sgl_get_buf(sgl, &buf, &buf_len); + buf_len = spdk_min(buf_len, data_len); + + if (offset_in_block < data_block_size) { + buf_len = spdk_min(buf_len, data_block_size - offset_in_block); + crc32c = spdk_crc32c_update(buf, buf_len, crc32c); + } + + _dif_sgl_advance(sgl, buf_len); + offset_in_block += buf_len; + data_len -= buf_len; + } + + return crc32c; +} + +static uint32_t +dif_update_crc32c_split(struct _dif_sgl *sgl, uint32_t num_blocks, + uint32_t crc32c, const struct spdk_dif_ctx *ctx) +{ + uint32_t offset_blocks; + + for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { + crc32c = _dif_update_crc32c_split(sgl, 0, ctx->block_size, crc32c, ctx); + } + + return crc32c; +} + +int +spdk_dif_update_crc32c(struct iovec *iovs, int iovcnt, uint32_t num_blocks, + uint32_t *_crc32c, const struct spdk_dif_ctx *ctx) +{ + struct _dif_sgl sgl; + + if (_crc32c == NULL) { + return -EINVAL; + } + + _dif_sgl_init(&sgl, iovs, iovcnt); + + if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { + SPDK_ERRLOG("Size of iovec array is not valid.\n"); + return -EINVAL; + } + + if (_dif_sgl_is_bytes_multiple(&sgl, ctx->block_size)) { + *_crc32c = dif_update_crc32c(&sgl, num_blocks, *_crc32c, ctx); + } else { + *_crc32c = dif_update_crc32c_split(&sgl, num_blocks, *_crc32c, ctx); + } + + return 0; +} + +static void +dif_generate_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, + uint32_t num_blocks, const struct spdk_dif_ctx *ctx) +{ + uint32_t offset_blocks = 0, data_block_size; + void *src, *dst; + uint16_t guard; + + data_block_size = ctx->block_size - ctx->md_size; + + while (offset_blocks < num_blocks) { + _dif_sgl_get_buf(src_sgl, &src, NULL); + _dif_sgl_get_buf(dst_sgl, &dst, NULL); + + guard = 0; + if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { + guard = spdk_crc16_t10dif_copy(ctx->guard_seed, dst, src, data_block_size); + guard = spdk_crc16_t10dif(guard, dst + data_block_size, + ctx->guard_interval - data_block_size); + } else { + memcpy(dst, src, data_block_size); + } + + _dif_generate(dst + ctx->guard_interval, guard, offset_blocks, ctx); + + _dif_sgl_advance(src_sgl, data_block_size); + _dif_sgl_advance(dst_sgl, ctx->block_size); + offset_blocks++; + } +} + +static void +_dif_generate_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, + uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) +{ + uint32_t offset_in_block, src_len, data_block_size; + uint16_t guard = 0; + void *src, *dst; + + _dif_sgl_get_buf(dst_sgl, &dst, NULL); + + data_block_size = ctx->block_size - ctx->md_size; + + if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { + guard = ctx->guard_seed; + } + offset_in_block = 0; + + while (offset_in_block < data_block_size) { + /* Compute CRC over split logical block data and copy + * data to bounce buffer. + */ + _dif_sgl_get_buf(src_sgl, &src, &src_len); + src_len = spdk_min(src_len, data_block_size - offset_in_block); + + if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { + guard = spdk_crc16_t10dif_copy(guard, dst + offset_in_block, + src, src_len); + } else { + memcpy(dst + offset_in_block, src, src_len); + } + + _dif_sgl_advance(src_sgl, src_len); + offset_in_block += src_len; + } + + if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { + guard = spdk_crc16_t10dif(guard, dst + data_block_size, + ctx->guard_interval - data_block_size); + } + + _dif_sgl_advance(dst_sgl, ctx->block_size); + + _dif_generate(dst + ctx->guard_interval, guard, offset_blocks, ctx); +} + +static void +dif_generate_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, + uint32_t num_blocks, const struct spdk_dif_ctx *ctx) +{ + uint32_t offset_blocks; + + for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { + _dif_generate_copy_split(src_sgl, dst_sgl, offset_blocks, ctx); + } +} + +int +spdk_dif_generate_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iov, + uint32_t num_blocks, const struct spdk_dif_ctx *ctx) +{ + struct _dif_sgl src_sgl, dst_sgl; + uint32_t data_block_size; + + _dif_sgl_init(&src_sgl, iovs, iovcnt); + _dif_sgl_init(&dst_sgl, bounce_iov, 1); + + data_block_size = ctx->block_size - ctx->md_size; + + if (!_dif_sgl_is_valid(&src_sgl, data_block_size * num_blocks) || + !_dif_sgl_is_valid(&dst_sgl, ctx->block_size * num_blocks)) { + SPDK_ERRLOG("Size of iovec arrays are not valid.\n"); + return -EINVAL; + } + + if (_dif_is_disabled(ctx->dif_type)) { + return 0; + } + + if (_dif_sgl_is_bytes_multiple(&src_sgl, data_block_size)) { + dif_generate_copy(&src_sgl, &dst_sgl, num_blocks, ctx); + } else { + dif_generate_copy_split(&src_sgl, &dst_sgl, num_blocks, ctx); + } + + return 0; +} + +static int +dif_verify_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, + uint32_t num_blocks, const struct spdk_dif_ctx *ctx, + struct spdk_dif_error *err_blk) +{ + uint32_t offset_blocks = 0, data_block_size; + void *src, *dst; + int rc; + uint16_t guard; + + data_block_size = ctx->block_size - ctx->md_size; + + while (offset_blocks < num_blocks) { + _dif_sgl_get_buf(src_sgl, &src, NULL); + _dif_sgl_get_buf(dst_sgl, &dst, NULL); + + guard = 0; + if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { + guard = spdk_crc16_t10dif_copy(ctx->guard_seed, dst, src, data_block_size); + guard = spdk_crc16_t10dif(guard, src + data_block_size, + ctx->guard_interval - data_block_size); + } else { + memcpy(dst, src, data_block_size); + } + + rc = _dif_verify(src + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); + if (rc != 0) { + return rc; + } + + _dif_sgl_advance(src_sgl, ctx->block_size); + _dif_sgl_advance(dst_sgl, data_block_size); + offset_blocks++; + } + + return 0; +} + +static int +_dif_verify_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, + uint32_t offset_blocks, const struct spdk_dif_ctx *ctx, + struct spdk_dif_error *err_blk) +{ + uint32_t offset_in_block, dst_len, data_block_size; + uint16_t guard = 0; + void *src, *dst; + + _dif_sgl_get_buf(src_sgl, &src, NULL); + + data_block_size = ctx->block_size - ctx->md_size; + + if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { + guard = ctx->guard_seed; + } + offset_in_block = 0; + + while (offset_in_block < data_block_size) { + /* Compute CRC over split logical block data and copy + * data to bounce buffer. + */ + _dif_sgl_get_buf(dst_sgl, &dst, &dst_len); + dst_len = spdk_min(dst_len, data_block_size - offset_in_block); + + if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { + guard = spdk_crc16_t10dif_copy(guard, dst, + src + offset_in_block, dst_len); + } else { + memcpy(dst, src + offset_in_block, dst_len); + } + + _dif_sgl_advance(dst_sgl, dst_len); + offset_in_block += dst_len; + } + + if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { + guard = spdk_crc16_t10dif(guard, src + data_block_size, + ctx->guard_interval - data_block_size); + } + + _dif_sgl_advance(src_sgl, ctx->block_size); + + return _dif_verify(src + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); +} + +static int +dif_verify_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, + uint32_t num_blocks, const struct spdk_dif_ctx *ctx, + struct spdk_dif_error *err_blk) +{ + uint32_t offset_blocks; + int rc; + + for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { + rc = _dif_verify_copy_split(src_sgl, dst_sgl, offset_blocks, ctx, err_blk); + if (rc != 0) { + return rc; + } + } + + return 0; +} + +int +spdk_dif_verify_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iov, + uint32_t num_blocks, const struct spdk_dif_ctx *ctx, + struct spdk_dif_error *err_blk) +{ + struct _dif_sgl src_sgl, dst_sgl; + uint32_t data_block_size; + + _dif_sgl_init(&src_sgl, bounce_iov, 1); + _dif_sgl_init(&dst_sgl, iovs, iovcnt); + + data_block_size = ctx->block_size - ctx->md_size; + + if (!_dif_sgl_is_valid(&dst_sgl, data_block_size * num_blocks) || + !_dif_sgl_is_valid(&src_sgl, ctx->block_size * num_blocks)) { + SPDK_ERRLOG("Size of iovec arrays are not valid\n"); + return -EINVAL; + } + + if (_dif_is_disabled(ctx->dif_type)) { + return 0; + } + + if (_dif_sgl_is_bytes_multiple(&dst_sgl, data_block_size)) { + return dif_verify_copy(&src_sgl, &dst_sgl, num_blocks, ctx, err_blk); + } else { + return dif_verify_copy_split(&src_sgl, &dst_sgl, num_blocks, ctx, err_blk); + } +} + +static void +_bit_flip(uint8_t *buf, uint32_t flip_bit) +{ + uint8_t byte; + + byte = *buf; + byte ^= 1 << flip_bit; + *buf = byte; +} + +static int +_dif_inject_error(struct _dif_sgl *sgl, + uint32_t block_size, uint32_t num_blocks, + uint32_t inject_offset_blocks, + uint32_t inject_offset_bytes, + uint32_t inject_offset_bits) +{ + uint32_t offset_in_block, buf_len; + void *buf; + + _dif_sgl_advance(sgl, block_size * inject_offset_blocks); + + offset_in_block = 0; + + while (offset_in_block < block_size) { + _dif_sgl_get_buf(sgl, &buf, &buf_len); + buf_len = spdk_min(buf_len, block_size - offset_in_block); + + if (inject_offset_bytes >= offset_in_block && + inject_offset_bytes < offset_in_block + buf_len) { + buf += inject_offset_bytes - offset_in_block; + _bit_flip(buf, inject_offset_bits); + return 0; + } + + _dif_sgl_advance(sgl, buf_len); + offset_in_block += buf_len; + } + + return -1; +} + +static int +dif_inject_error(struct _dif_sgl *sgl, uint32_t block_size, uint32_t num_blocks, + uint32_t start_inject_bytes, uint32_t inject_range_bytes, + uint32_t *inject_offset) +{ + uint32_t inject_offset_blocks, inject_offset_bytes, inject_offset_bits; + uint32_t offset_blocks; + int rc; + + srand(time(0)); + + inject_offset_blocks = rand() % num_blocks; + inject_offset_bytes = start_inject_bytes + (rand() % inject_range_bytes); + inject_offset_bits = rand() % 8; + + for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { + if (offset_blocks == inject_offset_blocks) { + rc = _dif_inject_error(sgl, block_size, num_blocks, + inject_offset_blocks, + inject_offset_bytes, + inject_offset_bits); + if (rc == 0) { + *inject_offset = inject_offset_blocks; + } + return rc; + } + } + + return -1; +} + +#define _member_size(type, member) sizeof(((type *)0)->member) + +int +spdk_dif_inject_error(struct iovec *iovs, int iovcnt, uint32_t num_blocks, + const struct spdk_dif_ctx *ctx, uint32_t inject_flags, + uint32_t *inject_offset) +{ + struct _dif_sgl sgl; + int rc; + + _dif_sgl_init(&sgl, iovs, iovcnt); + + if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { + SPDK_ERRLOG("Size of iovec array is not valid.\n"); + return -EINVAL; + } + + if (inject_flags & SPDK_DIF_REFTAG_ERROR) { + rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, + ctx->guard_interval + offsetof(struct spdk_dif, ref_tag), + _member_size(struct spdk_dif, ref_tag), + inject_offset); + if (rc != 0) { + SPDK_ERRLOG("Failed to inject error to Reference Tag.\n"); + return rc; + } + } + + if (inject_flags & SPDK_DIF_APPTAG_ERROR) { + rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, + ctx->guard_interval + offsetof(struct spdk_dif, app_tag), + _member_size(struct spdk_dif, app_tag), + inject_offset); + if (rc != 0) { + SPDK_ERRLOG("Failed to inject error to Application Tag.\n"); + return rc; + } + } + if (inject_flags & SPDK_DIF_GUARD_ERROR) { + rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, + ctx->guard_interval, + _member_size(struct spdk_dif, guard), + inject_offset); + if (rc != 0) { + SPDK_ERRLOG("Failed to inject error to Guard.\n"); + return rc; + } + } + + if (inject_flags & SPDK_DIF_DATA_ERROR) { + /* If the DIF information is contained within the last 8 bytes of + * metadata, then the CRC covers all metadata bytes up to but excluding + * the last 8 bytes. But error injection does not cover these metadata + * because classification is not determined yet. + * + * Note: Error injection to data block is expected to be detected as + * guard error. + */ + rc = dif_inject_error(&sgl, ctx->block_size, num_blocks, + 0, + ctx->block_size - ctx->md_size, + inject_offset); + if (rc != 0) { + SPDK_ERRLOG("Failed to inject error to data block.\n"); + return rc; + } + } + + return 0; +} + +static void +dix_generate(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, + uint32_t num_blocks, const struct spdk_dif_ctx *ctx) +{ + uint32_t offset_blocks = 0; + uint16_t guard; + void *data_buf, *md_buf; + + while (offset_blocks < num_blocks) { + _dif_sgl_get_buf(data_sgl, &data_buf, NULL); + _dif_sgl_get_buf(md_sgl, &md_buf, NULL); + + guard = 0; + if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { + guard = spdk_crc16_t10dif(ctx->guard_seed, data_buf, ctx->block_size); + guard = spdk_crc16_t10dif(guard, md_buf, ctx->guard_interval); + } + + _dif_generate(md_buf + ctx->guard_interval, guard, offset_blocks, ctx); + + _dif_sgl_advance(data_sgl, ctx->block_size); + _dif_sgl_advance(md_sgl, ctx->md_size); + offset_blocks++; + } +} + +static void +_dix_generate_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, + uint32_t offset_blocks, const struct spdk_dif_ctx *ctx) +{ + uint32_t offset_in_block, data_buf_len; + uint16_t guard = 0; + void *data_buf, *md_buf; + + _dif_sgl_get_buf(md_sgl, &md_buf, NULL); + + if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { + guard = ctx->guard_seed; + } + offset_in_block = 0; + + while (offset_in_block < ctx->block_size) { + _dif_sgl_get_buf(data_sgl, &data_buf, &data_buf_len); + data_buf_len = spdk_min(data_buf_len, ctx->block_size - offset_in_block); + + if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { + guard = spdk_crc16_t10dif(guard, data_buf, data_buf_len); + } + + _dif_sgl_advance(data_sgl, data_buf_len); + offset_in_block += data_buf_len; + } + + if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { + guard = spdk_crc16_t10dif(guard, md_buf, ctx->guard_interval); + } + + _dif_sgl_advance(md_sgl, ctx->md_size); + + _dif_generate(md_buf + ctx->guard_interval, guard, offset_blocks, ctx); +} + +static void +dix_generate_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, + uint32_t num_blocks, const struct spdk_dif_ctx *ctx) +{ + uint32_t offset_blocks; + + for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { + _dix_generate_split(data_sgl, md_sgl, offset_blocks, ctx); + } +} + +int +spdk_dix_generate(struct iovec *iovs, int iovcnt, struct iovec *md_iov, + uint32_t num_blocks, const struct spdk_dif_ctx *ctx) +{ + struct _dif_sgl data_sgl, md_sgl; + + _dif_sgl_init(&data_sgl, iovs, iovcnt); + _dif_sgl_init(&md_sgl, md_iov, 1); + + if (!_dif_sgl_is_valid(&data_sgl, ctx->block_size * num_blocks) || + !_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) { + SPDK_ERRLOG("Size of iovec array is not valid.\n"); + return -EINVAL; + } + + if (_dif_is_disabled(ctx->dif_type)) { + return 0; + } + + if (_dif_sgl_is_bytes_multiple(&data_sgl, ctx->block_size)) { + dix_generate(&data_sgl, &md_sgl, num_blocks, ctx); + } else { + dix_generate_split(&data_sgl, &md_sgl, num_blocks, ctx); + } + + return 0; +} + +static int +dix_verify(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, + uint32_t num_blocks, const struct spdk_dif_ctx *ctx, + struct spdk_dif_error *err_blk) +{ + uint32_t offset_blocks = 0; + uint16_t guard; + void *data_buf, *md_buf; + int rc; + + while (offset_blocks < num_blocks) { + _dif_sgl_get_buf(data_sgl, &data_buf, NULL); + _dif_sgl_get_buf(md_sgl, &md_buf, NULL); + + guard = 0; + if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { + guard = spdk_crc16_t10dif(ctx->guard_seed, data_buf, ctx->block_size); + guard = spdk_crc16_t10dif(guard, md_buf, ctx->guard_interval); + } + + rc = _dif_verify(md_buf + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); + if (rc != 0) { + return rc; + } + + _dif_sgl_advance(data_sgl, ctx->block_size); + _dif_sgl_advance(md_sgl, ctx->md_size); + offset_blocks++; + } + + return 0; +} + +static int +_dix_verify_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, + uint32_t offset_blocks, const struct spdk_dif_ctx *ctx, + struct spdk_dif_error *err_blk) +{ + uint32_t offset_in_block, data_buf_len; + uint16_t guard = 0; + void *data_buf, *md_buf; + + _dif_sgl_get_buf(md_sgl, &md_buf, NULL); + + if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { + guard = ctx->guard_seed; + } + offset_in_block = 0; + + while (offset_in_block < ctx->block_size) { + _dif_sgl_get_buf(data_sgl, &data_buf, &data_buf_len); + data_buf_len = spdk_min(data_buf_len, ctx->block_size - offset_in_block); + + if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { + guard = spdk_crc16_t10dif(guard, data_buf, data_buf_len); + } + + _dif_sgl_advance(data_sgl, data_buf_len); + offset_in_block += data_buf_len; + } + + if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { + guard = spdk_crc16_t10dif(guard, md_buf, ctx->guard_interval); + } + + _dif_sgl_advance(md_sgl, ctx->md_size); + + return _dif_verify(md_buf + ctx->guard_interval, guard, offset_blocks, ctx, err_blk); +} + +static int +dix_verify_split(struct _dif_sgl *data_sgl, struct _dif_sgl *md_sgl, + uint32_t num_blocks, const struct spdk_dif_ctx *ctx, + struct spdk_dif_error *err_blk) +{ + uint32_t offset_blocks; + int rc; + + for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { + rc = _dix_verify_split(data_sgl, md_sgl, offset_blocks, ctx, err_blk); + if (rc != 0) { + return rc; + } + } + + return 0; +} + +int +spdk_dix_verify(struct iovec *iovs, int iovcnt, struct iovec *md_iov, + uint32_t num_blocks, const struct spdk_dif_ctx *ctx, + struct spdk_dif_error *err_blk) +{ + struct _dif_sgl data_sgl, md_sgl; + + _dif_sgl_init(&data_sgl, iovs, iovcnt); + _dif_sgl_init(&md_sgl, md_iov, 1); + + if (!_dif_sgl_is_valid(&data_sgl, ctx->block_size * num_blocks) || + !_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) { + SPDK_ERRLOG("Size of iovec array is not valid.\n"); + return -EINVAL; + } + + if (_dif_is_disabled(ctx->dif_type)) { + return 0; + } + + if (_dif_sgl_is_bytes_multiple(&data_sgl, ctx->block_size)) { + return dix_verify(&data_sgl, &md_sgl, num_blocks, ctx, err_blk); + } else { + return dix_verify_split(&data_sgl, &md_sgl, num_blocks, ctx, err_blk); + } +} + +int +spdk_dix_inject_error(struct iovec *iovs, int iovcnt, struct iovec *md_iov, + uint32_t num_blocks, const struct spdk_dif_ctx *ctx, + uint32_t inject_flags, uint32_t *inject_offset) +{ + struct _dif_sgl data_sgl, md_sgl; + int rc; + + _dif_sgl_init(&data_sgl, iovs, iovcnt); + _dif_sgl_init(&md_sgl, md_iov, 1); + + if (!_dif_sgl_is_valid(&data_sgl, ctx->block_size * num_blocks) || + !_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) { + SPDK_ERRLOG("Size of iovec array is not valid.\n"); + return -EINVAL; + } + + if (inject_flags & SPDK_DIF_REFTAG_ERROR) { + rc = dif_inject_error(&md_sgl, ctx->md_size, num_blocks, + ctx->guard_interval + offsetof(struct spdk_dif, ref_tag), + _member_size(struct spdk_dif, ref_tag), + inject_offset); + if (rc != 0) { + SPDK_ERRLOG("Failed to inject error to Reference Tag.\n"); + return rc; + } + } + + if (inject_flags & SPDK_DIF_APPTAG_ERROR) { + rc = dif_inject_error(&md_sgl, ctx->md_size, num_blocks, + ctx->guard_interval + offsetof(struct spdk_dif, app_tag), + _member_size(struct spdk_dif, app_tag), + inject_offset); + if (rc != 0) { + SPDK_ERRLOG("Failed to inject error to Application Tag.\n"); + return rc; + } + } + + if (inject_flags & SPDK_DIF_GUARD_ERROR) { + rc = dif_inject_error(&md_sgl, ctx->md_size, num_blocks, + ctx->guard_interval, + _member_size(struct spdk_dif, guard), + inject_offset); + if (rc != 0) { + SPDK_ERRLOG("Failed to inject error to Guard.\n"); + return rc; + } + } + + if (inject_flags & SPDK_DIF_DATA_ERROR) { + /* Note: Error injection to data block is expected to be detected + * as guard error. + */ + rc = dif_inject_error(&data_sgl, ctx->block_size, num_blocks, + 0, + ctx->block_size, + inject_offset); + if (rc != 0) { + SPDK_ERRLOG("Failed to inject error to Guard.\n"); + return rc; + } + } + + return 0; +} + +static uint32_t +_to_next_boundary(uint32_t offset, uint32_t boundary) +{ + return boundary - (offset % boundary); +} + +static uint32_t +_to_size_with_md(uint32_t size, uint32_t data_block_size, uint32_t block_size) +{ + return (size / data_block_size) * block_size + (size % data_block_size); +} + +int +spdk_dif_set_md_interleave_iovs(struct iovec *iovs, int iovcnt, + struct iovec *buf_iovs, int buf_iovcnt, + uint32_t data_offset, uint32_t data_len, + uint32_t *_mapped_len, + const struct spdk_dif_ctx *ctx) +{ + uint32_t data_block_size, data_unalign, buf_len, buf_offset, len; + struct _dif_sgl dif_sgl; + struct _dif_sgl buf_sgl; + + if (iovs == NULL || iovcnt == 0 || buf_iovs == NULL || buf_iovcnt == 0) { + return -EINVAL; + } + + data_block_size = ctx->block_size - ctx->md_size; + + data_unalign = ctx->data_offset % data_block_size; + + buf_len = _to_size_with_md(data_unalign + data_offset + data_len, data_block_size, + ctx->block_size); + buf_len -= data_unalign; + + _dif_sgl_init(&dif_sgl, iovs, iovcnt); + _dif_sgl_init(&buf_sgl, buf_iovs, buf_iovcnt); + + if (!_dif_sgl_is_valid(&buf_sgl, buf_len)) { + SPDK_ERRLOG("Buffer overflow will occur.\n"); + return -ERANGE; + } + + buf_offset = _to_size_with_md(data_unalign + data_offset, data_block_size, ctx->block_size); + buf_offset -= data_unalign; + + _dif_sgl_advance(&buf_sgl, buf_offset); + + while (data_len != 0) { + len = spdk_min(data_len, _to_next_boundary(ctx->data_offset + data_offset, data_block_size)); + if (!_dif_sgl_append_split(&dif_sgl, &buf_sgl, len)) { + break; + } + _dif_sgl_advance(&buf_sgl, ctx->md_size); + data_offset += len; + data_len -= len; + } + + if (_mapped_len != NULL) { + *_mapped_len = dif_sgl.total_size; + } + + return iovcnt - dif_sgl.iovcnt; +} + +static int +_dif_sgl_setup_stream(struct _dif_sgl *sgl, uint32_t *_buf_offset, uint32_t *_buf_len, + uint32_t data_offset, uint32_t data_len, + const struct spdk_dif_ctx *ctx) +{ + uint32_t data_block_size, data_unalign, buf_len, buf_offset; + + data_block_size = ctx->block_size - ctx->md_size; + + data_unalign = ctx->data_offset % data_block_size; + + /* If the last data block is complete, DIF of the data block is + * inserted or verified in this turn. + */ + buf_len = _to_size_with_md(data_unalign + data_offset + data_len, data_block_size, + ctx->block_size); + buf_len -= data_unalign; + + if (!_dif_sgl_is_valid(sgl, buf_len)) { + return -ERANGE; + } + + buf_offset = _to_size_with_md(data_unalign + data_offset, data_block_size, ctx->block_size); + buf_offset -= data_unalign; + + _dif_sgl_advance(sgl, buf_offset); + buf_len -= buf_offset; + + buf_offset += data_unalign; + + *_buf_offset = buf_offset; + *_buf_len = buf_len; + + return 0; +} + +int +spdk_dif_generate_stream(struct iovec *iovs, int iovcnt, + uint32_t data_offset, uint32_t data_len, + struct spdk_dif_ctx *ctx) +{ + uint32_t buf_len = 0, buf_offset = 0; + uint32_t len, offset_in_block, offset_blocks; + uint16_t guard = 0; + struct _dif_sgl sgl; + int rc; + + if (iovs == NULL || iovcnt == 0) { + return -EINVAL; + } + + if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { + guard = ctx->last_guard; + } + + _dif_sgl_init(&sgl, iovs, iovcnt); + + rc = _dif_sgl_setup_stream(&sgl, &buf_offset, &buf_len, data_offset, data_len, ctx); + if (rc != 0) { + return rc; + } + + while (buf_len != 0) { + len = spdk_min(buf_len, _to_next_boundary(buf_offset, ctx->block_size)); + offset_in_block = buf_offset % ctx->block_size; + offset_blocks = buf_offset / ctx->block_size; + + guard = _dif_generate_split(&sgl, offset_in_block, len, guard, offset_blocks, ctx); + + buf_len -= len; + buf_offset += len; + } + + if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { + ctx->last_guard = guard; + } + + return 0; +} + +int +spdk_dif_verify_stream(struct iovec *iovs, int iovcnt, + uint32_t data_offset, uint32_t data_len, + struct spdk_dif_ctx *ctx, + struct spdk_dif_error *err_blk) +{ + uint32_t buf_len = 0, buf_offset = 0; + uint32_t len, offset_in_block, offset_blocks; + uint16_t guard = 0; + struct _dif_sgl sgl; + int rc = 0; + + if (iovs == NULL || iovcnt == 0) { + return -EINVAL; + } + + if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { + guard = ctx->last_guard; + } + + _dif_sgl_init(&sgl, iovs, iovcnt); + + rc = _dif_sgl_setup_stream(&sgl, &buf_offset, &buf_len, data_offset, data_len, ctx); + if (rc != 0) { + return rc; + } + + while (buf_len != 0) { + len = spdk_min(buf_len, _to_next_boundary(buf_offset, ctx->block_size)); + offset_in_block = buf_offset % ctx->block_size; + offset_blocks = buf_offset / ctx->block_size; + + rc = _dif_verify_split(&sgl, offset_in_block, len, &guard, offset_blocks, + ctx, err_blk); + if (rc != 0) { + goto error; + } + + buf_len -= len; + buf_offset += len; + } + + if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) { + ctx->last_guard = guard; + } +error: + return rc; +} + +int +spdk_dif_update_crc32c_stream(struct iovec *iovs, int iovcnt, + uint32_t data_offset, uint32_t data_len, + uint32_t *_crc32c, const struct spdk_dif_ctx *ctx) +{ + uint32_t buf_len = 0, buf_offset = 0, len, offset_in_block; + uint32_t crc32c; + struct _dif_sgl sgl; + int rc; + + if (iovs == NULL || iovcnt == 0) { + return -EINVAL; + } + + crc32c = *_crc32c; + _dif_sgl_init(&sgl, iovs, iovcnt); + + rc = _dif_sgl_setup_stream(&sgl, &buf_offset, &buf_len, data_offset, data_len, ctx); + if (rc != 0) { + return rc; + } + + while (buf_len != 0) { + len = spdk_min(buf_len, _to_next_boundary(buf_offset, ctx->block_size)); + offset_in_block = buf_offset % ctx->block_size; + + crc32c = _dif_update_crc32c_split(&sgl, offset_in_block, len, crc32c, ctx); + + buf_len -= len; + buf_offset += len; + } + + *_crc32c = crc32c; + + return 0; +} + +void +spdk_dif_get_range_with_md(uint32_t data_offset, uint32_t data_len, + uint32_t *_buf_offset, uint32_t *_buf_len, + const struct spdk_dif_ctx *ctx) +{ + uint32_t data_block_size, data_unalign, buf_offset, buf_len; + + if (!ctx->md_interleave) { + buf_offset = data_offset; + buf_len = data_len; + } else { + data_block_size = ctx->block_size - ctx->md_size; + + data_unalign = data_offset % data_block_size; + + buf_offset = _to_size_with_md(data_offset, data_block_size, ctx->block_size); + buf_len = _to_size_with_md(data_unalign + data_len, data_block_size, ctx->block_size) - + data_unalign; + } + + if (_buf_offset != NULL) { + *_buf_offset = buf_offset; + } + + if (_buf_len != NULL) { + *_buf_len = buf_len; + } +} + +uint32_t +spdk_dif_get_length_with_md(uint32_t data_len, const struct spdk_dif_ctx *ctx) +{ + uint32_t data_block_size; + + if (!ctx->md_interleave) { + return data_len; + } else { + data_block_size = ctx->block_size - ctx->md_size; + + return _to_size_with_md(data_len, data_block_size, ctx->block_size); + } +} + +static int +_dif_remap_ref_tag(struct _dif_sgl *sgl, uint32_t offset_blocks, + const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) +{ + uint32_t offset, buf_len, expected = 0, _actual, remapped; + void *buf; + struct _dif_sgl tmp_sgl; + struct spdk_dif dif; + + /* Fast forward to DIF field. */ + _dif_sgl_advance(sgl, ctx->guard_interval); + _dif_sgl_copy(&tmp_sgl, sgl); + + /* Copy the split DIF field to the temporary DIF buffer */ + offset = 0; + while (offset < sizeof(struct spdk_dif)) { + _dif_sgl_get_buf(sgl, &buf, &buf_len); + buf_len = spdk_min(buf_len, sizeof(struct spdk_dif) - offset); + + memcpy((uint8_t *)&dif + offset, buf, buf_len); + + _dif_sgl_advance(sgl, buf_len); + offset += buf_len; + } + + switch (ctx->dif_type) { + case SPDK_DIF_TYPE1: + case SPDK_DIF_TYPE2: + /* If Type 1 or 2 is used, then all DIF checks are disabled when + * the Application Tag is 0xFFFF. + */ + if (dif.app_tag == 0xFFFF) { + goto end; + } + break; + case SPDK_DIF_TYPE3: + /* If Type 3 is used, then all DIF checks are disabled when the + * Application Tag is 0xFFFF and the Reference Tag is 0xFFFFFFFF. + */ + if (dif.app_tag == 0xFFFF && dif.ref_tag == 0xFFFFFFFF) { + goto end; + } + break; + default: + break; + } + + /* For type 1 and 2, the Reference Tag is incremented for each + * subsequent logical block. For type 3, the Reference Tag + * remains the same as the initial Reference Tag. + */ + if (ctx->dif_type != SPDK_DIF_TYPE3) { + expected = ctx->init_ref_tag + ctx->ref_tag_offset + offset_blocks; + remapped = ctx->remapped_init_ref_tag + ctx->ref_tag_offset + offset_blocks; + } else { + remapped = ctx->remapped_init_ref_tag; + } + + /* Verify the stored Reference Tag. */ + switch (ctx->dif_type) { + case SPDK_DIF_TYPE1: + case SPDK_DIF_TYPE2: + /* Compare the DIF Reference Tag field to the computed Reference Tag. + * The computed Reference Tag will be the least significant 4 bytes + * of the LBA when Type 1 is used, and application specific value + * if Type 2 is used. + */ + _actual = from_be32(&dif.ref_tag); + if (_actual != expected) { + _dif_error_set(err_blk, SPDK_DIF_REFTAG_ERROR, expected, + _actual, offset_blocks); + SPDK_ERRLOG("Failed to compare Ref Tag: LBA=%" PRIu32 "," \ + " Expected=%x, Actual=%x\n", + expected, expected, _actual); + return -1; + } + break; + case SPDK_DIF_TYPE3: + /* For type 3, the computed Reference Tag remains unchanged. + * Hence ignore the Reference Tag field. + */ + break; + default: + break; + } + + /* Update the stored Reference Tag to the remapped one. */ + to_be32(&dif.ref_tag, remapped); + + offset = 0; + while (offset < sizeof(struct spdk_dif)) { + _dif_sgl_get_buf(&tmp_sgl, &buf, &buf_len); + buf_len = spdk_min(buf_len, sizeof(struct spdk_dif) - offset); + + memcpy(buf, (uint8_t *)&dif + offset, buf_len); + + _dif_sgl_advance(&tmp_sgl, buf_len); + offset += buf_len; + } + +end: + _dif_sgl_advance(sgl, ctx->block_size - ctx->guard_interval - sizeof(struct spdk_dif)); + + return 0; +} + +int +spdk_dif_remap_ref_tag(struct iovec *iovs, int iovcnt, uint32_t num_blocks, + const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) +{ + struct _dif_sgl sgl; + uint32_t offset_blocks; + int rc; + + _dif_sgl_init(&sgl, iovs, iovcnt); + + if (!_dif_sgl_is_valid(&sgl, ctx->block_size * num_blocks)) { + SPDK_ERRLOG("Size of iovec array is not valid.\n"); + return -EINVAL; + } + + if (_dif_is_disabled(ctx->dif_type)) { + return 0; + } + + if (!(ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK)) { + return 0; + } + + for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { + rc = _dif_remap_ref_tag(&sgl, offset_blocks, ctx, err_blk); + if (rc != 0) { + return rc; + } + } + + return 0; +} + +static int +_dix_remap_ref_tag(struct _dif_sgl *md_sgl, uint32_t offset_blocks, + const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk) +{ + uint32_t expected = 0, _actual, remapped; + uint8_t *md_buf; + struct spdk_dif *dif; + + _dif_sgl_get_buf(md_sgl, (void *)&md_buf, NULL); + + dif = (struct spdk_dif *)(md_buf + ctx->guard_interval); + + switch (ctx->dif_type) { + case SPDK_DIF_TYPE1: + case SPDK_DIF_TYPE2: + /* If Type 1 or 2 is used, then all DIF checks are disabled when + * the Application Tag is 0xFFFF. + */ + if (dif->app_tag == 0xFFFF) { + goto end; + } + break; + case SPDK_DIF_TYPE3: + /* If Type 3 is used, then all DIF checks are disabled when the + * Application Tag is 0xFFFF and the Reference Tag is 0xFFFFFFFF. + */ + if (dif->app_tag == 0xFFFF && dif->ref_tag == 0xFFFFFFFF) { + goto end; + } + break; + default: + break; + } + + /* For type 1 and 2, the Reference Tag is incremented for each + * subsequent logical block. For type 3, the Reference Tag + * remains the same as the initialReference Tag. + */ + if (ctx->dif_type != SPDK_DIF_TYPE3) { + expected = ctx->init_ref_tag + ctx->ref_tag_offset + offset_blocks; + remapped = ctx->remapped_init_ref_tag + ctx->ref_tag_offset + offset_blocks; + } else { + remapped = ctx->remapped_init_ref_tag; + } + + /* Verify the stored Reference Tag. */ + switch (ctx->dif_type) { + case SPDK_DIF_TYPE1: + case SPDK_DIF_TYPE2: + /* Compare the DIF Reference Tag field to the computed Reference Tag. + * The computed Reference Tag will be the least significant 4 bytes + * of the LBA when Type 1 is used, and application specific value + * if Type 2 is used. + */ + _actual = from_be32(&dif->ref_tag); + if (_actual != expected) { + _dif_error_set(err_blk, SPDK_DIF_REFTAG_ERROR, expected, + _actual, offset_blocks); + SPDK_ERRLOG("Failed to compare Ref Tag: LBA=%" PRIu32 "," \ + " Expected=%x, Actual=%x\n", + expected, expected, _actual); + return -1; + } + break; + case SPDK_DIF_TYPE3: + /* For type 3, the computed Reference Tag remains unchanged. + * Hence ignore the Reference Tag field. + */ + break; + default: + break; + } + + /* Update the stored Reference Tag to the remapped one. */ + to_be32(&dif->ref_tag, remapped); + +end: + _dif_sgl_advance(md_sgl, ctx->md_size); + + return 0; +} + +int +spdk_dix_remap_ref_tag(struct iovec *md_iov, uint32_t num_blocks, + const struct spdk_dif_ctx *ctx, + struct spdk_dif_error *err_blk) +{ + struct _dif_sgl md_sgl; + uint32_t offset_blocks; + int rc; + + _dif_sgl_init(&md_sgl, md_iov, 1); + + if (!_dif_sgl_is_valid(&md_sgl, ctx->md_size * num_blocks)) { + SPDK_ERRLOG("Size of metadata iovec array is not valid.\n"); + return -EINVAL; + } + + if (_dif_is_disabled(ctx->dif_type)) { + return 0; + } + + if (!(ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK)) { + return 0; + } + + for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) { + rc = _dix_remap_ref_tag(&md_sgl, offset_blocks, ctx, err_blk); + if (rc != 0) { + return rc; + } + } + + return 0; +} diff --git a/src/spdk/lib/util/fd.c b/src/spdk/lib/util/fd.c new file mode 100644 index 000000000..6b0d0d554 --- /dev/null +++ b/src/spdk/lib/util/fd.c @@ -0,0 +1,103 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "spdk/stdinc.h" + +#include "spdk/fd.h" + +#ifdef __linux__ +#include +#endif + +static uint64_t +dev_get_size(int fd) +{ +#if defined(DIOCGMEDIASIZE) /* FreeBSD */ + off_t size; + + if (ioctl(fd, DIOCGMEDIASIZE, &size) == 0) { + return size; + } +#elif defined(__linux__) && defined(BLKGETSIZE64) + uint64_t size; + + if (ioctl(fd, BLKGETSIZE64, &size) == 0) { + return size; + } +#endif + + return 0; +} + +uint32_t +spdk_fd_get_blocklen(int fd) +{ +#if defined(DKIOCGETBLOCKSIZE) /* FreeBSD */ + uint32_t blocklen; + + if (ioctl(fd, DKIOCGETBLOCKSIZE, &blocklen) == 0) { + return blocklen; + } +#elif defined(__linux__) && defined(BLKSSZGET) + uint32_t blocklen; + + if (ioctl(fd, BLKSSZGET, &blocklen) == 0) { + return blocklen; + } +#endif + + return 0; +} + +uint64_t +spdk_fd_get_size(int fd) +{ + struct stat st; + + if (fstat(fd, &st) != 0) { + return 0; + } + + if (S_ISLNK(st.st_mode)) { + return 0; + } + + if (S_ISBLK(st.st_mode) || S_ISCHR(st.st_mode)) { + return dev_get_size(fd); + } else if (S_ISREG(st.st_mode)) { + return st.st_size; + } + + /* Not REG, CHR or BLK */ + return 0; +} diff --git a/src/spdk/lib/util/file.c b/src/spdk/lib/util/file.c new file mode 100644 index 000000000..2ba08547b --- /dev/null +++ b/src/spdk/lib/util/file.c @@ -0,0 +1,71 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "spdk/file.h" + +void * +spdk_posix_file_load(FILE *file, size_t *size) +{ + void *newbuf, *buf = NULL; + size_t rc, buf_size, cur_size = 0; + + *size = 0; + buf_size = 128 * 1024; + + while (buf_size <= 1024 * 1024 * 1024) { + newbuf = realloc(buf, buf_size); + if (newbuf == NULL) { + free(buf); + return NULL; + } + buf = newbuf; + + rc = fread(buf + cur_size, 1, buf_size - cur_size, file); + cur_size += rc; + + if (feof(file)) { + *size = cur_size; + return buf; + } + + if (ferror(file)) { + free(buf); + return NULL; + } + + buf_size *= 2; + } + + free(buf); + return NULL; +} diff --git a/src/spdk/lib/util/iov.c b/src/spdk/lib/util/iov.c new file mode 100644 index 000000000..e89ef9d21 --- /dev/null +++ b/src/spdk/lib/util/iov.c @@ -0,0 +1,111 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "spdk/util.h" + +size_t +spdk_iovcpy(struct iovec *siov, size_t siovcnt, struct iovec *diov, size_t diovcnt) +{ + size_t total_sz; + size_t sidx; + size_t didx; + int siov_len; + uint8_t *siov_base; + int diov_len; + uint8_t *diov_base; + + /* d prefix = destination. s prefix = source. */ + + assert(diovcnt > 0); + assert(siovcnt > 0); + + total_sz = 0; + sidx = 0; + didx = 0; + siov_len = siov[0].iov_len; + siov_base = siov[0].iov_base; + diov_len = diov[0].iov_len; + diov_base = diov[0].iov_base; + while (siov_len > 0 && diov_len > 0) { + if (siov_len == diov_len) { + memcpy(diov_base, siov_base, siov_len); + total_sz += siov_len; + + /* Advance both iovs to the next element */ + sidx++; + if (sidx == siovcnt) { + break; + } + + didx++; + if (didx == diovcnt) { + break; + } + + siov_len = siov[sidx].iov_len; + siov_base = siov[sidx].iov_base; + diov_len = diov[didx].iov_len; + diov_base = diov[didx].iov_base; + } else if (siov_len < diov_len) { + memcpy(diov_base, siov_base, siov_len); + total_sz += siov_len; + + /* Advance only the source to the next element */ + sidx++; + if (sidx == siovcnt) { + break; + } + + diov_base += siov_len; + diov_len -= siov_len; + siov_len = siov[sidx].iov_len; + siov_base = siov[sidx].iov_base; + } else { + memcpy(diov_base, siov_base, diov_len); + total_sz += diov_len; + + /* Advance only the destination to the next element */ + didx++; + if (didx == diovcnt) { + break; + } + + siov_base += diov_len; + siov_len -= diov_len; + diov_len = diov[didx].iov_len; + diov_base = diov[didx].iov_base; + } + } + + return total_sz; +} diff --git a/src/spdk/lib/util/math.c b/src/spdk/lib/util/math.c new file mode 100644 index 000000000..7d1852421 --- /dev/null +++ b/src/spdk/lib/util/math.c @@ -0,0 +1,69 @@ +/*- + * BSD LICENSE + * + * Copyright(c) Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "spdk/stdinc.h" +#include "spdk/util.h" + +/* The following will automatically generate several version of + * this function, targeted at different architectures. This + * is only supported by GCC 6 or newer. */ +#if defined(__GNUC__) && __GNUC__ >= 6 && !defined(__clang__) \ + && (defined(__i386__) || defined(__x86_64__)) +__attribute__((target_clones("bmi", "arch=core2", "arch=atom", "default"))) +#endif +uint32_t +spdk_u32log2(uint32_t x) +{ + if (x == 0) { + /* log(0) is undefined */ + return 0; + } + return 31u - __builtin_clz(x); +} + +/* The following will automatically generate several version of + * this function, targeted at different architectures. This + * is only supported by GCC 6 or newer. */ +#if defined(__GNUC__) && __GNUC__ >= 6 && !defined(__clang__) \ + && (defined(__i386__) || defined(__x86_64__)) +__attribute__((target_clones("bmi", "arch=core2", "arch=atom", "default"))) +#endif +uint64_t +spdk_u64log2(uint64_t x) +{ + if (x == 0) { + /* log(0) is undefined */ + return 0; + } + return 63u - __builtin_clzl(x); +} diff --git a/src/spdk/lib/util/pipe.c b/src/spdk/lib/util/pipe.c new file mode 100644 index 000000000..1c640dd2e --- /dev/null +++ b/src/spdk/lib/util/pipe.c @@ -0,0 +1,246 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "spdk/pipe.h" +#include "spdk/util.h" + +struct spdk_pipe { + uint8_t *buf; + uint32_t sz; + + uint32_t write; + uint32_t read; +}; + +struct spdk_pipe * +spdk_pipe_create(void *buf, uint32_t sz) +{ + struct spdk_pipe *pipe; + + pipe = calloc(1, sizeof(*pipe)); + if (pipe == NULL) { + return NULL; + } + + pipe->buf = buf; + pipe->sz = sz; + + return pipe; +} + +void +spdk_pipe_destroy(struct spdk_pipe *pipe) +{ + free(pipe); +} + +int +spdk_pipe_writer_get_buffer(struct spdk_pipe *pipe, uint32_t requested_sz, struct iovec *iovs) +{ + uint32_t sz; + uint32_t read; + uint32_t write; + + read = pipe->read; + write = pipe->write; + + if (read <= write) { + requested_sz = spdk_min(requested_sz, ((read + pipe->sz) - write - 1)); + + sz = spdk_min(requested_sz, pipe->sz - write); + + iovs[0].iov_base = (sz == 0) ? NULL : (pipe->buf + write); + iovs[0].iov_len = sz; + + requested_sz -= sz; + + if (requested_sz > 0) { + sz = spdk_min(requested_sz, read); + + iovs[1].iov_base = (sz == 0) ? NULL : pipe->buf; + iovs[1].iov_len = sz; + } else { + iovs[1].iov_base = NULL; + iovs[1].iov_len = 0; + } + } else { + sz = spdk_min(requested_sz, read - write - 1); + + iovs[0].iov_base = (sz == 0) ? NULL : (pipe->buf + write); + iovs[0].iov_len = sz; + iovs[1].iov_base = NULL; + iovs[1].iov_len = 0; + } + + return iovs[0].iov_len + iovs[1].iov_len; +} + +int +spdk_pipe_writer_advance(struct spdk_pipe *pipe, uint32_t requested_sz) +{ + uint32_t sz; + uint32_t read; + uint32_t write; + + read = pipe->read; + write = pipe->write; + + if (requested_sz > pipe->sz - 1) { + return -EINVAL; + } + + if (read <= write) { + if (requested_sz > (read + pipe->sz) - write) { + return -EINVAL; + } + + sz = spdk_min(requested_sz, pipe->sz - write); + + write += sz; + if (write > pipe->sz - 1) { + write = 0; + } + requested_sz -= sz; + + if (requested_sz > 0) { + if (requested_sz >= read) { + return -EINVAL; + } + + write = requested_sz; + } + } else { + if (requested_sz > (read - write - 1)) { + return -EINVAL; + } + + write += requested_sz; + } + + pipe->write = write; + + return 0; +} + +uint32_t +spdk_pipe_reader_bytes_available(struct spdk_pipe *pipe) +{ + uint32_t read; + uint32_t write; + + read = pipe->read; + write = pipe->write; + + if (read <= write) { + return write - read; + } + + return (write + pipe->sz) - read; +} + +int +spdk_pipe_reader_get_buffer(struct spdk_pipe *pipe, uint32_t requested_sz, struct iovec *iovs) +{ + uint32_t sz; + uint32_t read; + uint32_t write; + + read = pipe->read; + write = pipe->write; + + if (read <= write) { + sz = spdk_min(requested_sz, write - read); + + iovs[0].iov_base = (sz == 0) ? NULL : (pipe->buf + read); + iovs[0].iov_len = sz; + iovs[1].iov_base = NULL; + iovs[1].iov_len = 0; + } else { + sz = spdk_min(requested_sz, pipe->sz - read); + + iovs[0].iov_base = (sz == 0) ? NULL : (pipe->buf + read); + iovs[0].iov_len = sz; + + requested_sz -= sz; + + if (requested_sz > 0) { + sz = spdk_min(requested_sz, write); + iovs[1].iov_base = (sz == 0) ? NULL : pipe->buf; + iovs[1].iov_len = sz; + } else { + iovs[1].iov_base = NULL; + iovs[1].iov_len = 0; + } + } + + return iovs[0].iov_len + iovs[1].iov_len; +} + +int +spdk_pipe_reader_advance(struct spdk_pipe *pipe, uint32_t requested_sz) +{ + uint32_t sz; + uint32_t read; + uint32_t write; + + read = pipe->read; + write = pipe->write; + + if (read <= write) { + if (requested_sz > (write - read)) { + return -EINVAL; + } + + read += requested_sz; + } else { + sz = spdk_min(requested_sz, pipe->sz - read); + + read += sz; + if (read > pipe->sz - 1) { + read = 0; + } + requested_sz -= sz; + + if (requested_sz > 0) { + if (requested_sz > write) { + return -EINVAL; + } + + read = requested_sz; + } + } + + pipe->read = read; + + return 0; +} diff --git a/src/spdk/lib/util/spdk_util.map b/src/spdk/lib/util/spdk_util.map new file mode 100644 index 000000000..07e067faa --- /dev/null +++ b/src/spdk/lib/util/spdk_util.map @@ -0,0 +1,128 @@ +{ + global: + + # public functions in base64.h + spdk_base64_encode; + spdk_base64_urlsafe_encode; + spdk_base64_decode; + spdk_base64_urlsafe_decode; + + # public functions in bit_array.h + spdk_bit_array_capacity; + spdk_bit_array_create; + spdk_bit_array_free; + spdk_bit_array_resize; + spdk_bit_array_get; + spdk_bit_array_set; + spdk_bit_array_clear; + spdk_bit_array_find_first_set; + spdk_bit_array_find_first_clear; + spdk_bit_array_count_set; + spdk_bit_array_count_clear; + spdk_bit_array_store_mask; + spdk_bit_array_load_mask; + spdk_bit_array_clear_mask; + + # public functions in cpuset.h + spdk_cpuset_alloc; + spdk_cpuset_free; + spdk_cpuset_equal; + spdk_cpuset_copy; + spdk_cpuset_and; + spdk_cpuset_or; + spdk_cpuset_xor; + spdk_cpuset_negate; + spdk_cpuset_zero; + spdk_cpuset_set_cpu; + spdk_cpuset_get_cpu; + spdk_cpuset_count; + spdk_cpuset_fmt; + spdk_cpuset_parse; + + # public functions in crc16.h + spdk_crc16_t10dif; + spdk_crc16_t10dif_copy; + + # public functions in crc32.h + spdk_crc32_ieee_update; + spdk_crc32c_update; + + # public functions in dif.h + spdk_dif_ctx_init; + spdk_dif_ctx_set_data_offset; + spdk_dif_ctx_set_remapped_init_ref_tag; + spdk_dif_generate; + spdk_dif_verify; + spdk_dif_update_crc32c; + spdk_dif_generate_copy; + spdk_dif_verify_copy; + spdk_dif_inject_error; + spdk_dix_generate; + spdk_dix_verify; + spdk_dix_inject_error; + spdk_dif_set_md_interleave_iovs; + spdk_dif_generate_stream; + spdk_dif_verify_stream; + spdk_dif_update_crc32c_stream; + spdk_dif_get_range_with_md; + spdk_dif_get_length_with_md; + spdk_dif_remap_ref_tag; + spdk_dix_remap_ref_tag; + + # public functions in fd.h + spdk_fd_get_size; + spdk_fd_get_blocklen; + + # public functions in file.h + spdk_posix_file_load; + + # public functions in pipe.h + spdk_pipe_create; + spdk_pipe_destroy; + spdk_pipe_writer_get_buffer; + spdk_pipe_writer_advance; + spdk_pipe_reader_bytes_available; + spdk_pipe_reader_get_buffer; + spdk_pipe_reader_advance; + + # public functions in string.h + spdk_sprintf_alloc; + spdk_vsprintf_alloc; + spdk_sprintf_append_realloc; + spdk_vsprintf_append_realloc; + spdk_strlwr; + spdk_strsepq; + spdk_str_trim; + spdk_strerror_r; + spdk_strerror; + spdk_str_chomp; + spdk_strcpy_pad; + spdk_strlen_pad; + spdk_parse_ip_addr; + spdk_parse_capacity; + spdk_mem_all_zero; + spdk_strtol; + spdk_strtoll; + + # public functions in util.h + spdk_u32log2; + spdk_u64log2; + spdk_iovcpy; + + # resolvers for functions in util.h + spdk_u32log2.resolver; + spdk_u64log2.resolver; + + # public functions in uuid.h + spdk_uuid_parse; + spdk_uuid_fmt_lower; + spdk_uuid_compare; + spdk_uuid_generate; + spdk_uuid_copy; + + + + + + local: *; +}; diff --git a/src/spdk/lib/util/strerror_tls.c b/src/spdk/lib/util/strerror_tls.c new file mode 100644 index 000000000..c9dc8f13f --- /dev/null +++ b/src/spdk/lib/util/strerror_tls.c @@ -0,0 +1,43 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "spdk/string.h" + +static __thread char strerror_message[64]; + +const char * +spdk_strerror(int errnum) +{ + spdk_strerror_r(errnum, strerror_message, sizeof(strerror_message)); + return strerror_message; +} diff --git a/src/spdk/lib/util/string.c b/src/spdk/lib/util/string.c new file mode 100644 index 000000000..30ac1628a --- /dev/null +++ b/src/spdk/lib/util/string.c @@ -0,0 +1,476 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "spdk/stdinc.h" + +#include "spdk/string.h" + +char * +spdk_vsprintf_append_realloc(char *buffer, const char *format, va_list args) +{ + va_list args_copy; + char *new_buffer; + int orig_size = 0, new_size; + + /* Original buffer size */ + if (buffer) { + orig_size = strlen(buffer); + } + + /* Necessary buffer size */ + va_copy(args_copy, args); + new_size = vsnprintf(NULL, 0, format, args_copy); + va_end(args_copy); + + if (new_size < 0) { + return NULL; + } + new_size += orig_size + 1; + + new_buffer = realloc(buffer, new_size); + if (new_buffer == NULL) { + return NULL; + } + + vsnprintf(new_buffer + orig_size, new_size - orig_size, format, args); + + return new_buffer; +} + +char * +spdk_sprintf_append_realloc(char *buffer, const char *format, ...) +{ + va_list args; + char *ret; + + va_start(args, format); + ret = spdk_vsprintf_append_realloc(buffer, format, args); + va_end(args); + + return ret; +} + +char * +spdk_vsprintf_alloc(const char *format, va_list args) +{ + return spdk_vsprintf_append_realloc(NULL, format, args); +} + +char * +spdk_sprintf_alloc(const char *format, ...) +{ + va_list args; + char *ret; + + va_start(args, format); + ret = spdk_vsprintf_alloc(format, args); + va_end(args); + + return ret; +} + +char * +spdk_strlwr(char *s) +{ + char *p; + + if (s == NULL) { + return NULL; + } + + p = s; + while (*p != '\0') { + *p = tolower(*p); + p++; + } + + return s; +} + +char * +spdk_strsepq(char **stringp, const char *delim) +{ + char *p, *q, *r; + int quoted = 0, bslash = 0; + + p = *stringp; + if (p == NULL) { + return NULL; + } + + r = q = p; + while (*q != '\0' && *q != '\n') { + /* eat quoted characters */ + if (bslash) { + bslash = 0; + *r++ = *q++; + continue; + } else if (quoted) { + if (quoted == '"' && *q == '\\') { + bslash = 1; + q++; + continue; + } else if (*q == quoted) { + quoted = 0; + q++; + continue; + } + *r++ = *q++; + continue; + } else if (*q == '\\') { + bslash = 1; + q++; + continue; + } else if (*q == '"' || *q == '\'') { + quoted = *q; + q++; + continue; + } + + /* separator? */ + if (strchr(delim, *q) == NULL) { + *r++ = *q++; + continue; + } + + /* new string */ + q++; + break; + } + *r = '\0'; + + /* skip tailer */ + while (*q != '\0' && strchr(delim, *q) != NULL) { + q++; + } + if (*q != '\0') { + *stringp = q; + } else { + *stringp = NULL; + } + + return p; +} + +char * +spdk_str_trim(char *s) +{ + char *p, *q; + + if (s == NULL) { + return NULL; + } + + /* remove header */ + p = s; + while (*p != '\0' && isspace(*p)) { + p++; + } + + /* remove tailer */ + q = p + strlen(p); + while (q - 1 >= p && isspace(*(q - 1))) { + q--; + *q = '\0'; + } + + /* if remove header, move */ + if (p != s) { + q = s; + while (*p != '\0') { + *q++ = *p++; + } + *q = '\0'; + } + + return s; +} + +void +spdk_strcpy_pad(void *dst, const char *src, size_t size, int pad) +{ + size_t len; + + len = strlen(src); + if (len < size) { + memcpy(dst, src, len); + memset((char *)dst + len, pad, size - len); + } else { + memcpy(dst, src, size); + } +} + +size_t +spdk_strlen_pad(const void *str, size_t size, int pad) +{ + const uint8_t *start; + const uint8_t *iter; + uint8_t pad_byte; + + pad_byte = (uint8_t)pad; + start = (const uint8_t *)str; + + if (size == 0) { + return 0; + } + + iter = start + size - 1; + while (1) { + if (*iter != pad_byte) { + return iter - start + 1; + } + + if (iter == start) { + /* Hit the start of the string finding only pad_byte. */ + return 0; + } + iter--; + } +} + +int +spdk_parse_ip_addr(char *ip, char **host, char **port) +{ + char *p; + + if (ip == NULL) { + return -EINVAL; + } + + *host = NULL; + *port = NULL; + + if (ip[0] == '[') { + /* IPv6 */ + p = strchr(ip, ']'); + if (p == NULL) { + return -EINVAL; + } + *host = &ip[1]; + *p = '\0'; + + p++; + if (*p == '\0') { + return 0; + } else if (*p != ':') { + return -EINVAL; + } + + p++; + if (*p == '\0') { + return 0; + } + + *port = p; + } else { + /* IPv4 */ + p = strchr(ip, ':'); + if (p == NULL) { + *host = ip; + return 0; + } + + *host = ip; + *p = '\0'; + + p++; + if (*p == '\0') { + return 0; + } + + *port = p; + } + + return 0; +} + +size_t +spdk_str_chomp(char *s) +{ + size_t len = strlen(s); + size_t removed = 0; + + while (len > 0) { + if (s[len - 1] != '\r' && s[len - 1] != '\n') { + break; + } + + s[len - 1] = '\0'; + len--; + removed++; + } + + return removed; +} + +void +spdk_strerror_r(int errnum, char *buf, size_t buflen) +{ + int rc; + +#if defined(__USE_GNU) + char *new_buffer; + new_buffer = strerror_r(errnum, buf, buflen); + if (new_buffer == buf) { + rc = 0; + } else if (new_buffer != NULL) { + snprintf(buf, buflen, "%s", new_buffer); + rc = 0; + } else { + rc = 1; + } +#else + rc = strerror_r(errnum, buf, buflen); +#endif + + if (rc != 0) { + snprintf(buf, buflen, "Unknown error %d", errnum); + } +} + +int +spdk_parse_capacity(const char *cap_str, uint64_t *cap, bool *has_prefix) +{ + int rc; + char bin_prefix; + + rc = sscanf(cap_str, "%"SCNu64"%c", cap, &bin_prefix); + if (rc == 1) { + *has_prefix = false; + return 0; + } else if (rc == 0) { + if (errno == 0) { + /* No scanf matches - the string does not start with a digit */ + return -EINVAL; + } else { + /* Parsing error */ + return -errno; + } + } + + *has_prefix = true; + switch (bin_prefix) { + case 'k': + case 'K': + *cap *= 1024; + break; + case 'm': + case 'M': + *cap *= 1024 * 1024; + break; + case 'g': + case 'G': + *cap *= 1024 * 1024 * 1024; + break; + default: + return -EINVAL; + } + + return 0; +} + +bool +spdk_mem_all_zero(const void *data, size_t size) +{ + const uint8_t *buf = data; + + while (size--) { + if (*buf++ != 0) { + return false; + } + } + + return true; +} + +long int +spdk_strtol(const char *nptr, int base) +{ + long val; + char *endptr; + + /* Since strtoll() can legitimately return 0, LONG_MAX, or LONG_MIN + * on both success and failure, the calling program should set errno + * to 0 before the call. + */ + errno = 0; + + val = strtol(nptr, &endptr, base); + + if (!errno && *endptr != '\0') { + /* Non integer character was found. */ + return -EINVAL; + } else if (errno == ERANGE && (val == LONG_MAX || val == LONG_MIN)) { + /* Overflow occurred. */ + return -ERANGE; + } else if (errno != 0 && val == 0) { + /* Other error occurred. */ + return -errno; + } else if (val < 0) { + /* Input string was negative number. */ + return -ERANGE; + } + + return val; +} + +long long int +spdk_strtoll(const char *nptr, int base) +{ + long long val; + char *endptr; + + /* Since strtoll() can legitimately return 0, LLONG_MAX, or LLONG_MIN + * on both success and failure, the calling program should set errno + * to 0 before the call. + */ + errno = 0; + + val = strtoll(nptr, &endptr, base); + + if (!errno && *endptr != '\0') { + /* Non integer character was found. */ + return -EINVAL; + } else if (errno == ERANGE && (val == LLONG_MAX || val == LLONG_MIN)) { + /* Overflow occurred. */ + return -ERANGE; + } else if (errno != 0 && val == 0) { + /* Other error occurred. */ + return -errno; + } else if (val < 0) { + /* Input string was negative number. */ + return -ERANGE; + } + + return val; +} diff --git a/src/spdk/lib/util/util_internal.h b/src/spdk/lib/util/util_internal.h new file mode 100644 index 000000000..655ef513d --- /dev/null +++ b/src/spdk/lib/util/util_internal.h @@ -0,0 +1,77 @@ +/*- + * BSD LICENSE + * + * Copyright (C) 2008-2012 Daisuke Aoyama . + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef SPDK_UTIL_INTERNAL_H +#define SPDK_UTIL_INTERNAL_H + +#include "spdk/stdinc.h" + +/** + * IEEE CRC-32 polynomial (bit reflected) + */ +#define SPDK_CRC32_POLYNOMIAL_REFLECT 0xedb88320UL + +/** + * CRC-32C (Castagnoli) polynomial (bit reflected) + */ +#define SPDK_CRC32C_POLYNOMIAL_REFLECT 0x82f63b78UL + +struct spdk_crc32_table { + uint32_t table[256]; +}; + +/** + * Initialize a CRC32 lookup table for a given polynomial. + * + * \param table Table to fill with precalculated CRC-32 data. + * \param polynomial_reflect Bit-reflected CRC-32 polynomial. + */ +void crc32_table_init(struct spdk_crc32_table *table, + uint32_t polynomial_reflect); + + +/** + * Calculate a partial CRC-32 checksum. + * + * \param table CRC-32 table initialized with crc32_table_init(). + * \param buf Data buffer to checksum. + * \param len Length of buf in bytes. + * \param crc Previous CRC-32 value. + * \return Updated CRC-32 value. + */ +uint32_t crc32_update(const struct spdk_crc32_table *table, + const void *buf, size_t len, + uint32_t crc); + +#endif /* SPDK_UTIL_INTERNAL_H */ diff --git a/src/spdk/lib/util/uuid.c b/src/spdk/lib/util/uuid.c new file mode 100644 index 000000000..176f65880 --- /dev/null +++ b/src/spdk/lib/util/uuid.c @@ -0,0 +1,73 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "spdk/uuid.h" + +#include + +SPDK_STATIC_ASSERT(sizeof(struct spdk_uuid) == sizeof(uuid_t), "Size mismatch"); + +int +spdk_uuid_parse(struct spdk_uuid *uuid, const char *uuid_str) +{ + return uuid_parse(uuid_str, (void *)uuid) == 0 ? 0 : -EINVAL; +} + +int +spdk_uuid_fmt_lower(char *uuid_str, size_t uuid_str_size, const struct spdk_uuid *uuid) +{ + if (uuid_str_size < SPDK_UUID_STRING_LEN) { + return -EINVAL; + } + + uuid_unparse_lower((void *)uuid, uuid_str); + return 0; +} + +int +spdk_uuid_compare(const struct spdk_uuid *u1, const struct spdk_uuid *u2) +{ + return uuid_compare((void *)u1, (void *)u2); +} + +void +spdk_uuid_generate(struct spdk_uuid *uuid) +{ + uuid_generate((void *)uuid); +} + +void +spdk_uuid_copy(struct spdk_uuid *dst, const struct spdk_uuid *src) +{ + uuid_copy((void *)dst, (void *)src); +} -- cgit v1.2.3