summaryrefslogtreecommitdiffstats
path: root/wsutil/unicode-utils.c
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--wsutil/unicode-utils.c368
1 files changed, 368 insertions, 0 deletions
diff --git a/wsutil/unicode-utils.c b/wsutil/unicode-utils.c
new file mode 100644
index 0000000..4ed4b32
--- /dev/null
+++ b/wsutil/unicode-utils.c
@@ -0,0 +1,368 @@
+/* unicode-utils.c
+ * Unicode utility routines
+ *
+ * Wireshark - Network traffic analyzer
+ * By Gerald Combs <gerald@wireshark.org>
+ * Copyright 2006 Gerald Combs
+ *
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ */
+
+#include "config.h"
+
+#include "unicode-utils.h"
+
+int ws_utf8_seqlen[256] = {
+ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* 0x00...0x0f */
+ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* 0x10...0x1f */
+ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* 0x20...0x2f */
+ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* 0x30...0x3f */
+ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* 0x40...0x4f */
+ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* 0x50...0x5f */
+ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* 0x60...0x6f */
+ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* 0x70...0x7f */
+ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, /* 0x80...0x8f */
+ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, /* 0x90...0x9f */
+ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, /* 0xa0...0xaf */
+ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, /* 0xb0...0xbf */
+ 0,0,2,2,2,2,2,2,2,2,2,2,2,2,2,2, /* 0xc0...0xcf */
+ 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, /* 0xd0...0xdf */
+ 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3, /* 0xe0...0xef */
+ 4,4,4,4,4,0,0,0,0,0,0,0,0,0,0,0, /* 0xf0...0xff */
+};
+
+/* Given a pointer and a length, validates a string of bytes as UTF-8.
+ * Returns the number of valid bytes, and a pointer immediately past
+ * the checked region.
+ *
+ * Differs from Glib's g_utf8_validate_len in that null bytes are
+ * considered valid UTF-8, and that maximal subparts are replaced as
+ * a unit. (I.e., given a sequence of 2 or 3 bytes which are a
+ * truncated version of a 3 or 4 byte UTF-8 character, but the next
+ * byte does not continue the character, the set of 2 or 3 bytes
+ * are replaced with one REPLACMENT CHARACTER.)
+ */
+static inline size_t
+utf_8_validate(const uint8_t *start, ssize_t length, const uint8_t **end)
+{
+ const uint8_t *ptr = start;
+ uint8_t ch;
+ size_t unichar_len, valid_bytes = 0;
+
+ while (length > 0) {
+
+ ch = *ptr;
+
+ if (ch < 0x80) {
+ valid_bytes++;
+ ptr++;
+ length--;
+ continue;
+ }
+
+ ch = *ptr;
+
+ if (ch < 0xc2 || ch > 0xf4) {
+ ptr++;
+ length--;
+ *end = ptr;
+ return valid_bytes;
+ }
+
+ if (ch < 0xe0) { /* 110xxxxx, 2 byte char */
+ unichar_len = 2;
+ } else if (ch < 0xf0) { /* 1110xxxx, 3 byte char */
+ unichar_len = 3;
+ ptr++;
+ length--;
+ if (length < 1) {
+ *end = ptr;
+ return valid_bytes;
+ }
+ switch (ch) {
+ case 0xe0:
+ if (*ptr < 0xa0 || *ptr > 0xbf) {
+ *end = ptr;
+ return valid_bytes;
+ }
+ break;
+ case 0xed:
+ if (*ptr < 0x80 || *ptr > 0x9f) {
+ *end = ptr;
+ return valid_bytes;
+ }
+ break;
+ default:
+ if (*ptr < 0x80 || *ptr > 0xbf) {
+ *end = ptr;
+ return valid_bytes;
+ }
+ }
+ } else { /* 11110xxx, 4 byte char - > 0xf4 excluded above */
+ unichar_len = 4;
+ ptr++;
+ length--;
+ if (length < 1) {
+ *end = ptr;
+ return valid_bytes;
+ }
+ switch (ch) {
+ case 0xf0:
+ if (*ptr < 0x90 || *ptr > 0xbf) {
+ *end = ptr;
+ return valid_bytes;
+ }
+ break;
+ case 0xf4:
+ if (*ptr < 0x80 || *ptr > 0x8f) {
+ *end = ptr;
+ return valid_bytes;
+ }
+ break;
+ default:
+ if (*ptr < 0x80 || *ptr > 0xbf) {
+ *end = ptr;
+ return valid_bytes;
+ }
+ }
+ ptr++;
+ length--;
+ if (length < 1) {
+ *end = ptr;
+ return valid_bytes;
+ }
+ if (*ptr < 0x80 || *ptr > 0xbf) {
+ *end = ptr;
+ return valid_bytes;
+ }
+ }
+
+ ptr++;
+ length--;
+ if (length < 1) {
+ *end = ptr;
+ return valid_bytes;
+ }
+ if (*ptr < 0x80 || *ptr > 0xbf) {
+ *end = ptr;
+ return valid_bytes;
+ } else {
+ ptr++;
+ length--;
+ valid_bytes += unichar_len;
+ }
+
+ }
+ *end = ptr;
+ return valid_bytes;
+}
+
+/*
+ * Given a wmem scope, a pointer, and a length, treat the string of bytes
+ * referred to by the pointer and length as a UTF-8 string, and return a
+ * pointer to a UTF-8 string, allocated using the wmem scope, with all
+ * ill-formed sequences replaced with the Unicode REPLACEMENT CHARACTER
+ * according to the recommended "best practices" given in the Unicode
+ * Standard and specified by W3C/WHATWG.
+ *
+ * Note that in conformance with the Unicode Standard, this treats three
+ * byte sequences corresponding to UTF-16 surrogate halves (paired or unpaired)
+ * and two byte overlong encodings of 7-bit ASCII characters as invalid and
+ * substitutes REPLACEMENT CHARACTER for them. Explicit support for nonstandard
+ * derivative encoding formats (e.g. CESU-8, Java Modified UTF-8, WTF-8) could
+ * be added later.
+ *
+ * Compared with g_utf8_make_valid(), this function does not consider
+ * internal NUL bytes as invalid and replace them with replacment characters.
+ * It also replaces maximal subparts as a unit; i.e., a sequence of 2 or 3
+ * bytes which are a truncated version of a valid 3 or 4 byte character (but
+ * the next byte does not continue the character) are replaced with a single
+ * REPLACEMENT CHARACTER, whereas the Glib function replaces each byte of the
+ * sequence with its own (3 octet) REPLACEMENT CHARACTER.
+ *
+ * XXX: length should probably be a size_t instead of a int in all
+ * these encoding functions
+ * XXX: the buffer returned can be of different length than the input,
+ * and can have internal NULs as well (so that strlen doesn't give its
+ * length). As with the other encoding functions, we should return the
+ * length of the output buffer (or a wmem_strbuf_t directly) and an
+ * indication of whether there was an invalid character (i.e.
+ * REPLACEMENT CHARACTER was used.)
+ */
+wmem_strbuf_t *
+ws_utf8_make_valid_strbuf(wmem_allocator_t *scope, const uint8_t *ptr, ssize_t length)
+{
+ wmem_strbuf_t *str;
+
+ str = wmem_strbuf_new_sized(scope, length+1);
+
+ /* See the Unicode Standard conformance chapter at
+ * https://www.unicode.org/versions/Unicode15.0.0/ch03.pdf especially
+ * Table 3-7 "Well-Formed UTF-8 Byte Sequences" and
+ * U+FFFD Substitution of Maximal Subparts. */
+
+ while (length > 0) {
+ const uint8_t *prev = ptr;
+ size_t valid_bytes = utf_8_validate(prev, length, &ptr);
+
+ if (valid_bytes) {
+ wmem_strbuf_append_len(str, prev, valid_bytes);
+ }
+ length -= ptr - prev;
+ prev += valid_bytes;
+ if (ptr - prev) {
+ wmem_strbuf_append_unichar_repl(str);
+ }
+ }
+
+ return str;
+}
+
+uint8_t *
+ws_utf8_make_valid(wmem_allocator_t *scope, const uint8_t *ptr, ssize_t length)
+{
+ wmem_strbuf_t *str = ws_utf8_make_valid_strbuf(scope, ptr, length);
+ return wmem_strbuf_finalize(str);
+}
+
+#ifdef _WIN32
+
+#include <strsafe.h>
+
+/** @file
+ * Unicode utilities (internal interface)
+ *
+ * We define UNICODE and _UNICODE under Windows. This means that
+ * Windows SDK routines expect UTF-16 strings, in contrast to newer
+ * versions of Glib and GTK+ which expect UTF-8. This module provides
+ * convenience routines for converting between UTF-8 and UTF-16.
+ */
+
+#define INITIAL_UTFBUF_SIZE 128
+
+/*
+ * XXX - Should we use g_utf8_to_utf16() and g_utf16_to_utf8()
+ * instead? The goal of the functions below was to provide simple
+ * wrappers for UTF-8 <-> UTF-16 conversion without making the
+ * caller worry about freeing up memory afterward.
+ */
+
+/* Convert from UTF-8 to UTF-16. */
+const wchar_t *
+utf_8to16(const char *utf8str)
+{
+ static wchar_t *utf16buf[3];
+ static int utf16buf_len[3];
+ static int idx;
+
+ if (utf8str == NULL)
+ return NULL;
+
+ idx = (idx + 1) % 3;
+
+ /*
+ * Allocate the buffer if it's not already allocated.
+ */
+ if (utf16buf[idx] == NULL) {
+ utf16buf_len[idx] = INITIAL_UTFBUF_SIZE;
+ utf16buf[idx] = g_malloc(utf16buf_len[idx] * sizeof(wchar_t));
+ }
+
+ while (MultiByteToWideChar(CP_UTF8, 0, utf8str, -1, NULL, 0) >= utf16buf_len[idx]) {
+ /*
+ * Double the buffer's size if it's not big enough.
+ * The size of the buffer starts at 128, so doubling its size
+ * adds at least another 128 bytes, which is more than enough
+ * for one more character plus a terminating '\0'.
+ */
+ utf16buf_len[idx] *= 2;
+ utf16buf[idx] = g_realloc(utf16buf[idx], utf16buf_len[idx] * sizeof(wchar_t));
+ }
+
+ if (MultiByteToWideChar(CP_UTF8, 0, utf8str, -1, utf16buf[idx], utf16buf_len[idx]) == 0)
+ return NULL;
+
+ return utf16buf[idx];
+}
+
+void
+utf_8to16_snprintf(TCHAR *utf16buf, int utf16buf_len, const char* fmt, ...)
+{
+ va_list ap;
+ char* dst;
+
+ va_start(ap,fmt);
+ dst = ws_strdup_vprintf(fmt, ap);
+ va_end(ap);
+
+ StringCchPrintf(utf16buf, utf16buf_len, _T("%s"), utf_8to16(dst));
+
+ g_free(dst);
+}
+
+/* Convert from UTF-16 to UTF-8. */
+char *
+utf_16to8(const wchar_t *utf16str)
+{
+ static char *utf8buf[3];
+ static int utf8buf_len[3];
+ static int idx;
+
+ if (utf16str == NULL)
+ return NULL;
+
+ idx = (idx + 1) % 3;
+
+ /*
+ * Allocate the buffer if it's not already allocated.
+ */
+ if (utf8buf[idx] == NULL) {
+ utf8buf_len[idx] = INITIAL_UTFBUF_SIZE;
+ utf8buf[idx] = g_malloc(utf8buf_len[idx]);
+ }
+
+ while (WideCharToMultiByte(CP_UTF8, 0, utf16str, -1, NULL, 0, NULL, NULL) >= utf8buf_len[idx]) {
+ /*
+ * Double the buffer's size if it's not big enough.
+ * The size of the buffer starts at 128, so doubling its size
+ * adds at least another 128 bytes, which is more than enough
+ * for one more character plus a terminating '\0'.
+ */
+ utf8buf_len[idx] *= 2;
+ utf8buf[idx] = g_realloc(utf8buf[idx], utf8buf_len[idx]);
+ }
+
+ if (WideCharToMultiByte(CP_UTF8, 0, utf16str, -1, utf8buf[idx], utf8buf_len[idx], NULL, NULL) == 0)
+ return NULL;
+
+ return utf8buf[idx];
+}
+
+/* Convert our argument list from UTF-16 to UTF-8. */
+char **
+arg_list_utf_16to8(int argc, wchar_t *wc_argv[]) {
+ char **argv;
+ int i;
+
+ argv = (char **)g_malloc((argc + 1) * sizeof(char *));
+ for (i = 0; i < argc; i++) {
+ argv[i] = g_utf16_to_utf8(wc_argv[i], -1, NULL, NULL, NULL);
+ }
+ argv[argc] = NULL;
+ return argv;
+}
+
+#endif
+
+/*
+ * Editor modelines - https://www.wireshark.org/tools/modelines.html
+ *
+ * Local variables:
+ * c-basic-offset: 4
+ * tab-width: 8
+ * indent-tabs-mode: nil
+ * End:
+ *
+ * vi: set shiftwidth=4 tabstop=8 expandtab:
+ * :indentSize=4:tabSize=8:noTabs=true:
+ */
generated by cgit v1.2.3 (git 2.39.1) at 2024-07-25 00:50:05 +0000