Adding upstream version 1:2.3.19.1+dfsg1.upstream/1%2.3.19.1+dfsg1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 204 insertions, 0 deletions

diff --git a/src/lib-fts/fts-icu.c b/src/lib-fts/fts-icu.c
new file mode 100644
index 0000000..ca652c3
--- /dev/null
+++ b/src/lib-fts/fts-icu.c
@@ -0,0 +1,204 @@
+/* Copyright (c) 2014-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "mempool.h"
+#include "array.h"
+#include "str.h"
+#include "unichar.h"
+#include "fts-icu.h"
+
+#include <unicode/uchar.h>
+#include <unicode/ucasemap.h>
+#include <unicode/uclean.h>
+
+static struct UCaseMap *icu_csm = NULL;
+
+static struct UCaseMap *fts_icu_csm(void)
+{
+	UErrorCode err = U_ZERO_ERROR;
+
+	if (icu_csm != NULL)
+		return icu_csm;
+	icu_csm = ucasemap_open(NULL, U_FOLD_CASE_DEFAULT, &err);
+	if (U_FAILURE(err)) {
+		i_fatal("LibICU ucasemap_open() failed: %s",
+			u_errorName(err));
+	}
+	return icu_csm;
+}
+
+void fts_icu_utf8_to_utf16(ARRAY_TYPE(icu_utf16) *dest_utf16,
+			   const char *src_utf8)
+{
+	buffer_t *dest_buf = dest_utf16->arr.buffer;
+	UErrorCode err = U_ZERO_ERROR;
+	size_t src_bytes = strlen(src_utf8);
+	int32_t utf16_len;
+	UChar *dest_data, *retp = NULL;
+	int32_t avail_uchars = 0;
+
+	/* try to encode with the current buffer size */
+	avail_uchars = buffer_get_writable_size(dest_buf) / sizeof(UChar);
+	dest_data = buffer_get_space_unsafe(dest_buf, 0,
+				buffer_get_writable_size(dest_buf));
+	retp = u_strFromUTF8Lenient(dest_data, avail_uchars,
+				    &utf16_len, src_utf8, src_bytes, &err);
+	if (err == U_BUFFER_OVERFLOW_ERROR) {
+		/* try again with a larger buffer */
+		dest_data = buffer_get_space_unsafe(dest_buf, 0,
+						    utf16_len * sizeof(UChar));
+		err = U_ZERO_ERROR;
+		retp = u_strFromUTF8Lenient(dest_data, utf16_len,
+					    &utf16_len, src_utf8,
+					    src_bytes, &err);
+	}
+	if (U_FAILURE(err)) {
+		i_panic("LibICU u_strFromUTF8Lenient() failed: %s",
+			u_errorName(err));
+	}
+	buffer_set_used_size(dest_buf, utf16_len * sizeof(UChar));
+	i_assert(retp == dest_data);
+}
+
+void fts_icu_utf16_to_utf8(string_t *dest_utf8, const UChar *src_utf16,
+			   unsigned int src_len)
+{
+	int32_t dest_len = 0;
+	int32_t sub_num = 0;
+	char *dest_data, *retp = NULL;
+	UErrorCode err = U_ZERO_ERROR;
+
+	/* try to encode with the current buffer size */
+	dest_data = buffer_get_space_unsafe(dest_utf8, 0,
+					    buffer_get_writable_size(dest_utf8));
+	retp = u_strToUTF8WithSub(dest_data, buffer_get_writable_size(dest_utf8),
+				  &dest_len, src_utf16, src_len,
+				  UNICODE_REPLACEMENT_CHAR, &sub_num, &err);
+	if (err == U_BUFFER_OVERFLOW_ERROR) {
+		/* try again with a larger buffer */
+		dest_data = buffer_get_space_unsafe(dest_utf8, 0, dest_len);
+		err = U_ZERO_ERROR;
+		retp = u_strToUTF8WithSub(dest_data, buffer_get_writable_size(dest_utf8), &dest_len,
+					  src_utf16, src_len,
+					  UNICODE_REPLACEMENT_CHAR,
+					  &sub_num, &err);
+	}
+	if (U_FAILURE(err)) {
+		i_panic("LibICU u_strToUTF8WithSub() failed: %s",
+			u_errorName(err));
+	}
+	buffer_set_used_size(dest_utf8, dest_len);
+	i_assert(retp == dest_data);
+}
+
+int fts_icu_translate(ARRAY_TYPE(icu_utf16) *dest_utf16, const UChar *src_utf16,
+		      unsigned int src_len, UTransliterator *transliterator,
+		      const char **error_r)
+{
+	buffer_t *dest_buf = dest_utf16->arr.buffer;
+	UErrorCode err = U_ZERO_ERROR;
+	int32_t utf16_len = src_len;
+	UChar *dest_data;
+	int32_t avail_uchars, limit = src_len;
+	size_t dest_pos = dest_buf->used;
+
+	/* translation is done in-place in the buffer. try first with the
+	   current buffer size. */
+	array_append(dest_utf16, src_utf16, src_len);
+
+	avail_uchars = (buffer_get_writable_size(dest_buf)-dest_pos) / sizeof(UChar);
+	dest_data = buffer_get_space_unsafe(dest_buf, dest_pos,
+			buffer_get_writable_size(dest_buf) - dest_pos);
+	utrans_transUChars(transliterator, dest_data, &utf16_len,
+			   avail_uchars, 0, &limit, &err);
+	if (err == U_BUFFER_OVERFLOW_ERROR) {
+		/* try again with a larger buffer */
+		err = U_ZERO_ERROR;
+		avail_uchars = utf16_len;
+		limit = utf16_len = src_len;
+		buffer_write(dest_buf, dest_pos,
+			     src_utf16, src_len*sizeof(UChar));
+		dest_data = buffer_get_space_unsafe(dest_buf, dest_pos,
+						    avail_uchars * sizeof(UChar));
+		utrans_transUChars(transliterator, dest_data, &utf16_len,
+				   avail_uchars, 0, &limit, &err);
+		i_assert(err != U_BUFFER_OVERFLOW_ERROR);
+	}
+	if (U_FAILURE(err)) {
+		*error_r = t_strdup_printf("LibICU utrans_transUChars() failed: %s",
+					   u_errorName(err));
+		buffer_set_used_size(dest_buf, dest_pos);
+		return -1;
+	}
+	buffer_set_used_size(dest_buf, utf16_len * sizeof(UChar));
+	return 0;
+}
+
+void fts_icu_lcase(string_t *dest_utf8, const char *src_utf8)
+{
+	struct UCaseMap *csm = fts_icu_csm();
+	size_t avail_bytes, dest_pos = dest_utf8->used;
+	char *dest_data;
+	int dest_full_len;
+	UErrorCode err = U_ZERO_ERROR;
+
+	avail_bytes = buffer_get_writable_size(dest_utf8) - dest_pos;
+	dest_data = buffer_get_space_unsafe(dest_utf8, dest_pos, avail_bytes);
+
+	/* ucasemap_utf8ToLower() may need to be called multiple times, because
+	   the first return value may not be large enough. */
+	for (unsigned int i = 0;; i++) {
+		dest_full_len = ucasemap_utf8ToLower(csm, dest_data, avail_bytes,
+						     src_utf8, -1, &err);
+		if (err != U_BUFFER_OVERFLOW_ERROR || i == 2)
+			break;
+
+		err = U_ZERO_ERROR;
+		dest_data = buffer_get_space_unsafe(dest_utf8, dest_pos, dest_full_len);
+		avail_bytes = dest_full_len;
+	}
+	if (U_FAILURE(err)) {
+		i_fatal("LibICU ucasemap_utf8ToLower() failed: %s",
+			u_errorName(err));
+	}
+	buffer_set_used_size(dest_utf8, dest_full_len);
+}
+
+void fts_icu_deinit(void)
+{
+	if (icu_csm != NULL) {
+		ucasemap_close(icu_csm);
+		icu_csm = NULL;
+	}
+	u_cleanup();
+}
+
+int fts_icu_transliterator_create(const char *id,
+                                  UTransliterator **transliterator_r,
+                                  const char **error_r)
+{
+	UErrorCode err = U_ZERO_ERROR;
+	UParseError perr;
+	ARRAY_TYPE(icu_utf16) id_utf16;
+	i_zero(&perr);
+
+	t_array_init(&id_utf16, strlen(id));
+	fts_icu_utf8_to_utf16(&id_utf16, id);
+	*transliterator_r = utrans_openU(array_front(&id_utf16),
+					 array_count(&id_utf16),
+					 UTRANS_FORWARD, NULL, 0, &perr, &err);
+	if (U_FAILURE(err)) {
+		string_t *str = t_str_new(128);
+
+		str_printfa(str, "Failed to open transliterator for id '%s': %s",
+			    id, u_errorName(err));
+		if (perr.line >= 1) {
+			/* we have only one line in our ID */
+			str_printfa(str, " (parse error on offset %u)",
+				    perr.offset);
+		}
+		*error_r = str_c(str);
+		return -1;
+	}
+	return 0;
+}