summaryrefslogtreecommitdiffstats
path: root/src/lib-fts/fts-icu.c
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--src/lib-fts/fts-icu.c204
1 files changed, 204 insertions, 0 deletions
diff --git a/src/lib-fts/fts-icu.c b/src/lib-fts/fts-icu.c
new file mode 100644
index 0000000..ca652c3
--- /dev/null
+++ b/src/lib-fts/fts-icu.c
@@ -0,0 +1,204 @@
+/* Copyright (c) 2014-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "mempool.h"
+#include "array.h"
+#include "str.h"
+#include "unichar.h"
+#include "fts-icu.h"
+
+#include <unicode/uchar.h>
+#include <unicode/ucasemap.h>
+#include <unicode/uclean.h>
+
+static struct UCaseMap *icu_csm = NULL;
+
+static struct UCaseMap *fts_icu_csm(void)
+{
+ UErrorCode err = U_ZERO_ERROR;
+
+ if (icu_csm != NULL)
+ return icu_csm;
+ icu_csm = ucasemap_open(NULL, U_FOLD_CASE_DEFAULT, &err);
+ if (U_FAILURE(err)) {
+ i_fatal("LibICU ucasemap_open() failed: %s",
+ u_errorName(err));
+ }
+ return icu_csm;
+}
+
+void fts_icu_utf8_to_utf16(ARRAY_TYPE(icu_utf16) *dest_utf16,
+ const char *src_utf8)
+{
+ buffer_t *dest_buf = dest_utf16->arr.buffer;
+ UErrorCode err = U_ZERO_ERROR;
+ size_t src_bytes = strlen(src_utf8);
+ int32_t utf16_len;
+ UChar *dest_data, *retp = NULL;
+ int32_t avail_uchars = 0;
+
+ /* try to encode with the current buffer size */
+ avail_uchars = buffer_get_writable_size(dest_buf) / sizeof(UChar);
+ dest_data = buffer_get_space_unsafe(dest_buf, 0,
+ buffer_get_writable_size(dest_buf));
+ retp = u_strFromUTF8Lenient(dest_data, avail_uchars,
+ &utf16_len, src_utf8, src_bytes, &err);
+ if (err == U_BUFFER_OVERFLOW_ERROR) {
+ /* try again with a larger buffer */
+ dest_data = buffer_get_space_unsafe(dest_buf, 0,
+ utf16_len * sizeof(UChar));
+ err = U_ZERO_ERROR;
+ retp = u_strFromUTF8Lenient(dest_data, utf16_len,
+ &utf16_len, src_utf8,
+ src_bytes, &err);
+ }
+ if (U_FAILURE(err)) {
+ i_panic("LibICU u_strFromUTF8Lenient() failed: %s",
+ u_errorName(err));
+ }
+ buffer_set_used_size(dest_buf, utf16_len * sizeof(UChar));
+ i_assert(retp == dest_data);
+}
+
+void fts_icu_utf16_to_utf8(string_t *dest_utf8, const UChar *src_utf16,
+ unsigned int src_len)
+{
+ int32_t dest_len = 0;
+ int32_t sub_num = 0;
+ char *dest_data, *retp = NULL;
+ UErrorCode err = U_ZERO_ERROR;
+
+ /* try to encode with the current buffer size */
+ dest_data = buffer_get_space_unsafe(dest_utf8, 0,
+ buffer_get_writable_size(dest_utf8));
+ retp = u_strToUTF8WithSub(dest_data, buffer_get_writable_size(dest_utf8),
+ &dest_len, src_utf16, src_len,
+ UNICODE_REPLACEMENT_CHAR, &sub_num, &err);
+ if (err == U_BUFFER_OVERFLOW_ERROR) {
+ /* try again with a larger buffer */
+ dest_data = buffer_get_space_unsafe(dest_utf8, 0, dest_len);
+ err = U_ZERO_ERROR;
+ retp = u_strToUTF8WithSub(dest_data, buffer_get_writable_size(dest_utf8), &dest_len,
+ src_utf16, src_len,
+ UNICODE_REPLACEMENT_CHAR,
+ &sub_num, &err);
+ }
+ if (U_FAILURE(err)) {
+ i_panic("LibICU u_strToUTF8WithSub() failed: %s",
+ u_errorName(err));
+ }
+ buffer_set_used_size(dest_utf8, dest_len);
+ i_assert(retp == dest_data);
+}
+
+int fts_icu_translate(ARRAY_TYPE(icu_utf16) *dest_utf16, const UChar *src_utf16,
+ unsigned int src_len, UTransliterator *transliterator,
+ const char **error_r)
+{
+ buffer_t *dest_buf = dest_utf16->arr.buffer;
+ UErrorCode err = U_ZERO_ERROR;
+ int32_t utf16_len = src_len;
+ UChar *dest_data;
+ int32_t avail_uchars, limit = src_len;
+ size_t dest_pos = dest_buf->used;
+
+ /* translation is done in-place in the buffer. try first with the
+ current buffer size. */
+ array_append(dest_utf16, src_utf16, src_len);
+
+ avail_uchars = (buffer_get_writable_size(dest_buf)-dest_pos) / sizeof(UChar);
+ dest_data = buffer_get_space_unsafe(dest_buf, dest_pos,
+ buffer_get_writable_size(dest_buf) - dest_pos);
+ utrans_transUChars(transliterator, dest_data, &utf16_len,
+ avail_uchars, 0, &limit, &err);
+ if (err == U_BUFFER_OVERFLOW_ERROR) {
+ /* try again with a larger buffer */
+ err = U_ZERO_ERROR;
+ avail_uchars = utf16_len;
+ limit = utf16_len = src_len;
+ buffer_write(dest_buf, dest_pos,
+ src_utf16, src_len*sizeof(UChar));
+ dest_data = buffer_get_space_unsafe(dest_buf, dest_pos,
+ avail_uchars * sizeof(UChar));
+ utrans_transUChars(transliterator, dest_data, &utf16_len,
+ avail_uchars, 0, &limit, &err);
+ i_assert(err != U_BUFFER_OVERFLOW_ERROR);
+ }
+ if (U_FAILURE(err)) {
+ *error_r = t_strdup_printf("LibICU utrans_transUChars() failed: %s",
+ u_errorName(err));
+ buffer_set_used_size(dest_buf, dest_pos);
+ return -1;
+ }
+ buffer_set_used_size(dest_buf, utf16_len * sizeof(UChar));
+ return 0;
+}
+
+void fts_icu_lcase(string_t *dest_utf8, const char *src_utf8)
+{
+ struct UCaseMap *csm = fts_icu_csm();
+ size_t avail_bytes, dest_pos = dest_utf8->used;
+ char *dest_data;
+ int dest_full_len;
+ UErrorCode err = U_ZERO_ERROR;
+
+ avail_bytes = buffer_get_writable_size(dest_utf8) - dest_pos;
+ dest_data = buffer_get_space_unsafe(dest_utf8, dest_pos, avail_bytes);
+
+ /* ucasemap_utf8ToLower() may need to be called multiple times, because
+ the first return value may not be large enough. */
+ for (unsigned int i = 0;; i++) {
+ dest_full_len = ucasemap_utf8ToLower(csm, dest_data, avail_bytes,
+ src_utf8, -1, &err);
+ if (err != U_BUFFER_OVERFLOW_ERROR || i == 2)
+ break;
+
+ err = U_ZERO_ERROR;
+ dest_data = buffer_get_space_unsafe(dest_utf8, dest_pos, dest_full_len);
+ avail_bytes = dest_full_len;
+ }
+ if (U_FAILURE(err)) {
+ i_fatal("LibICU ucasemap_utf8ToLower() failed: %s",
+ u_errorName(err));
+ }
+ buffer_set_used_size(dest_utf8, dest_full_len);
+}
+
+void fts_icu_deinit(void)
+{
+ if (icu_csm != NULL) {
+ ucasemap_close(icu_csm);
+ icu_csm = NULL;
+ }
+ u_cleanup();
+}
+
+int fts_icu_transliterator_create(const char *id,
+ UTransliterator **transliterator_r,
+ const char **error_r)
+{
+ UErrorCode err = U_ZERO_ERROR;
+ UParseError perr;
+ ARRAY_TYPE(icu_utf16) id_utf16;
+ i_zero(&perr);
+
+ t_array_init(&id_utf16, strlen(id));
+ fts_icu_utf8_to_utf16(&id_utf16, id);
+ *transliterator_r = utrans_openU(array_front(&id_utf16),
+ array_count(&id_utf16),
+ UTRANS_FORWARD, NULL, 0, &perr, &err);
+ if (U_FAILURE(err)) {
+ string_t *str = t_str_new(128);
+
+ str_printfa(str, "Failed to open transliterator for id '%s': %s",
+ id, u_errorName(err));
+ if (perr.line >= 1) {
+ /* we have only one line in our ID */
+ str_printfa(str, " (parse error on offset %u)",
+ perr.offset);
+ }
+ *error_r = str_c(str);
+ return -1;
+ }
+ return 0;
+}