summaryrefslogtreecommitdiffstats
path: root/sal/textenc/tcvtutf8.cxx
diff options
context:
space:
mode:
Diffstat (limited to 'sal/textenc/tcvtutf8.cxx')
-rw-r--r--sal/textenc/tcvtutf8.cxx458
1 files changed, 458 insertions, 0 deletions
diff --git a/sal/textenc/tcvtutf8.cxx b/sal/textenc/tcvtutf8.cxx
new file mode 100644
index 000000000..f210b654d
--- /dev/null
+++ b/sal/textenc/tcvtutf8.cxx
@@ -0,0 +1,458 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/*
+ * This file is part of the LibreOffice project.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * This file incorporates work covered by the following license notice:
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed
+ * with this work for additional information regarding copyright
+ * ownership. The ASF licenses this file to you under the Apache
+ * License, Version 2.0 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of
+ * the License at http://www.apache.org/licenses/LICENSE-2.0 .
+ */
+
+#include <sal/config.h>
+
+#include <cassert>
+
+#include <sal/types.h>
+#include <rtl/character.hxx>
+#include <rtl/textcvt.h>
+
+#include "converter.hxx"
+#include "tcvtutf8.hxx"
+#include "tenchelp.hxx"
+#include "unichars.hxx"
+
+namespace {
+
+struct ImplUtf8ToUnicodeContext
+{
+ sal_uInt32 nUtf32;
+ int nBytes;
+ int nShift;
+ bool bCheckBom;
+};
+
+struct ImplUnicodeToUtf8Context
+{
+ sal_Unicode nHighSurrogate; /* 0xFFFF: write BOM */
+};
+
+}
+
+void * ImplCreateUtf8ToUnicodeContext()
+{
+ ImplUtf8ToUnicodeContext * p = new ImplUtf8ToUnicodeContext;
+ ImplResetUtf8ToUnicodeContext(p);
+ return p;
+}
+
+void ImplResetUtf8ToUnicodeContext(void * pContext)
+{
+ if (pContext != nullptr)
+ {
+ static_cast< ImplUtf8ToUnicodeContext * >(pContext)->nShift = -1;
+ static_cast< ImplUtf8ToUnicodeContext * >(pContext)->bCheckBom = true;
+ }
+}
+
+void ImplDestroyUtf8ToUnicodeContext(void * pContext)
+{
+ delete static_cast< ImplUtf8ToUnicodeContext * >(pContext);
+}
+
+sal_Size ImplConvertUtf8ToUnicode(
+ void const * pData, void * pContext, char const * pSrcBuf,
+ sal_Size nSrcBytes, sal_Unicode * pDestBuf, sal_Size nDestChars,
+ sal_uInt32 nFlags, sal_uInt32 * pInfo, sal_Size * pSrcCvtBytes)
+{
+ bool bJavaUtf8 = pData != nullptr;
+ sal_uInt32 nUtf32 = 0;
+ int nBytes = int();
+ int nShift = -1;
+ bool bCheckBom = true;
+ sal_uInt32 nInfo = 0;
+ unsigned char const * pSrcBufPtr = reinterpret_cast<unsigned char const *>(pSrcBuf);
+ unsigned char const * pSrcBufEnd = pSrcBufPtr + nSrcBytes;
+ sal_Unicode * pDestBufPtr = pDestBuf;
+ sal_Unicode * pDestBufEnd = pDestBufPtr + nDestChars;
+ unsigned char const * startOfCurrentChar = pSrcBufPtr;
+
+ if (pContext != nullptr)
+ {
+ nUtf32 = static_cast< ImplUtf8ToUnicodeContext * >(pContext)->nUtf32;
+ nBytes = static_cast< ImplUtf8ToUnicodeContext * >(pContext)->nBytes;
+ nShift = static_cast< ImplUtf8ToUnicodeContext * >(pContext)->nShift;
+ bCheckBom = static_cast< ImplUtf8ToUnicodeContext * >(pContext)->bCheckBom;
+ }
+
+ while (pSrcBufPtr < pSrcBufEnd)
+ {
+ bool bConsume = true;
+ sal_uInt32 nChar = *pSrcBufPtr++;
+ if (nShift < 0)
+ // Allow (illegal) 5 and 6 byte sequences, so they are read as a
+ // single individual bad character:
+ if (nChar <= 0x7F)
+ {
+ nUtf32 = nChar;
+ nBytes = 1;
+ goto transform;
+ }
+ else if (nChar <= 0xBF)
+ goto bad_input;
+ else if (nChar <= 0xDF)
+ {
+ nUtf32 = (nChar & 0x1F) << 6;
+ nBytes = 2;
+ nShift = 0;
+ }
+ else if (nChar <= 0xEF)
+ {
+ nUtf32 = (nChar & 0x0F) << 12;
+ nBytes = 3;
+ nShift = 6;
+ }
+ else if (nChar <= 0xF7)
+ {
+ nUtf32 = (nChar & 0x07) << 18;
+ nBytes = 4;
+ nShift = 12;
+ }
+ else if (nChar <= 0xFB)
+ {
+ nUtf32 = (nChar & 0x03) << 24;
+ nBytes = 5;
+ nShift = 18;
+ }
+ else if (nChar <= 0xFD)
+ {
+ nUtf32 = (nChar & 0x01) << 30;
+ nBytes = 6;
+ nShift = 24;
+ }
+ else
+ goto bad_input;
+ else if ((nChar & 0xC0) == 0x80)
+ {
+ nUtf32 |= (nChar & 0x3F) << nShift;
+ if (nShift == 0)
+ goto transform;
+ else
+ nShift -= 6;
+ }
+ else
+ {
+ /*
+ This byte is preceded by a broken UTF-8 sequence; if this byte
+ is neither in the range [0x80..0xBF] nor in the range
+ [0xFE..0xFF], assume that this byte does not belong to that
+ broken sequence, but instead starts a new, legal UTF-8 sequence:
+ */
+ bConsume = nChar >= 0xFE;
+ goto bad_input;
+ }
+ continue;
+
+ transform:
+ if (!bCheckBom || nUtf32 != 0xFEFF || nBytes != 3
+ || (nFlags & RTL_TEXTTOUNICODE_FLAGS_GLOBAL_SIGNATURE) == 0
+ || bJavaUtf8)
+ {
+ switch (nBytes) {
+ case 1:
+ if (bJavaUtf8 && nUtf32 == 0) {
+ goto bad_input;
+ }
+ break;
+ case 2:
+ if (nUtf32 < 0x80 && !(bJavaUtf8 && nUtf32 == 0)) {
+ goto bad_input;
+ }
+ break;
+ case 3:
+ if (nUtf32 < 0x800 || (!bJavaUtf8 && rtl::isSurrogate(nUtf32)))
+ {
+ goto bad_input;
+ }
+ break;
+ case 4:
+ if (nUtf32 < 0x10000 || !rtl::isUnicodeCodePoint(nUtf32)
+ || bJavaUtf8)
+ {
+ goto bad_input;
+ }
+ break;
+ default:
+ goto bad_input;
+ }
+ if (nUtf32 <= 0xFFFF)
+ if (pDestBufPtr != pDestBufEnd)
+ *pDestBufPtr++ = static_cast<sal_Unicode>(nUtf32);
+ else
+ goto no_output;
+ else if (pDestBufEnd - pDestBufPtr >= 2)
+ pDestBufPtr += rtl::splitSurrogates(nUtf32, pDestBufPtr);
+ else
+ goto no_output;
+ }
+ nShift = -1;
+ bCheckBom = false;
+ startOfCurrentChar = pSrcBufPtr;
+ continue;
+
+ bad_input:
+ switch (sal::detail::textenc::handleBadInputTextToUnicodeConversion(
+ false, nBytes != 1, 0, nFlags, &pDestBufPtr, pDestBufEnd,
+ &nInfo))
+ {
+ case sal::detail::textenc::BAD_INPUT_STOP:
+ nShift = -1;
+ bCheckBom = false;
+ if ((nFlags & RTL_TEXTTOUNICODE_FLAGS_FLUSH) == 0) {
+ if (!bConsume)
+ --pSrcBufPtr;
+ } else {
+ pSrcBufPtr = startOfCurrentChar;
+ }
+ break;
+
+ case sal::detail::textenc::BAD_INPUT_CONTINUE:
+ nShift = -1;
+ bCheckBom = false;
+ if (!bConsume)
+ --pSrcBufPtr;
+ startOfCurrentChar = pSrcBufPtr;
+ continue;
+
+ case sal::detail::textenc::BAD_INPUT_NO_OUTPUT:
+ goto no_output;
+ }
+ break;
+
+ no_output:
+ --pSrcBufPtr;
+ nInfo |= RTL_TEXTTOUNICODE_INFO_DESTBUFFERTOOSMALL;
+ break;
+ }
+
+ if (nShift >= 0
+ && (nInfo & (RTL_TEXTTOUNICODE_INFO_ERROR
+ | RTL_TEXTTOUNICODE_INFO_DESTBUFFERTOOSMALL))
+ == 0)
+ {
+ if ((nFlags & RTL_TEXTTOUNICODE_FLAGS_FLUSH) == 0)
+ nInfo |= RTL_TEXTTOUNICODE_INFO_SRCBUFFERTOOSMALL;
+ else
+ switch (sal::detail::textenc::handleBadInputTextToUnicodeConversion(
+ false, true, 0, nFlags, &pDestBufPtr, pDestBufEnd,
+ &nInfo))
+ {
+ case sal::detail::textenc::BAD_INPUT_STOP:
+ if ((nFlags & RTL_TEXTTOUNICODE_FLAGS_FLUSH) != 0) {
+ pSrcBufPtr = startOfCurrentChar;
+ }
+ [[fallthrough]];
+ case sal::detail::textenc::BAD_INPUT_CONTINUE:
+ nShift = -1;
+ bCheckBom = false;
+ break;
+
+ case sal::detail::textenc::BAD_INPUT_NO_OUTPUT:
+ nInfo |= RTL_TEXTTOUNICODE_INFO_DESTBUFFERTOOSMALL;
+ break;
+ }
+ }
+
+ if (pContext != nullptr)
+ {
+ static_cast< ImplUtf8ToUnicodeContext * >(pContext)->nUtf32 = nUtf32;
+ static_cast< ImplUtf8ToUnicodeContext * >(pContext)->nBytes = nBytes;
+ static_cast< ImplUtf8ToUnicodeContext * >(pContext)->nShift = nShift;
+ static_cast< ImplUtf8ToUnicodeContext * >(pContext)->bCheckBom = bCheckBom;
+ }
+ if (pInfo != nullptr)
+ *pInfo = nInfo;
+ if (pSrcCvtBytes != nullptr)
+ *pSrcCvtBytes = reinterpret_cast< char const * >(pSrcBufPtr) - pSrcBuf;
+ return pDestBufPtr - pDestBuf;
+}
+
+void * ImplCreateUnicodeToUtf8Context()
+{
+ ImplUnicodeToUtf8Context * p = new ImplUnicodeToUtf8Context;
+ ImplResetUnicodeToUtf8Context(p);
+ return p;
+}
+
+void ImplResetUnicodeToUtf8Context(void * pContext)
+{
+ if (pContext != nullptr)
+ static_cast< ImplUnicodeToUtf8Context * >(pContext)->nHighSurrogate = 0xFFFF;
+}
+
+void ImplDestroyUnicodeToUtf8Context(void * pContext)
+{
+ delete static_cast< ImplUnicodeToUtf8Context * >(pContext);
+}
+
+sal_Size ImplConvertUnicodeToUtf8(
+ void const * pData, void * pContext, sal_Unicode const * pSrcBuf,
+ sal_Size nSrcChars, char * pDestBuf, sal_Size nDestBytes, sal_uInt32 nFlags,
+ sal_uInt32 * pInfo, sal_Size * pSrcCvtChars)
+{
+ bool bJavaUtf8 = pData != nullptr;
+ sal_Unicode nHighSurrogate = 0xFFFF;
+ sal_uInt32 nInfo = 0;
+ sal_Unicode const * pSrcBufPtr = pSrcBuf;
+ sal_Unicode const * pSrcBufEnd = pSrcBufPtr + nSrcChars;
+ char * pDestBufPtr = pDestBuf;
+ char * pDestBufEnd = pDestBufPtr + nDestBytes;
+
+ if (pContext != nullptr)
+ nHighSurrogate
+ = static_cast< ImplUnicodeToUtf8Context * >(pContext)->nHighSurrogate;
+
+ if (nHighSurrogate == 0xFFFF)
+ {
+ if ((nFlags & RTL_UNICODETOTEXT_FLAGS_GLOBAL_SIGNATURE) != 0
+ && !bJavaUtf8)
+ {
+ if (pDestBufEnd - pDestBufPtr >= 3)
+ {
+ /* Write BOM (U+FEFF) as UTF-8: */
+ *pDestBufPtr++ = static_cast< char >(static_cast< unsigned char >(0xEF));
+ *pDestBufPtr++ = static_cast< char >(static_cast< unsigned char >(0xBB));
+ *pDestBufPtr++ = static_cast< char >(static_cast< unsigned char >(0xBF));
+ }
+ else
+ {
+ nInfo |= RTL_UNICODETOTEXT_INFO_DESTBUFFERTOSMALL;
+ goto done;
+ }
+ }
+ nHighSurrogate = 0;
+ }
+
+ while (pSrcBufPtr < pSrcBufEnd)
+ {
+ sal_uInt32 nChar = *pSrcBufPtr++;
+ if (nHighSurrogate == 0)
+ {
+ if (rtl::isHighSurrogate(nChar) && !bJavaUtf8)
+ {
+ nHighSurrogate = static_cast<sal_Unicode>(nChar);
+ continue;
+ }
+ else if (rtl::isLowSurrogate(nChar) && !bJavaUtf8)
+ {
+ goto bad_input;
+ }
+ }
+ else if (rtl::isLowSurrogate(nChar) && !bJavaUtf8)
+ nChar = rtl::combineSurrogates(nHighSurrogate, nChar);
+ else
+ goto bad_input;
+
+ assert(bJavaUtf8 ? nChar <= 0xFFFF : rtl::isUnicodeScalarValue(nChar));
+
+ if (nChar <= 0x7F && (!bJavaUtf8 || nChar != 0))
+ if (pDestBufPtr != pDestBufEnd)
+ *pDestBufPtr++ = static_cast< char >(nChar);
+ else
+ goto no_output;
+ else if (nChar <= 0x7FF)
+ if (pDestBufEnd - pDestBufPtr >= 2)
+ {
+ *pDestBufPtr++ = static_cast< char >(0xC0 | (nChar >> 6));
+ *pDestBufPtr++ = static_cast< char >(0x80 | (nChar & 0x3F));
+ }
+ else
+ goto no_output;
+ else if (nChar <= 0xFFFF)
+ if (pDestBufEnd - pDestBufPtr >= 3)
+ {
+ *pDestBufPtr++ = static_cast< char >(0xE0 | (nChar >> 12));
+ *pDestBufPtr++ = static_cast< char >(0x80 | ((nChar >> 6) & 0x3F));
+ *pDestBufPtr++ = static_cast< char >(0x80 | (nChar & 0x3F));
+ }
+ else
+ goto no_output;
+ else if (pDestBufEnd - pDestBufPtr >= 4)
+ {
+ *pDestBufPtr++ = static_cast< char >(0xF0 | (nChar >> 18));
+ *pDestBufPtr++ = static_cast< char >(0x80 | ((nChar >> 12) & 0x3F));
+ *pDestBufPtr++ = static_cast< char >(0x80 | ((nChar >> 6) & 0x3F));
+ *pDestBufPtr++ = static_cast< char >(0x80 | (nChar & 0x3F));
+ }
+ else
+ goto no_output;
+ nHighSurrogate = 0;
+ continue;
+
+ bad_input:
+ switch (sal::detail::textenc::handleBadInputUnicodeToTextConversion(
+ false, 0, nFlags, &pDestBufPtr, pDestBufEnd, &nInfo, nullptr,
+ 0, nullptr))
+ {
+ case sal::detail::textenc::BAD_INPUT_STOP:
+ nHighSurrogate = 0;
+ break;
+
+ case sal::detail::textenc::BAD_INPUT_CONTINUE:
+ nHighSurrogate = 0;
+ continue;
+
+ case sal::detail::textenc::BAD_INPUT_NO_OUTPUT:
+ goto no_output;
+ }
+ break;
+
+ no_output:
+ --pSrcBufPtr;
+ nInfo |= RTL_UNICODETOTEXT_INFO_DESTBUFFERTOSMALL;
+ break;
+ }
+
+ if (nHighSurrogate != 0
+ && (nInfo & (RTL_UNICODETOTEXT_INFO_ERROR
+ | RTL_UNICODETOTEXT_INFO_DESTBUFFERTOSMALL))
+ == 0)
+ {
+ if ((nFlags & RTL_UNICODETOTEXT_FLAGS_FLUSH) != 0)
+ nInfo |= RTL_UNICODETOTEXT_INFO_SRCBUFFERTOSMALL;
+ else
+ switch (sal::detail::textenc::handleBadInputUnicodeToTextConversion(
+ false, 0, nFlags, &pDestBufPtr, pDestBufEnd, &nInfo,
+ nullptr, 0, nullptr))
+ {
+ case sal::detail::textenc::BAD_INPUT_STOP:
+ case sal::detail::textenc::BAD_INPUT_CONTINUE:
+ nHighSurrogate = 0;
+ break;
+
+ case sal::detail::textenc::BAD_INPUT_NO_OUTPUT:
+ nInfo |= RTL_UNICODETOTEXT_INFO_DESTBUFFERTOSMALL;
+ break;
+ }
+ }
+
+ done:
+ if (pContext != nullptr)
+ static_cast< ImplUnicodeToUtf8Context * >(pContext)->nHighSurrogate
+ = nHighSurrogate;
+ if (pInfo != nullptr)
+ *pInfo = nInfo;
+ if (pSrcCvtChars != nullptr)
+ *pSrcCvtChars = pSrcBufPtr - pSrcBuf;
+ return pDestBufPtr - pDestBuf;
+}
+
+/* vim:set shiftwidth=4 softtabstop=4 expandtab: */