// © 2016 and later: Unicode, Inc. and others. // License & terms of use: http://www.unicode.org/copyright.html /* ******************************************************************************* * * Copyright (C) 2009-2016, International Business Machines * Corporation and others. All Rights Reserved. * ******************************************************************************* * file name: normalizer2.cpp * encoding: UTF-8 * tab size: 8 (not used) * indentation:4 * * created on: 2009nov22 * created by: Markus W. Scherer */ #include "unicode/utypes.h" #if !UCONFIG_NO_NORMALIZATION #include "unicode/edits.h" #include "unicode/normalizer2.h" #include "unicode/stringoptions.h" #include "unicode/unistr.h" #include "unicode/unorm.h" #include "cstring.h" #include "mutex.h" #include "norm2allmodes.h" #include "normalizer2impl.h" #include "uassert.h" #include "ucln_cmn.h" using icu::Normalizer2Impl; #if NORM2_HARDCODE_NFC_DATA // NFC/NFD data machine-generated by gennorm2 --csource #define INCLUDED_FROM_NORMALIZER2_CPP #include "norm2_nfc_data.h" #endif U_NAMESPACE_BEGIN // Public API dispatch via Normalizer2 subclasses -------------------------- *** Normalizer2::~Normalizer2() {} void Normalizer2::normalizeUTF8(uint32_t /*options*/, StringPiece src, ByteSink &sink, Edits *edits, UErrorCode &errorCode) const { if (U_FAILURE(errorCode)) { return; } if (edits != nullptr) { errorCode = U_UNSUPPORTED_ERROR; return; } UnicodeString src16 = UnicodeString::fromUTF8(src); normalize(src16, errorCode).toUTF8(sink); } UBool Normalizer2::getRawDecomposition(UChar32, UnicodeString &) const { return false; } UChar32 Normalizer2::composePair(UChar32, UChar32) const { return U_SENTINEL; } uint8_t Normalizer2::getCombiningClass(UChar32 /*c*/) const { return 0; } UBool Normalizer2::isNormalizedUTF8(StringPiece s, UErrorCode &errorCode) const { return U_SUCCESS(errorCode) && isNormalized(UnicodeString::fromUTF8(s), errorCode); } // Normalizer2 implementation for the old UNORM_NONE. class NoopNormalizer2 : public Normalizer2 { virtual ~NoopNormalizer2(); virtual UnicodeString & normalize(const UnicodeString &src, UnicodeString &dest, UErrorCode &errorCode) const override { if(U_SUCCESS(errorCode)) { if(&dest!=&src) { dest=src; } else { errorCode=U_ILLEGAL_ARGUMENT_ERROR; } } return dest; } virtual void normalizeUTF8(uint32_t options, StringPiece src, ByteSink &sink, Edits *edits, UErrorCode &errorCode) const override { if(U_SUCCESS(errorCode)) { if (edits != nullptr) { if ((options & U_EDITS_NO_RESET) == 0) { edits->reset(); } edits->addUnchanged(src.length()); } if ((options & U_OMIT_UNCHANGED_TEXT) == 0) { sink.Append(src.data(), src.length()); } sink.Flush(); } } virtual UnicodeString & normalizeSecondAndAppend(UnicodeString &first, const UnicodeString &second, UErrorCode &errorCode) const override { if(U_SUCCESS(errorCode)) { if(&first!=&second) { first.append(second); } else { errorCode=U_ILLEGAL_ARGUMENT_ERROR; } } return first; } virtual UnicodeString & append(UnicodeString &first, const UnicodeString &second, UErrorCode &errorCode) const override { if(U_SUCCESS(errorCode)) { if(&first!=&second) { first.append(second); } else { errorCode=U_ILLEGAL_ARGUMENT_ERROR; } } return first; } virtual UBool getDecomposition(UChar32, UnicodeString &) const override { return false; } // No need to override the default getRawDecomposition(). virtual UBool isNormalized(const UnicodeString &, UErrorCode &errorCode) const override { return U_SUCCESS(errorCode); } virtual UBool isNormalizedUTF8(StringPiece, UErrorCode &errorCode) const override { return U_SUCCESS(errorCode); } virtual UNormalizationCheckResult quickCheck(const UnicodeString &, UErrorCode &) const override { return UNORM_YES; } virtual int32_t spanQuickCheckYes(const UnicodeString &s, UErrorCode &) const override { return s.length(); } virtual UBool hasBoundaryBefore(UChar32) const override { return true; } virtual UBool hasBoundaryAfter(UChar32) const override { return true; } virtual UBool isInert(UChar32) const override { return true; } }; NoopNormalizer2::~NoopNormalizer2() {} Normalizer2WithImpl::~Normalizer2WithImpl() {} DecomposeNormalizer2::~DecomposeNormalizer2() {} ComposeNormalizer2::~ComposeNormalizer2() {} FCDNormalizer2::~FCDNormalizer2() {} // instance cache ---------------------------------------------------------- *** U_CDECL_BEGIN static UBool U_CALLCONV uprv_normalizer2_cleanup(); U_CDECL_END static Normalizer2 *noopSingleton; static icu::UInitOnce noopInitOnce {}; static void U_CALLCONV initNoopSingleton(UErrorCode &errorCode) { if(U_FAILURE(errorCode)) { return; } noopSingleton=new NoopNormalizer2; if(noopSingleton==nullptr) { errorCode=U_MEMORY_ALLOCATION_ERROR; return; } ucln_common_registerCleanup(UCLN_COMMON_NORMALIZER2, uprv_normalizer2_cleanup); } const Normalizer2 *Normalizer2Factory::getNoopInstance(UErrorCode &errorCode) { if(U_FAILURE(errorCode)) { return nullptr; } umtx_initOnce(noopInitOnce, &initNoopSingleton, errorCode); return noopSingleton; } const Normalizer2Impl * Normalizer2Factory::getImpl(const Normalizer2 *norm2) { return &((Normalizer2WithImpl *)norm2)->impl; } Norm2AllModes::~Norm2AllModes() { delete impl; } Norm2AllModes * Norm2AllModes::createInstance(Normalizer2Impl *impl, UErrorCode &errorCode) { if(U_FAILURE(errorCode)) { delete impl; return nullptr; } Norm2AllModes *allModes=new Norm2AllModes(impl); if(allModes==nullptr) { errorCode=U_MEMORY_ALLOCATION_ERROR; delete impl; return nullptr; } return allModes; } #if NORM2_HARDCODE_NFC_DATA Norm2AllModes * Norm2AllModes::createNFCInstance(UErrorCode &errorCode) { if(U_FAILURE(errorCode)) { return nullptr; } Normalizer2Impl *impl=new Normalizer2Impl; if(impl==nullptr) { errorCode=U_MEMORY_ALLOCATION_ERROR; return nullptr; } impl->init(norm2_nfc_data_indexes, &norm2_nfc_data_trie, norm2_nfc_data_extraData, norm2_nfc_data_smallFCD); return createInstance(impl, errorCode); } static Norm2AllModes *nfcSingleton; static icu::UInitOnce nfcInitOnce {}; static void U_CALLCONV initNFCSingleton(UErrorCode &errorCode) { nfcSingleton=Norm2AllModes::createNFCInstance(errorCode); ucln_common_registerCleanup(UCLN_COMMON_NORMALIZER2, uprv_normalizer2_cleanup); } const Norm2AllModes * Norm2AllModes::getNFCInstance(UErrorCode &errorCode) { if(U_FAILURE(errorCode)) { return nullptr; } umtx_initOnce(nfcInitOnce, &initNFCSingleton, errorCode); return nfcSingleton; } const Normalizer2 * Normalizer2::getNFCInstance(UErrorCode &errorCode) { const Norm2AllModes *allModes=Norm2AllModes::getNFCInstance(errorCode); return allModes!=nullptr ? &allModes->comp : nullptr; } const Normalizer2 * Normalizer2::getNFDInstance(UErrorCode &errorCode) { const Norm2AllModes *allModes=Norm2AllModes::getNFCInstance(errorCode); return allModes!=nullptr ? &allModes->decomp : nullptr; } const Normalizer2 *Normalizer2Factory::getFCDInstance(UErrorCode &errorCode) { const Norm2AllModes *allModes=Norm2AllModes::getNFCInstance(errorCode); return allModes!=nullptr ? &allModes->fcd : nullptr; } const Normalizer2 *Normalizer2Factory::getFCCInstance(UErrorCode &errorCode) { const Norm2AllModes *allModes=Norm2AllModes::getNFCInstance(errorCode); return allModes!=nullptr ? &allModes->fcc : nullptr; } const Normalizer2Impl * Normalizer2Factory::getNFCImpl(UErrorCode &errorCode) { const Norm2AllModes *allModes=Norm2AllModes::getNFCInstance(errorCode); return allModes!=nullptr ? allModes->impl : nullptr; } #endif // NORM2_HARDCODE_NFC_DATA U_CDECL_BEGIN static UBool U_CALLCONV uprv_normalizer2_cleanup() { delete noopSingleton; noopSingleton = nullptr; noopInitOnce.reset(); #if NORM2_HARDCODE_NFC_DATA delete nfcSingleton; nfcSingleton = nullptr; nfcInitOnce.reset(); #endif return true; } U_CDECL_END U_NAMESPACE_END // C API ------------------------------------------------------------------- *** U_NAMESPACE_USE U_CAPI const UNormalizer2 * U_EXPORT2 unorm2_getNFCInstance(UErrorCode *pErrorCode) { return (const UNormalizer2 *)Normalizer2::getNFCInstance(*pErrorCode); } U_CAPI const UNormalizer2 * U_EXPORT2 unorm2_getNFDInstance(UErrorCode *pErrorCode) { return (const UNormalizer2 *)Normalizer2::getNFDInstance(*pErrorCode); } U_CAPI void U_EXPORT2 unorm2_close(UNormalizer2 *norm2) { delete (Normalizer2 *)norm2; } U_CAPI int32_t U_EXPORT2 unorm2_normalize(const UNormalizer2 *norm2, const char16_t *src, int32_t length, char16_t *dest, int32_t capacity, UErrorCode *pErrorCode) { if(U_FAILURE(*pErrorCode)) { return 0; } if( (src==nullptr ? length!=0 : length<-1) || (dest==nullptr ? capacity!=0 : capacity<0) || (src==dest && src!=nullptr) ) { *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; return 0; } UnicodeString destString(dest, 0, capacity); // length==0: Nothing to do, and n2wi->normalize(nullptr, nullptr, buffer, ...) would crash. if(length!=0) { const Normalizer2 *n2=(const Normalizer2 *)norm2; const Normalizer2WithImpl *n2wi=dynamic_cast(n2); if(n2wi!=nullptr) { // Avoid duplicate argument checking and support NUL-terminated src. ReorderingBuffer buffer(n2wi->impl, destString); if(buffer.init(length, *pErrorCode)) { n2wi->normalize(src, length>=0 ? src+length : nullptr, buffer, *pErrorCode); } } else { UnicodeString srcString(length<0, src, length); n2->normalize(srcString, destString, *pErrorCode); } } return destString.extract(dest, capacity, *pErrorCode); } static int32_t normalizeSecondAndAppend(const UNormalizer2 *norm2, char16_t *first, int32_t firstLength, int32_t firstCapacity, const char16_t *second, int32_t secondLength, UBool doNormalize, UErrorCode *pErrorCode) { if(U_FAILURE(*pErrorCode)) { return 0; } if( (second==nullptr ? secondLength!=0 : secondLength<-1) || (first==nullptr ? (firstCapacity!=0 || firstLength!=0) : (firstCapacity<0 || firstLength<-1)) || (first==second && first!=nullptr) ) { *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; return 0; } UnicodeString firstString(first, firstLength, firstCapacity); firstLength=firstString.length(); // In case it was -1. // secondLength==0: Nothing to do, and n2wi->normalizeAndAppend(nullptr, nullptr, buffer, ...) would crash. if(secondLength!=0) { const Normalizer2 *n2=(const Normalizer2 *)norm2; const Normalizer2WithImpl *n2wi=dynamic_cast(n2); if(n2wi!=nullptr) { // Avoid duplicate argument checking and support NUL-terminated src. UnicodeString safeMiddle; { ReorderingBuffer buffer(n2wi->impl, firstString); if(buffer.init(firstLength+secondLength+1, *pErrorCode)) { // destCapacity>=-1 n2wi->normalizeAndAppend(second, secondLength>=0 ? second+secondLength : nullptr, doNormalize, safeMiddle, buffer, *pErrorCode); } } // The ReorderingBuffer destructor finalizes firstString. if(U_FAILURE(*pErrorCode) || firstString.length()>firstCapacity) { // Restore the modified suffix of the first string. // This does not restore first[] array contents between firstLength and firstCapacity. // (That might be uninitialized memory, as far as we know.) if(first!=nullptr) { /* don't dereference nullptr */ safeMiddle.extract(0, 0x7fffffff, first+firstLength-safeMiddle.length()); if(firstLengthnormalizeSecondAndAppend(firstString, secondString, *pErrorCode); } else { n2->append(firstString, secondString, *pErrorCode); } } } return firstString.extract(first, firstCapacity, *pErrorCode); } U_CAPI int32_t U_EXPORT2 unorm2_normalizeSecondAndAppend(const UNormalizer2 *norm2, char16_t *first, int32_t firstLength, int32_t firstCapacity, const char16_t *second, int32_t secondLength, UErrorCode *pErrorCode) { return normalizeSecondAndAppend(norm2, first, firstLength, firstCapacity, second, secondLength, true, pErrorCode); } U_CAPI int32_t U_EXPORT2 unorm2_append(const UNormalizer2 *norm2, char16_t *first, int32_t firstLength, int32_t firstCapacity, const char16_t *second, int32_t secondLength, UErrorCode *pErrorCode) { return normalizeSecondAndAppend(norm2, first, firstLength, firstCapacity, second, secondLength, false, pErrorCode); } U_CAPI int32_t U_EXPORT2 unorm2_getDecomposition(const UNormalizer2 *norm2, UChar32 c, char16_t *decomposition, int32_t capacity, UErrorCode *pErrorCode) { if(U_FAILURE(*pErrorCode)) { return 0; } if(decomposition==nullptr ? capacity!=0 : capacity<0) { *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; return 0; } UnicodeString destString(decomposition, 0, capacity); if(reinterpret_cast(norm2)->getDecomposition(c, destString)) { return destString.extract(decomposition, capacity, *pErrorCode); } else { return -1; } } U_CAPI int32_t U_EXPORT2 unorm2_getRawDecomposition(const UNormalizer2 *norm2, UChar32 c, char16_t *decomposition, int32_t capacity, UErrorCode *pErrorCode) { if(U_FAILURE(*pErrorCode)) { return 0; } if(decomposition==nullptr ? capacity!=0 : capacity<0) { *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; return 0; } UnicodeString destString(decomposition, 0, capacity); if(reinterpret_cast(norm2)->getRawDecomposition(c, destString)) { return destString.extract(decomposition, capacity, *pErrorCode); } else { return -1; } } U_CAPI UChar32 U_EXPORT2 unorm2_composePair(const UNormalizer2 *norm2, UChar32 a, UChar32 b) { return reinterpret_cast(norm2)->composePair(a, b); } U_CAPI uint8_t U_EXPORT2 unorm2_getCombiningClass(const UNormalizer2 *norm2, UChar32 c) { return reinterpret_cast(norm2)->getCombiningClass(c); } U_CAPI UBool U_EXPORT2 unorm2_isNormalized(const UNormalizer2 *norm2, const char16_t *s, int32_t length, UErrorCode *pErrorCode) { if(U_FAILURE(*pErrorCode)) { return 0; } if((s==nullptr && length!=0) || length<-1) { *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; return 0; } UnicodeString sString(length<0, s, length); return ((const Normalizer2 *)norm2)->isNormalized(sString, *pErrorCode); } U_CAPI UNormalizationCheckResult U_EXPORT2 unorm2_quickCheck(const UNormalizer2 *norm2, const char16_t *s, int32_t length, UErrorCode *pErrorCode) { if(U_FAILURE(*pErrorCode)) { return UNORM_NO; } if((s==nullptr && length!=0) || length<-1) { *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; return UNORM_NO; } UnicodeString sString(length<0, s, length); return ((const Normalizer2 *)norm2)->quickCheck(sString, *pErrorCode); } U_CAPI int32_t U_EXPORT2 unorm2_spanQuickCheckYes(const UNormalizer2 *norm2, const char16_t *s, int32_t length, UErrorCode *pErrorCode) { if(U_FAILURE(*pErrorCode)) { return 0; } if((s==nullptr && length!=0) || length<-1) { *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; return 0; } UnicodeString sString(length<0, s, length); return ((const Normalizer2 *)norm2)->spanQuickCheckYes(sString, *pErrorCode); } U_CAPI UBool U_EXPORT2 unorm2_hasBoundaryBefore(const UNormalizer2 *norm2, UChar32 c) { return ((const Normalizer2 *)norm2)->hasBoundaryBefore(c); } U_CAPI UBool U_EXPORT2 unorm2_hasBoundaryAfter(const UNormalizer2 *norm2, UChar32 c) { return ((const Normalizer2 *)norm2)->hasBoundaryAfter(c); } U_CAPI UBool U_EXPORT2 unorm2_isInert(const UNormalizer2 *norm2, UChar32 c) { return ((const Normalizer2 *)norm2)->isInert(c); } // Some properties APIs ---------------------------------------------------- *** U_CAPI uint8_t U_EXPORT2 u_getCombiningClass(UChar32 c) { UErrorCode errorCode=U_ZERO_ERROR; const Normalizer2 *nfd=Normalizer2::getNFDInstance(errorCode); if(U_SUCCESS(errorCode)) { return nfd->getCombiningClass(c); } else { return 0; } } U_CFUNC uint16_t unorm_getFCD16(UChar32 c) { UErrorCode errorCode=U_ZERO_ERROR; const Normalizer2Impl *impl=Normalizer2Factory::getNFCImpl(errorCode); if(U_SUCCESS(errorCode)) { return impl->getFCD16(c); } else { return 0; } } #endif // !UCONFIG_NO_NORMALIZATION