Adding upstream version 115.7.0esr.upstream/115.7.0esrupstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 482 insertions, 0 deletions

diff --git a/intl/icu/source/i18n/collationdatareader.cpp b/intl/icu/source/i18n/collationdatareader.cpp
new file mode 100644
index 0000000000..1884208eaa
--- /dev/null
+++ b/intl/icu/source/i18n/collationdatareader.cpp
@@ -0,0 +1,482 @@
+// © 2016 and later: Unicode, Inc. and others.
+// License & terms of use: http://www.unicode.org/copyright.html
+/*
+*******************************************************************************
+* Copyright (C) 2013-2015, International Business Machines
+* Corporation and others.  All Rights Reserved.
+*******************************************************************************
+* collationdatareader.cpp
+*
+* created on: 2013feb07
+* created by: Markus W. Scherer
+*/
+
+#include "unicode/utypes.h"
+
+#if !UCONFIG_NO_COLLATION
+
+#include "unicode/ucol.h"
+#include "unicode/udata.h"
+#include "unicode/uscript.h"
+#include "cmemory.h"
+#include "collation.h"
+#include "collationdata.h"
+#include "collationdatareader.h"
+#include "collationfastlatin.h"
+#include "collationkeys.h"
+#include "collationrootelements.h"
+#include "collationsettings.h"
+#include "collationtailoring.h"
+#include "collunsafe.h"
+#include "normalizer2impl.h"
+#include "uassert.h"
+#include "ucmndata.h"
+#include "utrie2.h"
+
+U_NAMESPACE_BEGIN
+
+namespace {
+
+int32_t getIndex(const int32_t *indexes, int32_t length, int32_t i) {
+    return (i < length) ? indexes[i] : -1;
+}
+
+}  // namespace
+
+void
+CollationDataReader::read(const CollationTailoring *base, const uint8_t *inBytes, int32_t inLength,
+                          CollationTailoring &tailoring, UErrorCode &errorCode) {
+    if(U_FAILURE(errorCode)) { return; }
+    if(base != nullptr) {
+        if(inBytes == nullptr || (0 <= inLength && inLength < 24)) {
+            errorCode = U_ILLEGAL_ARGUMENT_ERROR;
+            return;
+        }
+        const DataHeader *header = reinterpret_cast<const DataHeader *>(inBytes);
+        if(!(header->dataHeader.magic1 == 0xda && header->dataHeader.magic2 == 0x27 &&
+                isAcceptable(tailoring.version, nullptr, nullptr, &header->info))) {
+            errorCode = U_INVALID_FORMAT_ERROR;
+            return;
+        }
+        if(base->getUCAVersion() != tailoring.getUCAVersion()) {
+            errorCode = U_COLLATOR_VERSION_MISMATCH;
+            return;
+        }
+        int32_t headerLength = header->dataHeader.headerSize;
+        inBytes += headerLength;
+        if(inLength >= 0) {
+            inLength -= headerLength;
+        }
+    }
+
+    if(inBytes == nullptr || (0 <= inLength && inLength < 8)) {
+        errorCode = U_ILLEGAL_ARGUMENT_ERROR;
+        return;
+    }
+    const int32_t *inIndexes = reinterpret_cast<const int32_t *>(inBytes);
+    int32_t indexesLength = inIndexes[IX_INDEXES_LENGTH];
+    if(indexesLength < 2 || (0 <= inLength && inLength < indexesLength * 4)) {
+        errorCode = U_INVALID_FORMAT_ERROR;  // Not enough indexes.
+        return;
+    }
+
+    // Assume that the tailoring data is in initial state,
+    // with nullptr pointers and 0 lengths.
+
+    // Set pointers to non-empty data parts.
+    // Do this in order of their byte offsets. (Should help porting to Java.)
+
+    int32_t index;  // one of the indexes[] slots
+    int32_t offset;  // byte offset for the index part
+    int32_t length;  // number of bytes in the index part
+
+    if(indexesLength > IX_TOTAL_SIZE) {
+        length = inIndexes[IX_TOTAL_SIZE];
+    } else if(indexesLength > IX_REORDER_CODES_OFFSET) {
+        length = inIndexes[indexesLength - 1];
+    } else {
+        length = 0;  // only indexes, and inLength was already checked for them
+    }
+    if(0 <= inLength && inLength < length) {
+        errorCode = U_INVALID_FORMAT_ERROR;
+        return;
+    }
+
+    const CollationData *baseData = base == nullptr ? nullptr : base->data;
+    const int32_t *reorderCodes = nullptr;
+    int32_t reorderCodesLength = 0;
+    const uint32_t *reorderRanges = nullptr;
+    int32_t reorderRangesLength = 0;
+    index = IX_REORDER_CODES_OFFSET;
+    offset = getIndex(inIndexes, indexesLength, index);
+    length = getIndex(inIndexes, indexesLength, index + 1) - offset;
+    if(length >= 4) {
+        if(baseData == nullptr) {
+            // We assume for collation settings that
+            // the base data does not have a reordering.
+            errorCode = U_INVALID_FORMAT_ERROR;
+            return;
+        }
+        reorderCodes = reinterpret_cast<const int32_t *>(inBytes + offset);
+        reorderCodesLength = length / 4;
+
+        // The reorderRanges (if any) are the trailing reorderCodes entries.
+        // Split the array at the boundary.
+        // Script or reorder codes do not exceed 16-bit values.
+        // Range limits are stored in the upper 16 bits, and are never 0.
+        while(reorderRangesLength < reorderCodesLength &&
+                (reorderCodes[reorderCodesLength - reorderRangesLength - 1] & 0xffff0000) != 0) {
+            ++reorderRangesLength;
+        }
+        U_ASSERT(reorderRangesLength < reorderCodesLength);
+        if(reorderRangesLength != 0) {
+            reorderCodesLength -= reorderRangesLength;
+            reorderRanges = reinterpret_cast<const uint32_t *>(reorderCodes + reorderCodesLength);
+        }
+    }
+
+    // There should be a reorder table only if there are reorder codes.
+    // However, when there are reorder codes the reorder table may be omitted to reduce
+    // the data size.
+    const uint8_t *reorderTable = nullptr;
+    index = IX_REORDER_TABLE_OFFSET;
+    offset = getIndex(inIndexes, indexesLength, index);
+    length = getIndex(inIndexes, indexesLength, index + 1) - offset;
+    if(length >= 256) {
+        if(reorderCodesLength == 0) {
+            errorCode = U_INVALID_FORMAT_ERROR;  // Reordering table without reordering codes.
+            return;
+        }
+        reorderTable = inBytes + offset;
+    } else {
+        // If we have reorder codes, then build the reorderTable at the end,
+        // when the CollationData is otherwise complete.
+    }
+
+    if(baseData != nullptr && baseData->numericPrimary != (inIndexes[IX_OPTIONS] & 0xff000000)) {
+        errorCode = U_INVALID_FORMAT_ERROR;
+        return;
+    }
+    CollationData *data = nullptr;  // Remains nullptr if there are no mappings.
+
+    index = IX_TRIE_OFFSET;
+    offset = getIndex(inIndexes, indexesLength, index);
+    length = getIndex(inIndexes, indexesLength, index + 1) - offset;
+    if(length >= 8) {
+        if(!tailoring.ensureOwnedData(errorCode)) { return; }
+        data = tailoring.ownedData;
+        data->base = baseData;
+        data->numericPrimary = inIndexes[IX_OPTIONS] & 0xff000000;
+        data->trie = tailoring.trie = utrie2_openFromSerialized(
+            UTRIE2_32_VALUE_BITS, inBytes + offset, length, nullptr,
+            &errorCode);
+        if(U_FAILURE(errorCode)) { return; }
+    } else if(baseData != nullptr) {
+        // Use the base data. Only the settings are tailored.
+        tailoring.data = baseData;
+    } else {
+        errorCode = U_INVALID_FORMAT_ERROR;  // No mappings.
+        return;
+    }
+
+    index = IX_CES_OFFSET;
+    offset = getIndex(inIndexes, indexesLength, index);
+    length = getIndex(inIndexes, indexesLength, index + 1) - offset;
+    if(length >= 8) {
+        if(data == nullptr) {
+            errorCode = U_INVALID_FORMAT_ERROR;  // Tailored ces without tailored trie.
+            return;
+        }
+        data->ces = reinterpret_cast<const int64_t *>(inBytes + offset);
+        data->cesLength = length / 8;
+    }
+
+    index = IX_CE32S_OFFSET;
+    offset = getIndex(inIndexes, indexesLength, index);
+    length = getIndex(inIndexes, indexesLength, index + 1) - offset;
+    if(length >= 4) {
+        if(data == nullptr) {
+            errorCode = U_INVALID_FORMAT_ERROR;  // Tailored ce32s without tailored trie.
+            return;
+        }
+        data->ce32s = reinterpret_cast<const uint32_t *>(inBytes + offset);
+        data->ce32sLength = length / 4;
+    }
+
+    int32_t jamoCE32sStart = getIndex(inIndexes, indexesLength, IX_JAMO_CE32S_START);
+    if(jamoCE32sStart >= 0) {
+        if(data == nullptr || data->ce32s == nullptr) {
+            errorCode = U_INVALID_FORMAT_ERROR;  // Index into non-existent ce32s[].
+            return;
+        }
+        data->jamoCE32s = data->ce32s + jamoCE32sStart;
+    } else if(data == nullptr) {
+        // Nothing to do.
+    } else if(baseData != nullptr) {
+        data->jamoCE32s = baseData->jamoCE32s;
+    } else {
+        errorCode = U_INVALID_FORMAT_ERROR;  // No Jamo CE32s for Hangul processing.
+        return;
+    }
+
+    index = IX_ROOT_ELEMENTS_OFFSET;
+    offset = getIndex(inIndexes, indexesLength, index);
+    length = getIndex(inIndexes, indexesLength, index + 1) - offset;
+    if(length >= 4) {
+        length /= 4;
+        if(data == nullptr || length <= CollationRootElements::IX_SEC_TER_BOUNDARIES) {
+            errorCode = U_INVALID_FORMAT_ERROR;
+            return;
+        }
+        data->rootElements = reinterpret_cast<const uint32_t *>(inBytes + offset);
+        data->rootElementsLength = length;
+        uint32_t commonSecTer = data->rootElements[CollationRootElements::IX_COMMON_SEC_AND_TER_CE];
+        if(commonSecTer != Collation::COMMON_SEC_AND_TER_CE) {
+            errorCode = U_INVALID_FORMAT_ERROR;
+            return;
+        }
+        uint32_t secTerBoundaries = data->rootElements[CollationRootElements::IX_SEC_TER_BOUNDARIES];
+        if((secTerBoundaries >> 24) < CollationKeys::SEC_COMMON_HIGH) {
+            // [fixed last secondary common byte] is too low,
+            // and secondary weights would collide with compressed common secondaries.
+            errorCode = U_INVALID_FORMAT_ERROR;
+            return;
+        }
+    }
+
+    index = IX_CONTEXTS_OFFSET;
+    offset = getIndex(inIndexes, indexesLength, index);
+    length = getIndex(inIndexes, indexesLength, index + 1) - offset;
+    if(length >= 2) {
+        if(data == nullptr) {
+            errorCode = U_INVALID_FORMAT_ERROR;  // Tailored contexts without tailored trie.
+            return;
+        }
+        data->contexts = reinterpret_cast<const char16_t *>(inBytes + offset);
+        data->contextsLength = length / 2;
+    }
+
+    index = IX_UNSAFE_BWD_OFFSET;
+    offset = getIndex(inIndexes, indexesLength, index);
+    length = getIndex(inIndexes, indexesLength, index + 1) - offset;
+    if(length >= 2) {
+        if(data == nullptr) {
+            errorCode = U_INVALID_FORMAT_ERROR;
+            return;
+        }
+        if(baseData == nullptr) {
+#if defined(COLLUNSAFE_COLL_VERSION) && defined (COLLUNSAFE_SERIALIZE)
+          tailoring.unsafeBackwardSet = new UnicodeSet(unsafe_serializedData, unsafe_serializedCount, UnicodeSet::kSerialized, errorCode);
+          if(tailoring.unsafeBackwardSet == nullptr) {
+            errorCode = U_MEMORY_ALLOCATION_ERROR;
+            return;
+          } else if (U_FAILURE(errorCode)) {
+            return;
+          }
+#else
+            // Create the unsafe-backward set for the root collator.
+            // Include all non-zero combining marks and trail surrogates.
+            // We do this at load time, rather than at build time,
+            // to simplify Unicode version bootstrapping:
+            // The root data builder only needs the new FractionalUCA.txt data,
+            // but it need not be built with a version of ICU already updated to
+            // the corresponding new Unicode Character Database.
+            //
+            // The following is an optimized version of
+            // new UnicodeSet("[[:^lccc=0:][\\udc00-\\udfff]]").
+            // It is faster and requires fewer code dependencies.
+            tailoring.unsafeBackwardSet = new UnicodeSet(0xdc00, 0xdfff);  // trail surrogates
+            if(tailoring.unsafeBackwardSet == nullptr) {
+                errorCode = U_MEMORY_ALLOCATION_ERROR;
+                return;
+            }
+            data->nfcImpl.addLcccChars(*tailoring.unsafeBackwardSet);
+#endif // !COLLUNSAFE_SERIALIZE || !COLLUNSAFE_COLL_VERSION
+        } else {
+            // Clone the root collator's set contents.
+            tailoring.unsafeBackwardSet = static_cast<UnicodeSet *>(
+                baseData->unsafeBackwardSet->cloneAsThawed());
+            if(tailoring.unsafeBackwardSet == nullptr) {
+                errorCode = U_MEMORY_ALLOCATION_ERROR;
+                return;
+            }
+        }
+        // Add the ranges from the data file to the unsafe-backward set.
+        USerializedSet sset;
+        const uint16_t *unsafeData = reinterpret_cast<const uint16_t *>(inBytes + offset);
+        if(!uset_getSerializedSet(&sset, unsafeData, length / 2)) {
+            errorCode = U_INVALID_FORMAT_ERROR;
+            return;
+        }
+        int32_t count = uset_getSerializedRangeCount(&sset);
+        for(int32_t i = 0; i < count; ++i) {
+            UChar32 start, end;
+            uset_getSerializedRange(&sset, i, &start, &end);
+            tailoring.unsafeBackwardSet->add(start, end);
+        }
+        // Mark each lead surrogate as "unsafe"
+        // if any of its 1024 associated supplementary code points is "unsafe".
+        UChar32 c = 0x10000;
+        for(char16_t lead = 0xd800; lead < 0xdc00; ++lead, c += 0x400) {
+            if(!tailoring.unsafeBackwardSet->containsNone(c, c + 0x3ff)) {
+                tailoring.unsafeBackwardSet->add(lead);
+            }
+        }
+        tailoring.unsafeBackwardSet->freeze();
+        data->unsafeBackwardSet = tailoring.unsafeBackwardSet;
+    } else if(data == nullptr) {
+        // Nothing to do.
+    } else if(baseData != nullptr) {
+        // No tailoring-specific data: Alias the root collator's set.
+        data->unsafeBackwardSet = baseData->unsafeBackwardSet;
+    } else {
+        errorCode = U_INVALID_FORMAT_ERROR;  // No unsafeBackwardSet.
+        return;
+    }
+
+    // If the fast Latin format version is different,
+    // or the version is set to 0 for "no fast Latin table",
+    // then just always use the normal string comparison path.
+    if(data != nullptr) {
+        data->fastLatinTable = nullptr;
+        data->fastLatinTableLength = 0;
+        if(((inIndexes[IX_OPTIONS] >> 16) & 0xff) == CollationFastLatin::VERSION) {
+            index = IX_FAST_LATIN_TABLE_OFFSET;
+            offset = getIndex(inIndexes, indexesLength, index);
+            length = getIndex(inIndexes, indexesLength, index + 1) - offset;
+            if(length >= 2) {
+                data->fastLatinTable = reinterpret_cast<const uint16_t *>(inBytes + offset);
+                data->fastLatinTableLength = length / 2;
+                if((*data->fastLatinTable >> 8) != CollationFastLatin::VERSION) {
+                    errorCode = U_INVALID_FORMAT_ERROR;  // header vs. table version mismatch
+                    return;
+                }
+            } else if(baseData != nullptr) {
+                data->fastLatinTable = baseData->fastLatinTable;
+                data->fastLatinTableLength = baseData->fastLatinTableLength;
+            }
+        }
+    }
+
+    index = IX_SCRIPTS_OFFSET;
+    offset = getIndex(inIndexes, indexesLength, index);
+    length = getIndex(inIndexes, indexesLength, index + 1) - offset;
+    if(length >= 2) {
+        if(data == nullptr) {
+            errorCode = U_INVALID_FORMAT_ERROR;
+            return;
+        }
+        const uint16_t *scripts = reinterpret_cast<const uint16_t *>(inBytes + offset);
+        int32_t scriptsLength = length / 2;
+        data->numScripts = scripts[0];
+        // There must be enough entries for both arrays, including more than two range starts.
+        data->scriptStartsLength = scriptsLength - (1 + data->numScripts + 16);
+        if(data->scriptStartsLength <= 2 ||
+                CollationData::MAX_NUM_SCRIPT_RANGES < data->scriptStartsLength) {
+            errorCode = U_INVALID_FORMAT_ERROR;
+            return;
+        }
+        data->scriptsIndex = scripts + 1;
+        data->scriptStarts = scripts + 1 + data->numScripts + 16;
+        if(!(data->scriptStarts[0] == 0 &&
+                data->scriptStarts[1] == ((Collation::MERGE_SEPARATOR_BYTE + 1) << 8) &&
+                data->scriptStarts[data->scriptStartsLength - 1] ==
+                        (Collation::TRAIL_WEIGHT_BYTE << 8))) {
+            errorCode = U_INVALID_FORMAT_ERROR;
+            return;
+        }
+    } else if(data == nullptr) {
+        // Nothing to do.
+    } else if(baseData != nullptr) {
+        data->numScripts = baseData->numScripts;
+        data->scriptsIndex = baseData->scriptsIndex;
+        data->scriptStarts = baseData->scriptStarts;
+        data->scriptStartsLength = baseData->scriptStartsLength;
+    }
+
+    index = IX_COMPRESSIBLE_BYTES_OFFSET;
+    offset = getIndex(inIndexes, indexesLength, index);
+    length = getIndex(inIndexes, indexesLength, index + 1) - offset;
+    if(length >= 256) {
+        if(data == nullptr) {
+            errorCode = U_INVALID_FORMAT_ERROR;
+            return;
+        }
+        data->compressibleBytes = reinterpret_cast<const UBool *>(inBytes + offset);
+    } else if(data == nullptr) {
+        // Nothing to do.
+    } else if(baseData != nullptr) {
+        data->compressibleBytes = baseData->compressibleBytes;
+    } else {
+        errorCode = U_INVALID_FORMAT_ERROR;  // No compressibleBytes[].
+        return;
+    }
+
+    const CollationSettings &ts = *tailoring.settings;
+    int32_t options = inIndexes[IX_OPTIONS] & 0xffff;
+    uint16_t fastLatinPrimaries[CollationFastLatin::LATIN_LIMIT];
+    int32_t fastLatinOptions = CollationFastLatin::getOptions(
+            tailoring.data, ts, fastLatinPrimaries, UPRV_LENGTHOF(fastLatinPrimaries));
+    if(options == ts.options && ts.variableTop != 0 &&
+            reorderCodesLength == ts.reorderCodesLength &&
+            (reorderCodesLength == 0 ||
+                uprv_memcmp(reorderCodes, ts.reorderCodes, reorderCodesLength * 4) == 0) &&
+            fastLatinOptions == ts.fastLatinOptions &&
+            (fastLatinOptions < 0 ||
+                uprv_memcmp(fastLatinPrimaries, ts.fastLatinPrimaries,
+                            sizeof(fastLatinPrimaries)) == 0)) {
+        return;
+    }
+
+    CollationSettings *settings = SharedObject::copyOnWrite(tailoring.settings);
+    if(settings == nullptr) {
+        errorCode = U_MEMORY_ALLOCATION_ERROR;
+        return;
+    }
+    settings->options = options;
+    // Set variableTop from options and scripts data.
+    settings->variableTop = tailoring.data->getLastPrimaryForGroup(
+            UCOL_REORDER_CODE_FIRST + int32_t{settings->getMaxVariable()});
+    if(settings->variableTop == 0) {
+        errorCode = U_INVALID_FORMAT_ERROR;
+        return;
+    }
+
+    if(reorderCodesLength != 0) {
+        settings->aliasReordering(*baseData, reorderCodes, reorderCodesLength,
+                                  reorderRanges, reorderRangesLength,
+                                  reorderTable, errorCode);
+    }
+
+    settings->fastLatinOptions = CollationFastLatin::getOptions(
+        tailoring.data, *settings,
+        settings->fastLatinPrimaries, UPRV_LENGTHOF(settings->fastLatinPrimaries));
+}
+
+UBool U_CALLCONV
+CollationDataReader::isAcceptable(void *context,
+                                  const char * /* type */, const char * /*name*/,
+                                  const UDataInfo *pInfo) {
+    if(
+        pInfo->size >= 20 &&
+        pInfo->isBigEndian == U_IS_BIG_ENDIAN &&
+        pInfo->charsetFamily == U_CHARSET_FAMILY &&
+        pInfo->dataFormat[0] == 0x55 &&  // dataFormat="UCol"
+        pInfo->dataFormat[1] == 0x43 &&
+        pInfo->dataFormat[2] == 0x6f &&
+        pInfo->dataFormat[3] == 0x6c &&
+        pInfo->formatVersion[0] == 5
+    ) {
+        UVersionInfo *version = static_cast<UVersionInfo *>(context);
+        if(version != nullptr) {
+            uprv_memcpy(version, pInfo->dataVersion, 4);
+        }
+        return true;
+    } else {
+        return false;
+    }
+}
+
+U_NAMESPACE_END
+
+#endif  // !UCONFIG_NO_COLLATION