Adding upstream version 115.7.0esr.upstream/115.7.0esrupstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 559 insertions, 0 deletions

diff --git a/intl/icu/source/i18n/rbt_rule.cpp b/intl/icu/source/i18n/rbt_rule.cpp
new file mode 100644
index 0000000000..da8e4eaa1f
--- /dev/null
+++ b/intl/icu/source/i18n/rbt_rule.cpp
@@ -0,0 +1,559 @@
+// © 2016 and later: Unicode, Inc. and others.
+// License & terms of use: http://www.unicode.org/copyright.html
+/*
+ **********************************************************************
+ *   Copyright (C) 1999-2011, International Business Machines
+ *   Corporation and others.  All Rights Reserved.
+ **********************************************************************
+ *   Date        Name        Description
+ *   11/17/99    aliu        Creation.
+ **********************************************************************
+ */
+
+#include "unicode/utypes.h"
+
+#if !UCONFIG_NO_TRANSLITERATION
+
+#include "unicode/rep.h"
+#include "unicode/unifilt.h"
+#include "unicode/uniset.h"
+#include "unicode/utf16.h"
+#include "rbt_rule.h"
+#include "rbt_data.h"
+#include "cmemory.h"
+#include "strmatch.h"
+#include "strrepl.h"
+#include "util.h"
+#include "putilimp.h"
+
+static const char16_t FORWARD_OP[] = {32,62,32,0}; // " > "
+
+U_NAMESPACE_BEGIN
+
+/**
+ * Construct a new rule with the given input, output text, and other
+ * attributes.  A cursor position may be specified for the output text.
+ * @param input input string, including key and optional ante and
+ * post context
+ * @param anteContextPos offset into input to end of ante context, or -1 if
+ * none.  Must be <= input.length() if not -1.
+ * @param postContextPos offset into input to start of post context, or -1
+ * if none.  Must be <= input.length() if not -1, and must be >=
+ * anteContextPos.
+ * @param output output string
+ * @param cursorPosition offset into output at which cursor is located, or -1 if
+ * none.  If less than zero, then the cursor is placed after the
+ * <code>output</code>; that is, -1 is equivalent to
+ * <code>output.length()</code>.  If greater than
+ * <code>output.length()</code> then an exception is thrown.
+ * @param segs array of UnicodeFunctors corresponding to input pattern
+ * segments, or null if there are none.  The array itself is adopted,
+ * but the pointers within it are not.
+ * @param segsCount number of elements in segs[]
+ * @param anchorStart true if the the rule is anchored on the left to
+ * the context start
+ * @param anchorEnd true if the rule is anchored on the right to the
+ * context limit
+ */
+TransliterationRule::TransliterationRule(const UnicodeString& input,
+                                         int32_t anteContextPos, int32_t postContextPos,
+                                         const UnicodeString& outputStr,
+                                         int32_t cursorPosition, int32_t cursorOffset,
+                                         UnicodeFunctor** segs,
+                                         int32_t segsCount,
+                                         UBool anchorStart, UBool anchorEnd,
+                                         const TransliterationRuleData* theData,
+                                         UErrorCode& status) :
+    UMemory(),
+    segments(0),
+    data(theData) {
+
+    if (U_FAILURE(status)) {
+        return;
+    }
+    // Do range checks only when warranted to save time
+    if (anteContextPos < 0) {
+        anteContextLength = 0;
+    } else {
+        if (anteContextPos > input.length()) {
+            // throw new IllegalArgumentException("Invalid ante context");
+            status = U_ILLEGAL_ARGUMENT_ERROR;
+            return;
+        }
+        anteContextLength = anteContextPos;
+    }
+    if (postContextPos < 0) {
+        keyLength = input.length() - anteContextLength;
+    } else {
+        if (postContextPos < anteContextLength ||
+            postContextPos > input.length()) {
+            // throw new IllegalArgumentException("Invalid post context");
+            status = U_ILLEGAL_ARGUMENT_ERROR;
+            return;
+        }
+        keyLength = postContextPos - anteContextLength;
+    }
+    if (cursorPosition < 0) {
+        cursorPosition = outputStr.length();
+    } else if (cursorPosition > outputStr.length()) {
+        // throw new IllegalArgumentException("Invalid cursor position");
+        status = U_ILLEGAL_ARGUMENT_ERROR;
+        return;
+    }
+    // We don't validate the segments array.  The caller must
+    // guarantee that the segments are well-formed (that is, that
+    // all $n references in the output refer to indices of this
+    // array, and that no array elements are null).
+    this->segments = segs;
+    this->segmentsCount = segsCount;
+
+    pattern = input;
+    flags = 0;
+    if (anchorStart) {
+        flags |= ANCHOR_START;
+    }
+    if (anchorEnd) {
+        flags |= ANCHOR_END;
+    }
+
+    anteContext = nullptr;
+    if (anteContextLength > 0) {
+        anteContext = new StringMatcher(pattern, 0, anteContextLength,
+                                        false, *data);
+        /* test for nullptr */
+        if (anteContext == 0) {
+            status = U_MEMORY_ALLOCATION_ERROR;
+            return;
+        }
+    }
+    
+    key = nullptr;
+    if (keyLength > 0) {
+        key = new StringMatcher(pattern, anteContextLength, anteContextLength + keyLength,
+                                false, *data);
+        /* test for nullptr */
+        if (key == 0) {
+            status = U_MEMORY_ALLOCATION_ERROR;
+            return;
+        }
+    }
+    
+    int32_t postContextLength = pattern.length() - keyLength - anteContextLength;
+    postContext = nullptr;
+    if (postContextLength > 0) {
+        postContext = new StringMatcher(pattern, anteContextLength + keyLength, pattern.length(),
+                                        false, *data);
+        /* test for nullptr */
+        if (postContext == 0) {
+            status = U_MEMORY_ALLOCATION_ERROR;
+            return;
+        }
+    }
+
+    this->output = new StringReplacer(outputStr, cursorPosition + cursorOffset, data);
+    /* test for nullptr */
+    if (this->output == 0) {
+        status = U_MEMORY_ALLOCATION_ERROR;
+        return;
+    }
+}
+
+/**
+ * Copy constructor.
+ */
+TransliterationRule::TransliterationRule(TransliterationRule& other) :
+    UMemory(other),
+    anteContext(nullptr),
+    key(nullptr),
+    postContext(nullptr),
+    pattern(other.pattern),
+    anteContextLength(other.anteContextLength),
+    keyLength(other.keyLength),
+    flags(other.flags),
+    data(other.data) {
+
+    segments = nullptr;
+    segmentsCount = 0;
+    if (other.segmentsCount > 0) {
+        segments = (UnicodeFunctor **)uprv_malloc(other.segmentsCount * sizeof(UnicodeFunctor *));
+        uprv_memcpy(segments, other.segments, (size_t)other.segmentsCount*sizeof(segments[0]));
+    }
+
+    if (other.anteContext != nullptr) {
+        anteContext = other.anteContext->clone();
+    }
+    if (other.key != nullptr) {
+        key = other.key->clone();
+    }
+    if (other.postContext != nullptr) {
+        postContext = other.postContext->clone();
+    }
+    output = other.output->clone();
+}
+
+TransliterationRule::~TransliterationRule() {
+    uprv_free(segments);
+    delete anteContext;
+    delete key;
+    delete postContext;
+    delete output;
+}
+
+/**
+ * Return the preceding context length.  This method is needed to
+ * support the <code>Transliterator</code> method
+ * <code>getMaximumContextLength()</code>.  Internally, this is
+ * implemented as the anteContextLength, optionally plus one if
+ * there is a start anchor.  The one character anchor gap is
+ * needed to make repeated incremental transliteration with
+ * anchors work.
+ */
+int32_t TransliterationRule::getContextLength() const {
+    return anteContextLength + ((flags & ANCHOR_START) ? 1 : 0);
+}
+
+/**
+ * Internal method.  Returns 8-bit index value for this rule.
+ * This is the low byte of the first character of the key,
+ * unless the first character of the key is a set.  If it's a
+ * set, or otherwise can match multiple keys, the index value is -1.
+ */
+int16_t TransliterationRule::getIndexValue() const {
+    if (anteContextLength == pattern.length()) {
+        // A pattern with just ante context {such as foo)>bar} can
+        // match any key.
+        return -1;
+    }
+    UChar32 c = pattern.char32At(anteContextLength);
+    return (int16_t)(data->lookupMatcher(c) == nullptr ? (c & 0xFF) : -1);
+}
+
+/**
+ * Internal method.  Returns true if this rule matches the given
+ * index value.  The index value is an 8-bit integer, 0..255,
+ * representing the low byte of the first character of the key.
+ * It matches this rule if it matches the first character of the
+ * key, or if the first character of the key is a set, and the set
+ * contains any character with a low byte equal to the index
+ * value.  If the rule contains only ante context, as in foo)>bar,
+ * then it will match any key.
+ */
+UBool TransliterationRule::matchesIndexValue(uint8_t v) const {
+    // Delegate to the key, or if there is none, to the postContext.
+    // If there is neither then we match any key; return true.
+    UnicodeMatcher *m = (key != nullptr) ? key : postContext;
+    return (m != nullptr) ? m->matchesIndexValue(v) : true;
+}
+
+/**
+ * Return true if this rule masks another rule.  If r1 masks r2 then
+ * r1 matches any input string that r2 matches.  If r1 masks r2 and r2 masks
+ * r1 then r1 == r2.  Examples: "a>x" masks "ab>y".  "a>x" masks "a[b]>y".
+ * "[c]a>x" masks "[dc]a>y".
+ */
+UBool TransliterationRule::masks(const TransliterationRule& r2) const {
+    /* Rule r1 masks rule r2 if the string formed of the
+     * antecontext, key, and postcontext overlaps in the following
+     * way:
+     *
+     * r1:      aakkkpppp
+     * r2:     aaakkkkkpppp
+     *            ^
+     * 
+     * The strings must be aligned at the first character of the
+     * key.  The length of r1 to the left of the alignment point
+     * must be <= the length of r2 to the left; ditto for the
+     * right.  The characters of r1 must equal (or be a superset
+     * of) the corresponding characters of r2.  The superset
+     * operation should be performed to check for UnicodeSet
+     * masking.
+     *
+     * Anchors:  Two patterns that differ only in anchors only
+     * mask one another if they are exactly equal, and r2 has
+     * all the anchors r1 has (optionally, plus some).  Here Y
+     * means the row masks the column, N means it doesn't.
+     *
+     *         ab   ^ab    ab$  ^ab$
+     *   ab    Y     Y     Y     Y
+     *  ^ab    N     Y     N     Y
+     *   ab$   N     N     Y     Y
+     *  ^ab$   N     N     N     Y
+     *
+     * Post context: {a}b masks ab, but not vice versa, since {a}b
+     * matches everything ab matches, and {a}b matches {|a|}b but ab
+     * does not.  Pre context is different (a{b} does not align with
+     * ab).
+     */
+
+    /* LIMITATION of the current mask algorithm: Some rule
+     * maskings are currently not detected.  For example,
+     * "{Lu}]a>x" masks "A]a>y".  This can be added later. TODO
+     */
+
+    int32_t len = pattern.length();
+    int32_t left = anteContextLength;
+    int32_t left2 = r2.anteContextLength;
+    int32_t right = len - left;
+    int32_t right2 = r2.pattern.length() - left2;
+    int32_t cachedCompare = r2.pattern.compare(left2 - left, len, pattern);
+
+    // TODO Clean this up -- some logic might be combinable with the
+    // next statement.
+
+    // Test for anchor masking
+    if (left == left2 && right == right2 &&
+        keyLength <= r2.keyLength &&
+        0 == cachedCompare) {
+        // The following boolean logic implements the table above
+        return (flags == r2.flags) ||
+            (!(flags & ANCHOR_START) && !(flags & ANCHOR_END)) ||
+            ((r2.flags & ANCHOR_START) && (r2.flags & ANCHOR_END));
+    }
+
+    return left <= left2 &&
+        (right < right2 ||
+         (right == right2 && keyLength <= r2.keyLength)) &&
+         (0 == cachedCompare);
+}
+
+static inline int32_t posBefore(const Replaceable& str, int32_t pos) {
+    return (pos > 0) ?
+        pos - U16_LENGTH(str.char32At(pos-1)) :
+        pos - 1;
+}
+
+static inline int32_t posAfter(const Replaceable& str, int32_t pos) {
+    return (pos >= 0 && pos < str.length()) ?
+        pos + U16_LENGTH(str.char32At(pos)) :
+        pos + 1;
+}
+
+/**
+ * Attempt a match and replacement at the given position.  Return
+ * the degree of match between this rule and the given text.  The
+ * degree of match may be mismatch, a partial match, or a full
+ * match.  A mismatch means at least one character of the text
+ * does not match the context or key.  A partial match means some
+ * context and key characters match, but the text is not long
+ * enough to match all of them.  A full match means all context
+ * and key characters match.
+ * 
+ * If a full match is obtained, perform a replacement, update pos,
+ * and return U_MATCH.  Otherwise both text and pos are unchanged.
+ * 
+ * @param text the text
+ * @param pos the position indices
+ * @param incremental if true, test for partial matches that may
+ * be completed by additional text inserted at pos.limit.
+ * @return one of <code>U_MISMATCH</code>,
+ * <code>U_PARTIAL_MATCH</code>, or <code>U_MATCH</code>.  If
+ * incremental is false then U_PARTIAL_MATCH will not be returned.
+ */
+UMatchDegree TransliterationRule::matchAndReplace(Replaceable& text,
+                                                  UTransPosition& pos,
+                                                  UBool incremental) const {
+    // Matching and replacing are done in one method because the
+    // replacement operation needs information obtained during the
+    // match.  Another way to do this is to have the match method
+    // create a match result struct with relevant offsets, and to pass
+    // this into the replace method.
+
+    // ============================ MATCH ===========================
+
+    // Reset segment match data
+    if (segments != nullptr) {
+        for (int32_t i=0; i<segmentsCount; ++i) {
+            ((StringMatcher*) segments[i])->resetMatch();
+        }
+    }
+
+//    int32_t lenDelta, keyLimit;
+    int32_t keyLimit;
+
+    // ------------------------ Ante Context ------------------------
+
+    // A mismatch in the ante context, or with the start anchor,
+    // is an outright U_MISMATCH regardless of whether we are
+    // incremental or not.
+    int32_t oText; // offset into 'text'
+//    int32_t newStart = 0;
+    int32_t minOText;
+
+    // Note (1): We process text in 16-bit code units, rather than
+    // 32-bit code points.  This works because stand-ins are
+    // always in the BMP and because we are doing a literal match
+    // operation, which can be done 16-bits at a time.
+    
+    int32_t anteLimit = posBefore(text, pos.contextStart);
+
+    UMatchDegree match;
+
+    // Start reverse match at char before pos.start
+    oText = posBefore(text, pos.start);
+
+    if (anteContext != nullptr) {
+        match = anteContext->matches(text, oText, anteLimit, false);
+        if (match != U_MATCH) {
+            return U_MISMATCH;
+        }
+    }
+
+    minOText = posAfter(text, oText);
+
+    // ------------------------ Start Anchor ------------------------
+    
+    if (((flags & ANCHOR_START) != 0) && oText != anteLimit) {
+        return U_MISMATCH;
+    }
+
+    // -------------------- Key and Post Context --------------------
+    
+    oText = pos.start;
+
+    if (key != nullptr) {
+        match = key->matches(text, oText, pos.limit, incremental);
+        if (match != U_MATCH) {
+            return match;
+        }
+    }
+
+    keyLimit = oText;
+
+    if (postContext != nullptr) {
+        if (incremental && keyLimit == pos.limit) {
+            // The key matches just before pos.limit, and there is
+            // a postContext.  Since we are in incremental mode,
+            // we must assume more characters may be inserted at
+            // pos.limit -- this is a partial match.
+            return U_PARTIAL_MATCH;
+        }
+
+        match = postContext->matches(text, oText, pos.contextLimit, incremental);
+        if (match != U_MATCH) {
+            return match;
+        }
+    }
+    
+    // ------------------------- Stop Anchor ------------------------
+    
+    if (((flags & ANCHOR_END)) != 0) {
+        if (oText != pos.contextLimit) {
+            return U_MISMATCH;
+        }
+        if (incremental) {
+            return U_PARTIAL_MATCH;
+        }
+    }
+    
+    // =========================== REPLACE ==========================
+
+    // We have a full match.  The key is between pos.start and
+    // keyLimit.
+
+    int32_t newStart;
+    int32_t newLength = output->toReplacer()->replace(text, pos.start, keyLimit, newStart);
+    int32_t lenDelta = newLength - (keyLimit - pos.start);
+
+    oText += lenDelta;
+    pos.limit += lenDelta;
+    pos.contextLimit += lenDelta;
+    // Restrict new value of start to [minOText, min(oText, pos.limit)].
+    pos.start = uprv_max(minOText, uprv_min(uprv_min(oText, pos.limit), newStart));
+    return U_MATCH;
+}
+
+/**
+ * Create a source string that represents this rule.  Append it to the
+ * given string.
+ */
+UnicodeString& TransliterationRule::toRule(UnicodeString& rule,
+                                           UBool escapeUnprintable) const {
+
+    // Accumulate special characters (and non-specials following them)
+    // into quoteBuf.  Append quoteBuf, within single quotes, when
+    // a non-quoted element must be inserted.
+    UnicodeString str, quoteBuf;
+
+    // Do not emit the braces '{' '}' around the pattern if there
+    // is neither anteContext nor postContext.
+    UBool emitBraces =
+        (anteContext != nullptr) || (postContext != nullptr);
+
+    // Emit start anchor
+    if ((flags & ANCHOR_START) != 0) {
+        rule.append((char16_t)94/*^*/);
+    }
+
+    // Emit the input pattern
+    ICU_Utility::appendToRule(rule, anteContext, escapeUnprintable, quoteBuf);
+
+    if (emitBraces) {
+        ICU_Utility::appendToRule(rule, (char16_t) 0x007B /*{*/, true, escapeUnprintable, quoteBuf);
+    }
+
+    ICU_Utility::appendToRule(rule, key, escapeUnprintable, quoteBuf);
+
+    if (emitBraces) {
+        ICU_Utility::appendToRule(rule, (char16_t) 0x007D /*}*/, true, escapeUnprintable, quoteBuf);
+    }
+
+    ICU_Utility::appendToRule(rule, postContext, escapeUnprintable, quoteBuf);
+
+    // Emit end anchor
+    if ((flags & ANCHOR_END) != 0) {
+        rule.append((char16_t)36/*$*/);
+    }
+
+    ICU_Utility::appendToRule(rule, UnicodeString(true, FORWARD_OP, 3), true, escapeUnprintable, quoteBuf);
+
+    // Emit the output pattern
+
+    ICU_Utility::appendToRule(rule, output->toReplacer()->toReplacerPattern(str, escapeUnprintable),
+                              true, escapeUnprintable, quoteBuf);
+
+    ICU_Utility::appendToRule(rule, (char16_t) 0x003B /*;*/, true, escapeUnprintable, quoteBuf);
+
+    return rule;
+}
+
+void TransliterationRule::setData(const TransliterationRuleData* d) {
+    data = d;
+    if (anteContext != nullptr) anteContext->setData(d);
+    if (postContext != nullptr) postContext->setData(d);
+    if (key != nullptr) key->setData(d);
+    // assert(output != nullptr);
+    output->setData(d);
+    // Don't have to do segments since they are in the context or key
+}
+
+/**
+ * Union the set of all characters that may be modified by this rule
+ * into the given set.
+ */
+void TransliterationRule::addSourceSetTo(UnicodeSet& toUnionTo) const {
+    int32_t limit = anteContextLength + keyLength;
+    for (int32_t i=anteContextLength; i<limit; ) {
+        UChar32 ch = pattern.char32At(i);
+        i += U16_LENGTH(ch);
+        const UnicodeMatcher* matcher = data->lookupMatcher(ch);
+        if (matcher == nullptr) {
+            toUnionTo.add(ch);
+        } else {
+            matcher->addMatchSetTo(toUnionTo);
+        }
+    }
+}
+
+/**
+ * Union the set of all characters that may be emitted by this rule
+ * into the given set.
+ */
+void TransliterationRule::addTargetSetTo(UnicodeSet& toUnionTo) const {
+    output->toReplacer()->addReplacementSetTo(toUnionTo);
+}
+
+U_NAMESPACE_END
+
+#endif /* #if !UCONFIG_NO_TRANSLITERATION */
+
+//eof