Adding upstream version 115.7.0esr.upstream/115.7.0esrupstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 365 insertions, 0 deletions

diff --git a/intl/icu/source/i18n/numparse_impl.cpp b/intl/icu/source/i18n/numparse_impl.cpp
new file mode 100644
index 0000000000..91c60747f2
--- /dev/null
+++ b/intl/icu/source/i18n/numparse_impl.cpp
@@ -0,0 +1,365 @@
+// © 2018 and later: Unicode, Inc. and others.
+// License & terms of use: http://www.unicode.org/copyright.html
+
+#include "unicode/utypes.h"
+
+#if !UCONFIG_NO_FORMATTING
+
+// Allow implicit conversion from char16_t* to UnicodeString for this file:
+// Helpful in toString methods and elsewhere.
+#define UNISTR_FROM_STRING_EXPLICIT
+
+#include <typeinfo>
+#include <array>
+#include "number_types.h"
+#include "number_patternstring.h"
+#include "numparse_types.h"
+#include "numparse_impl.h"
+#include "numparse_symbols.h"
+#include "numparse_decimal.h"
+#include "unicode/numberformatter.h"
+#include "cstr.h"
+#include "number_mapper.h"
+#include "static_unicode_sets.h"
+
+using namespace icu;
+using namespace icu::number;
+using namespace icu::number::impl;
+using namespace icu::numparse;
+using namespace icu::numparse::impl;
+
+
+NumberParseMatcher::~NumberParseMatcher() = default;
+
+
+NumberParserImpl*
+NumberParserImpl::createSimpleParser(const Locale& locale, const UnicodeString& patternString,
+                                     parse_flags_t parseFlags, UErrorCode& status) {
+
+    LocalPointer<NumberParserImpl> parser(new NumberParserImpl(parseFlags));
+    DecimalFormatSymbols symbols(locale, status);
+
+    parser->fLocalMatchers.ignorables = {parseFlags};
+    IgnorablesMatcher& ignorables = parser->fLocalMatchers.ignorables;
+
+    DecimalFormatSymbols dfs(locale, status);
+    dfs.setSymbol(DecimalFormatSymbols::kCurrencySymbol, u"IU$");
+    dfs.setSymbol(DecimalFormatSymbols::kIntlCurrencySymbol, u"ICU");
+    CurrencySymbols currencySymbols({u"ICU", status}, locale, dfs, status);
+
+    ParsedPatternInfo patternInfo;
+    PatternParser::parseToPatternInfo(patternString, patternInfo, status);
+
+    // The following statements set up the affix matchers.
+    AffixTokenMatcherSetupData affixSetupData = {
+            currencySymbols, symbols, ignorables, locale, parseFlags};
+    parser->fLocalMatchers.affixTokenMatcherWarehouse = {&affixSetupData};
+    parser->fLocalMatchers.affixMatcherWarehouse = {&parser->fLocalMatchers.affixTokenMatcherWarehouse};
+    parser->fLocalMatchers.affixMatcherWarehouse.createAffixMatchers(
+            patternInfo, *parser, ignorables, parseFlags, status);
+
+    Grouper grouper = Grouper::forStrategy(UNUM_GROUPING_AUTO);
+    grouper.setLocaleData(patternInfo, locale);
+
+    parser->addMatcher(parser->fLocalMatchers.ignorables);
+    parser->addMatcher(parser->fLocalMatchers.decimal = {symbols, grouper, parseFlags});
+    parser->addMatcher(parser->fLocalMatchers.minusSign = {symbols, false});
+    parser->addMatcher(parser->fLocalMatchers.plusSign = {symbols, false});
+    parser->addMatcher(parser->fLocalMatchers.percent = {symbols});
+    parser->addMatcher(parser->fLocalMatchers.permille = {symbols});
+    parser->addMatcher(parser->fLocalMatchers.nan = {symbols});
+    parser->addMatcher(parser->fLocalMatchers.infinity = {symbols});
+    parser->addMatcher(parser->fLocalMatchers.padding = {u"@"});
+    parser->addMatcher(parser->fLocalMatchers.scientific = {symbols, grouper});
+    parser->addMatcher(parser->fLocalMatchers.currency = {currencySymbols, symbols, parseFlags, status});
+    parser->addMatcher(parser->fLocalValidators.number = {});
+
+    parser->freeze();
+    return parser.orphan();
+}
+
+NumberParserImpl*
+NumberParserImpl::createParserFromProperties(const number::impl::DecimalFormatProperties& properties,
+                                             const DecimalFormatSymbols& symbols, bool parseCurrency,
+                                             UErrorCode& status) {
+    Locale locale = symbols.getLocale();
+    AutoAffixPatternProvider affixProvider(properties, status);
+    if (U_FAILURE(status)) { return nullptr; }
+    CurrencyUnit currency = resolveCurrency(properties, locale, status);
+    CurrencySymbols currencySymbols(currency, locale, symbols, status);
+    bool isStrict = properties.parseMode.getOrDefault(PARSE_MODE_STRICT) == PARSE_MODE_STRICT;
+    Grouper grouper = Grouper::forProperties(properties);
+    int parseFlags = 0;
+    if (U_FAILURE(status)) { return nullptr; }
+    if (!properties.parseCaseSensitive) {
+        parseFlags |= PARSE_FLAG_IGNORE_CASE;
+    }
+    if (properties.parseIntegerOnly) {
+        parseFlags |= PARSE_FLAG_INTEGER_ONLY;
+    }
+    if (properties.signAlwaysShown) {
+        parseFlags |= PARSE_FLAG_PLUS_SIGN_ALLOWED;
+    }
+    if (isStrict) {
+        parseFlags |= PARSE_FLAG_STRICT_GROUPING_SIZE;
+        parseFlags |= PARSE_FLAG_STRICT_SEPARATORS;
+        parseFlags |= PARSE_FLAG_USE_FULL_AFFIXES;
+        parseFlags |= PARSE_FLAG_EXACT_AFFIX;
+        parseFlags |= PARSE_FLAG_STRICT_IGNORABLES;
+    } else {
+        parseFlags |= PARSE_FLAG_INCLUDE_UNPAIRED_AFFIXES;
+    }
+    if (grouper.getPrimary() <= 0) {
+        parseFlags |= PARSE_FLAG_GROUPING_DISABLED;
+    }
+    if (parseCurrency || affixProvider.get().hasCurrencySign()) {
+        parseFlags |= PARSE_FLAG_MONETARY_SEPARATORS;
+    }
+    if (!parseCurrency) {
+        parseFlags |= PARSE_FLAG_NO_FOREIGN_CURRENCY;
+    }
+
+    LocalPointer<NumberParserImpl> parser(new NumberParserImpl(parseFlags));
+
+    parser->fLocalMatchers.ignorables = {parseFlags};
+    IgnorablesMatcher& ignorables = parser->fLocalMatchers.ignorables;
+
+    //////////////////////
+    /// AFFIX MATCHERS ///
+    //////////////////////
+
+    // The following statements set up the affix matchers.
+    AffixTokenMatcherSetupData affixSetupData = {
+            currencySymbols, symbols, ignorables, locale, parseFlags};
+    parser->fLocalMatchers.affixTokenMatcherWarehouse = {&affixSetupData};
+    parser->fLocalMatchers.affixMatcherWarehouse = {&parser->fLocalMatchers.affixTokenMatcherWarehouse};
+    parser->fLocalMatchers.affixMatcherWarehouse.createAffixMatchers(
+            affixProvider.get(), *parser, ignorables, parseFlags, status);
+
+    ////////////////////////
+    /// CURRENCY MATCHER ///
+    ////////////////////////
+
+    if (parseCurrency || affixProvider.get().hasCurrencySign()) {
+        parser->addMatcher(parser->fLocalMatchers.currency = {currencySymbols, symbols, parseFlags, status});
+    }
+
+    ///////////////
+    /// PERCENT ///
+    ///////////////
+
+    // ICU-TC meeting, April 11, 2018: accept percent/permille only if it is in the pattern,
+    // and to maintain regressive behavior, divide by 100 even if no percent sign is present.
+    if (!isStrict && affixProvider.get().containsSymbolType(AffixPatternType::TYPE_PERCENT, status)) {
+        parser->addMatcher(parser->fLocalMatchers.percent = {symbols});
+    }
+    if (!isStrict && affixProvider.get().containsSymbolType(AffixPatternType::TYPE_PERMILLE, status)) {
+        parser->addMatcher(parser->fLocalMatchers.permille = {symbols});
+    }
+
+    ///////////////////////////////
+    /// OTHER STANDARD MATCHERS ///
+    ///////////////////////////////
+
+    if (!isStrict) {
+        parser->addMatcher(parser->fLocalMatchers.plusSign = {symbols, false});
+        parser->addMatcher(parser->fLocalMatchers.minusSign = {symbols, false});
+    }
+    parser->addMatcher(parser->fLocalMatchers.nan = {symbols});
+    parser->addMatcher(parser->fLocalMatchers.infinity = {symbols});
+    UnicodeString padString = properties.padString;
+    if (!padString.isBogus() && !ignorables.getSet()->contains(padString)) {
+        parser->addMatcher(parser->fLocalMatchers.padding = {padString});
+    }
+    parser->addMatcher(parser->fLocalMatchers.ignorables);
+    parser->addMatcher(parser->fLocalMatchers.decimal = {symbols, grouper, parseFlags});
+    // NOTE: parseNoExponent doesn't disable scientific parsing if we have a scientific formatter
+    if (!properties.parseNoExponent || properties.minimumExponentDigits > 0) {
+        parser->addMatcher(parser->fLocalMatchers.scientific = {symbols, grouper});
+    }
+
+    //////////////////
+    /// VALIDATORS ///
+    //////////////////
+
+    parser->addMatcher(parser->fLocalValidators.number = {});
+    if (isStrict) {
+        parser->addMatcher(parser->fLocalValidators.affix = {});
+    }
+    if (parseCurrency) {
+        parser->addMatcher(parser->fLocalValidators.currency = {});
+    }
+    if (properties.decimalPatternMatchRequired) {
+        bool patternHasDecimalSeparator =
+                properties.decimalSeparatorAlwaysShown || properties.maximumFractionDigits != 0;
+        parser->addMatcher(parser->fLocalValidators.decimalSeparator = {patternHasDecimalSeparator});
+    }
+    // The multiplier takes care of scaling percentages.
+    Scale multiplier = scaleFromProperties(properties);
+    if (multiplier.isValid()) {
+        parser->addMatcher(parser->fLocalValidators.multiplier = {multiplier});
+    }
+
+    parser->freeze();
+    return parser.orphan();
+}
+
+NumberParserImpl::NumberParserImpl(parse_flags_t parseFlags)
+        : fParseFlags(parseFlags) {
+}
+
+NumberParserImpl::~NumberParserImpl() {
+    fNumMatchers = 0;
+}
+
+void NumberParserImpl::addMatcher(NumberParseMatcher& matcher) {
+    if (fNumMatchers + 1 > fMatchers.getCapacity()) {
+        fMatchers.resize(fNumMatchers * 2, fNumMatchers);
+    }
+    fMatchers[fNumMatchers] = &matcher;
+    fNumMatchers++;
+}
+
+void NumberParserImpl::freeze() {
+    fFrozen = true;
+}
+
+parse_flags_t NumberParserImpl::getParseFlags() const {
+    return fParseFlags;
+}
+
+void NumberParserImpl::parse(const UnicodeString& input, bool greedy, ParsedNumber& result,
+                             UErrorCode& status) const {
+    return parse(input, 0, greedy, result, status);
+}
+
+void NumberParserImpl::parse(const UnicodeString& input, int32_t start, bool greedy, ParsedNumber& result,
+                             UErrorCode& status) const {
+    if (U_FAILURE(status)) {
+        return;
+    }
+    U_ASSERT(fFrozen);
+    // TODO: Check start >= 0 and start < input.length()
+    StringSegment segment(input, 0 != (fParseFlags & PARSE_FLAG_IGNORE_CASE));
+    segment.adjustOffset(start);
+    if (greedy) {
+        parseGreedy(segment, result, status);
+    } else if (0 != (fParseFlags & PARSE_FLAG_ALLOW_INFINITE_RECURSION)) {
+        // Start at 1 so that recursionLevels never gets to 0
+        parseLongestRecursive(segment, result, 1, status);
+    } else {
+        // Arbitrary recursion safety limit: 100 levels.
+        parseLongestRecursive(segment, result, -100, status);
+    }
+    for (int32_t i = 0; i < fNumMatchers; i++) {
+        fMatchers[i]->postProcess(result);
+    }
+    result.postProcess();
+}
+
+void NumberParserImpl::parseGreedy(StringSegment& segment, ParsedNumber& result,
+                                            UErrorCode& status) const {
+    // Note: this method is not recursive in order to avoid stack overflow.
+    for (int i = 0; i <fNumMatchers;) {
+        // Base Case
+        if (segment.length() == 0) {
+            return;
+        }
+        const NumberParseMatcher* matcher = fMatchers[i];
+        if (!matcher->smokeTest(segment)) {
+            // Matcher failed smoke test: try the next one
+            i++;
+            continue;
+        }
+        int32_t initialOffset = segment.getOffset();
+        matcher->match(segment, result, status);
+        if (U_FAILURE(status)) {
+            return;
+        }
+        if (segment.getOffset() != initialOffset) {
+            // Greedy heuristic: accept the match and loop back
+            i = 0;
+            continue;
+        } else {
+            // Matcher did not match: try the next one
+            i++;
+            continue;
+        }
+        UPRV_UNREACHABLE_EXIT;
+    }
+
+    // NOTE: If we get here, the greedy parse completed without consuming the entire string.
+}
+
+void NumberParserImpl::parseLongestRecursive(StringSegment& segment, ParsedNumber& result,
+                                             int32_t recursionLevels,
+                                             UErrorCode& status) const {
+    // Base Case
+    if (segment.length() == 0) {
+        return;
+    }
+
+    // Safety against stack overflow
+    if (recursionLevels == 0) {
+        return;
+    }
+
+    // TODO: Give a nice way for the matcher to reset the ParsedNumber?
+    ParsedNumber initial(result);
+    ParsedNumber candidate;
+
+    int initialOffset = segment.getOffset();
+    for (int32_t i = 0; i < fNumMatchers; i++) {
+        const NumberParseMatcher* matcher = fMatchers[i];
+        if (!matcher->smokeTest(segment)) {
+            continue;
+        }
+
+        // In a non-greedy parse, we attempt all possible matches and pick the best.
+        for (int32_t charsToConsume = 0; charsToConsume < segment.length();) {
+            charsToConsume += U16_LENGTH(segment.codePointAt(charsToConsume));
+
+            // Run the matcher on a segment of the current length.
+            candidate = initial;
+            segment.setLength(charsToConsume);
+            bool maybeMore = matcher->match(segment, candidate, status);
+            segment.resetLength();
+            if (U_FAILURE(status)) {
+                return;
+            }
+
+            // If the entire segment was consumed, recurse.
+            if (segment.getOffset() - initialOffset == charsToConsume) {
+                parseLongestRecursive(segment, candidate, recursionLevels + 1, status);
+                if (U_FAILURE(status)) {
+                    return;
+                }
+                if (candidate.isBetterThan(result)) {
+                    result = candidate;
+                }
+            }
+
+            // Since the segment can be re-used, reset the offset.
+            // This does not have an effect if the matcher did not consume any chars.
+            segment.setOffset(initialOffset);
+
+            // Unless the matcher wants to see the next char, continue to the next matcher.
+            if (!maybeMore) {
+                break;
+            }
+        }
+    }
+}
+
+UnicodeString NumberParserImpl::toString() const {
+    UnicodeString result(u"<NumberParserImpl matchers:[");
+    for (int32_t i = 0; i < fNumMatchers; i++) {
+        result.append(u' ');
+        result.append(fMatchers[i]->toString());
+    }
+    result.append(u" ]>", -1);
+    return result;
+}
+
+
+#endif /* #if !UCONFIG_NO_FORMATTING */