summaryrefslogtreecommitdiffstats
path: root/intl/icu/source/i18n/cpdtrans.cpp
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--intl/icu/source/i18n/cpdtrans.cpp617
1 files changed, 617 insertions, 0 deletions
diff --git a/intl/icu/source/i18n/cpdtrans.cpp b/intl/icu/source/i18n/cpdtrans.cpp
new file mode 100644
index 0000000000..9b10364689
--- /dev/null
+++ b/intl/icu/source/i18n/cpdtrans.cpp
@@ -0,0 +1,617 @@
+// © 2016 and later: Unicode, Inc. and others.
+// License & terms of use: http://www.unicode.org/copyright.html
+/*
+**********************************************************************
+* Copyright (C) 1999-2011, International Business Machines
+* Corporation and others. All Rights Reserved.
+**********************************************************************
+* Date Name Description
+* 11/17/99 aliu Creation.
+**********************************************************************
+*/
+
+#include "unicode/utypes.h"
+
+#if !UCONFIG_NO_TRANSLITERATION
+
+#include "unicode/unifilt.h"
+#include "unicode/uniset.h"
+#include "cpdtrans.h"
+#include "uvector.h"
+#include "tridpars.h"
+#include "cmemory.h"
+
+// keep in sync with Transliterator
+//static const char16_t ID_SEP = 0x002D; /*-*/
+static const char16_t ID_DELIM = 0x003B; /*;*/
+static const char16_t NEWLINE = 10;
+
+static const char16_t COLON_COLON[] = {0x3A, 0x3A, 0}; //"::"
+
+U_NAMESPACE_BEGIN
+
+const char16_t CompoundTransliterator::PASS_STRING[] = { 0x0025, 0x0050, 0x0061, 0x0073, 0x0073, 0 }; // "%Pass"
+
+UOBJECT_DEFINE_RTTI_IMPLEMENTATION(CompoundTransliterator)
+
+/**
+ * Constructs a new compound transliterator given an array of
+ * transliterators. The array of transliterators may be of any
+ * length, including zero or one, however, useful compound
+ * transliterators have at least two components.
+ * @param transliterators array of <code>Transliterator</code>
+ * objects
+ * @param transliteratorCount The number of
+ * <code>Transliterator</code> objects in transliterators.
+ * @param filter the filter. Any character for which
+ * <tt>filter.contains()</tt> returns <tt>false</tt> will not be
+ * altered by this transliterator. If <tt>filter</tt> is
+ * <tt>null</tt> then no filtering is applied.
+ */
+CompoundTransliterator::CompoundTransliterator(
+ Transliterator* const transliterators[],
+ int32_t transliteratorCount,
+ UnicodeFilter* adoptedFilter) :
+ Transliterator(joinIDs(transliterators, transliteratorCount), adoptedFilter),
+ trans(0), count(0), numAnonymousRBTs(0) {
+ setTransliterators(transliterators, transliteratorCount);
+}
+
+/**
+ * Splits an ID of the form "ID;ID;..." into a compound using each
+ * of the IDs.
+ * @param id of above form
+ * @param forward if false, does the list in reverse order, and
+ * takes the inverse of each ID.
+ */
+CompoundTransliterator::CompoundTransliterator(const UnicodeString& id,
+ UTransDirection direction,
+ UnicodeFilter* adoptedFilter,
+ UParseError& /*parseError*/,
+ UErrorCode& status) :
+ Transliterator(id, adoptedFilter),
+ trans(0), numAnonymousRBTs(0) {
+ // TODO add code for parseError...currently unused, but
+ // later may be used by parsing code...
+ init(id, direction, true, status);
+}
+
+CompoundTransliterator::CompoundTransliterator(const UnicodeString& id,
+ UParseError& /*parseError*/,
+ UErrorCode& status) :
+ Transliterator(id, 0), // set filter to 0 here!
+ trans(0), numAnonymousRBTs(0) {
+ // TODO add code for parseError...currently unused, but
+ // later may be used by parsing code...
+ init(id, UTRANS_FORWARD, true, status);
+}
+
+
+/**
+ * Private constructor for use of TransliteratorAlias
+ */
+CompoundTransliterator::CompoundTransliterator(const UnicodeString& newID,
+ UVector& list,
+ UnicodeFilter* adoptedFilter,
+ int32_t anonymousRBTs,
+ UParseError& /*parseError*/,
+ UErrorCode& status) :
+ Transliterator(newID, adoptedFilter),
+ trans(0), numAnonymousRBTs(anonymousRBTs)
+{
+ init(list, UTRANS_FORWARD, false, status);
+}
+
+/**
+ * Private constructor for Transliterator from a vector of
+ * transliterators. The caller is responsible for fixing up the
+ * ID.
+ */
+CompoundTransliterator::CompoundTransliterator(UVector& list,
+ UParseError& /*parseError*/,
+ UErrorCode& status) :
+ Transliterator(UnicodeString(), nullptr),
+ trans(0), numAnonymousRBTs(0)
+{
+ // TODO add code for parseError...currently unused, but
+ // later may be used by parsing code...
+ init(list, UTRANS_FORWARD, false, status);
+ // assume caller will fixup ID
+}
+
+CompoundTransliterator::CompoundTransliterator(UVector& list,
+ int32_t anonymousRBTs,
+ UParseError& /*parseError*/,
+ UErrorCode& status) :
+ Transliterator(UnicodeString(), nullptr),
+ trans(0), numAnonymousRBTs(anonymousRBTs)
+{
+ init(list, UTRANS_FORWARD, false, status);
+}
+
+/**
+ * Finish constructing a transliterator: only to be called by
+ * constructors. Before calling init(), set trans and filter to nullptr.
+ * @param id the id containing ';'-separated entries
+ * @param direction either FORWARD or REVERSE
+ * @param idSplitPoint the index into id at which the
+ * adoptedSplitTransliterator should be inserted, if there is one, or
+ * -1 if there is none.
+ * @param adoptedSplitTransliterator a transliterator to be inserted
+ * before the entry at offset idSplitPoint in the id string. May be
+ * nullptr to insert no entry.
+ * @param fixReverseID if true, then reconstruct the ID of reverse
+ * entries by calling getID() of component entries. Some constructors
+ * do not require this because they apply a facade ID anyway.
+ * @param status the error code indicating success or failure
+ */
+void CompoundTransliterator::init(const UnicodeString& id,
+ UTransDirection direction,
+ UBool fixReverseID,
+ UErrorCode& status) {
+ // assert(trans == 0);
+
+ if (U_FAILURE(status)) {
+ return;
+ }
+
+ UVector list(status);
+ UnicodeSet* compoundFilter = nullptr;
+ UnicodeString regenID;
+ if (!TransliteratorIDParser::parseCompoundID(id, direction,
+ regenID, list, compoundFilter)) {
+ status = U_INVALID_ID;
+ delete compoundFilter;
+ return;
+ }
+
+ TransliteratorIDParser::instantiateList(list, status);
+
+ init(list, direction, fixReverseID, status);
+
+ if (compoundFilter != nullptr) {
+ adoptFilter(compoundFilter);
+ }
+}
+
+/**
+ * Finish constructing a transliterator: only to be called by
+ * constructors. Before calling init(), set trans and filter to nullptr.
+ * @param list a vector of transliterator objects to be adopted. It
+ * should NOT be empty. The list should be in declared order. That
+ * is, it should be in the FORWARD order; if direction is REVERSE then
+ * the list order will be reversed.
+ * @param direction either FORWARD or REVERSE
+ * @param fixReverseID if true, then reconstruct the ID of reverse
+ * entries by calling getID() of component entries. Some constructors
+ * do not require this because they apply a facade ID anyway.
+ * @param status the error code indicating success or failure
+ */
+void CompoundTransliterator::init(UVector& list,
+ UTransDirection direction,
+ UBool fixReverseID,
+ UErrorCode& status) {
+ // assert(trans == 0);
+
+ // Allocate array
+ if (U_SUCCESS(status)) {
+ count = list.size();
+ trans = (Transliterator **)uprv_malloc(count * sizeof(Transliterator *));
+ /* test for nullptr */
+ if (trans == 0) {
+ status = U_MEMORY_ALLOCATION_ERROR;
+ return;
+ }
+ }
+
+ if (U_FAILURE(status) || trans == 0) {
+ // assert(trans == 0);
+ return;
+ }
+
+ // Move the transliterators from the vector into an array.
+ // Reverse the order if necessary.
+ int32_t i;
+ for (i=0; i<count; ++i) {
+ int32_t j = (direction == UTRANS_FORWARD) ? i : count - 1 - i;
+ trans[i] = (Transliterator*) list.elementAt(j);
+ }
+
+ // If the direction is UTRANS_REVERSE then we may need to fix the
+ // ID.
+ if (direction == UTRANS_REVERSE && fixReverseID) {
+ UnicodeString newID;
+ for (i=0; i<count; ++i) {
+ if (i > 0) {
+ newID.append(ID_DELIM);
+ }
+ newID.append(trans[i]->getID());
+ }
+ setID(newID);
+ }
+
+ computeMaximumContextLength();
+}
+
+/**
+ * Return the IDs of the given list of transliterators, concatenated
+ * with ID_DELIM delimiting them. Equivalent to the perlish expression
+ * join(ID_DELIM, map($_.getID(), transliterators).
+ */
+UnicodeString CompoundTransliterator::joinIDs(Transliterator* const transliterators[],
+ int32_t transCount) {
+ UnicodeString id;
+ for (int32_t i=0; i<transCount; ++i) {
+ if (i > 0) {
+ id.append(ID_DELIM);
+ }
+ id.append(transliterators[i]->getID());
+ }
+ return id; // Return temporary
+}
+
+/**
+ * Copy constructor.
+ */
+CompoundTransliterator::CompoundTransliterator(const CompoundTransliterator& t) :
+ Transliterator(t), trans(0), count(0), numAnonymousRBTs(-1) {
+ *this = t;
+}
+
+/**
+ * Destructor
+ */
+CompoundTransliterator::~CompoundTransliterator() {
+ freeTransliterators();
+}
+
+void CompoundTransliterator::freeTransliterators() {
+ if (trans != 0) {
+ for (int32_t i=0; i<count; ++i) {
+ delete trans[i];
+ }
+ uprv_free(trans);
+ }
+ trans = 0;
+ count = 0;
+}
+
+/**
+ * Assignment operator.
+ */
+CompoundTransliterator& CompoundTransliterator::operator=(
+ const CompoundTransliterator& t)
+{
+ if (this == &t) { return *this; } // self-assignment: no-op
+ Transliterator::operator=(t);
+ int32_t i = 0;
+ UBool failed = false;
+ if (trans != nullptr) {
+ for (i=0; i<count; ++i) {
+ delete trans[i];
+ trans[i] = 0;
+ }
+ }
+ if (t.count > count) {
+ if (trans != nullptr) {
+ uprv_free(trans);
+ }
+ trans = (Transliterator **)uprv_malloc(t.count * sizeof(Transliterator *));
+ }
+ count = t.count;
+ if (trans != nullptr) {
+ for (i=0; i<count; ++i) {
+ trans[i] = t.trans[i]->clone();
+ if (trans[i] == nullptr) {
+ failed = true;
+ break;
+ }
+ }
+ }
+
+ // if memory allocation failed delete backwards trans array
+ if (failed && i > 0) {
+ int32_t n;
+ for (n = i-1; n >= 0; n--) {
+ uprv_free(trans[n]);
+ trans[n] = nullptr;
+ }
+ }
+ numAnonymousRBTs = t.numAnonymousRBTs;
+ return *this;
+}
+
+/**
+ * Transliterator API.
+ */
+CompoundTransliterator* CompoundTransliterator::clone() const {
+ return new CompoundTransliterator(*this);
+}
+
+/**
+ * Returns the number of transliterators in this chain.
+ * @return number of transliterators in this chain.
+ */
+int32_t CompoundTransliterator::getCount() const {
+ return count;
+}
+
+/**
+ * Returns the transliterator at the given index in this chain.
+ * @param index index into chain, from 0 to <code>getCount() - 1</code>
+ * @return transliterator at the given index
+ */
+const Transliterator& CompoundTransliterator::getTransliterator(int32_t index) const {
+ return *trans[index];
+}
+
+void CompoundTransliterator::setTransliterators(Transliterator* const transliterators[],
+ int32_t transCount) {
+ Transliterator** a = (Transliterator **)uprv_malloc(transCount * sizeof(Transliterator *));
+ if (a == nullptr) {
+ return;
+ }
+ int32_t i = 0;
+ UBool failed = false;
+ for (i=0; i<transCount; ++i) {
+ a[i] = transliterators[i]->clone();
+ if (a[i] == nullptr) {
+ failed = true;
+ break;
+ }
+ }
+ if (failed && i > 0) {
+ int32_t n;
+ for (n = i-1; n >= 0; n--) {
+ uprv_free(a[n]);
+ a[n] = nullptr;
+ }
+ return;
+ }
+ adoptTransliterators(a, transCount);
+}
+
+void CompoundTransliterator::adoptTransliterators(Transliterator* adoptedTransliterators[],
+ int32_t transCount) {
+ // First free trans[] and set count to zero. Once this is done,
+ // orphan the filter. Set up the new trans[].
+ freeTransliterators();
+ trans = adoptedTransliterators;
+ count = transCount;
+ computeMaximumContextLength();
+ setID(joinIDs(trans, count));
+}
+
+/**
+ * Append c to buf, unless buf is empty or buf already ends in c.
+ */
+static void _smartAppend(UnicodeString& buf, char16_t c) {
+ if (buf.length() != 0 &&
+ buf.charAt(buf.length() - 1) != c) {
+ buf.append(c);
+ }
+}
+
+UnicodeString& CompoundTransliterator::toRules(UnicodeString& rulesSource,
+ UBool escapeUnprintable) const {
+ // We do NOT call toRules() on our component transliterators, in
+ // general. If we have several rule-based transliterators, this
+ // yields a concatenation of the rules -- not what we want. We do
+ // handle compound RBT transliterators specially -- those for which
+ // compoundRBTIndex >= 0. For the transliterator at compoundRBTIndex,
+ // we do call toRules() recursively.
+ rulesSource.truncate(0);
+ if (numAnonymousRBTs >= 1 && getFilter() != nullptr) {
+ // If we are a compound RBT and if we have a global
+ // filter, then emit it at the top.
+ UnicodeString pat;
+ rulesSource.append(COLON_COLON, 2).append(getFilter()->toPattern(pat, escapeUnprintable)).append(ID_DELIM);
+ }
+ for (int32_t i=0; i<count; ++i) {
+ UnicodeString rule;
+
+ // Anonymous RuleBasedTransliterators (inline rules and
+ // ::BEGIN/::END blocks) are given IDs that begin with
+ // "%Pass": use toRules() to write all the rules to the output
+ // (and insert "::Null;" if we have two in a row)
+ if (trans[i]->getID().startsWith(PASS_STRING, 5)) {
+ trans[i]->toRules(rule, escapeUnprintable);
+ if (numAnonymousRBTs > 1 && i > 0 && trans[i - 1]->getID().startsWith(PASS_STRING, 5))
+ rule = UNICODE_STRING_SIMPLE("::Null;") + rule;
+
+ // we also use toRules() on CompoundTransliterators (which we
+ // check for by looking for a semicolon in the ID)-- this gets
+ // the list of their child transliterators output in the right
+ // format
+ } else if (trans[i]->getID().indexOf(ID_DELIM) >= 0) {
+ trans[i]->toRules(rule, escapeUnprintable);
+
+ // for everything else, use Transliterator::toRules()
+ } else {
+ trans[i]->Transliterator::toRules(rule, escapeUnprintable);
+ }
+ _smartAppend(rulesSource, NEWLINE);
+ rulesSource.append(rule);
+ _smartAppend(rulesSource, ID_DELIM);
+ }
+ return rulesSource;
+}
+
+/**
+ * Implement Transliterator framework
+ */
+void CompoundTransliterator::handleGetSourceSet(UnicodeSet& result) const {
+ UnicodeSet set;
+ result.clear();
+ for (int32_t i=0; i<count; ++i) {
+ result.addAll(trans[i]->getSourceSet(set));
+ // Take the example of Hiragana-Latin. This is really
+ // Hiragana-Katakana; Katakana-Latin. The source set of
+ // these two is roughly [:Hiragana:] and [:Katakana:].
+ // But the source set for the entire transliterator is
+ // actually [:Hiragana:] ONLY -- that is, the first
+ // non-empty source set.
+
+ // This is a heuristic, and not 100% reliable.
+ if (!result.isEmpty()) {
+ break;
+ }
+ }
+}
+
+/**
+ * Override Transliterator framework
+ */
+UnicodeSet& CompoundTransliterator::getTargetSet(UnicodeSet& result) const {
+ UnicodeSet set;
+ result.clear();
+ for (int32_t i=0; i<count; ++i) {
+ // This is a heuristic, and not 100% reliable.
+ result.addAll(trans[i]->getTargetSet(set));
+ }
+ return result;
+}
+
+/**
+ * Implements {@link Transliterator#handleTransliterate}.
+ */
+void CompoundTransliterator::handleTransliterate(Replaceable& text, UTransPosition& index,
+ UBool incremental) const {
+ /* Call each transliterator with the same contextStart and
+ * start, but with the limit as modified
+ * by preceding transliterators. The start index must be
+ * reset for each transliterator to give each a chance to
+ * transliterate the text. The initial contextStart index is known
+ * to still point to the same place after each transliterator
+ * is called because each transliterator will not change the
+ * text between contextStart and the initial start index.
+ *
+ * IMPORTANT: After the first transliterator, each subsequent
+ * transliterator only gets to transliterate text committed by
+ * preceding transliterators; that is, the start (output
+ * value) of transliterator i becomes the limit (input value)
+ * of transliterator i+1. Finally, the overall limit is fixed
+ * up before we return.
+ *
+ * Assumptions we make here:
+ * (1) contextStart <= start <= limit <= contextLimit <= text.length()
+ * (2) start <= start' <= limit' ;cursor doesn't move back
+ * (3) start <= limit' ;text before cursor unchanged
+ * - start' is the value of start after calling handleKT
+ * - limit' is the value of limit after calling handleKT
+ */
+
+ /**
+ * Example: 3 transliterators. This example illustrates the
+ * mechanics we need to implement. C, S, and L are the contextStart,
+ * start, and limit. gl is the globalLimit. contextLimit is
+ * equal to limit throughout.
+ *
+ * 1. h-u, changes hex to Unicode
+ *
+ * 4 7 a d 0 4 7 a
+ * abc/u0061/u => abca/u
+ * C S L C S L gl=f->a
+ *
+ * 2. upup, changes "x" to "XX"
+ *
+ * 4 7 a 4 7 a
+ * abca/u => abcAA/u
+ * C SL C S
+ * L gl=a->b
+ * 3. u-h, changes Unicode to hex
+ *
+ * 4 7 a 4 7 a d 0 3
+ * abcAA/u => abc/u0041/u0041/u
+ * C S L C S
+ * L gl=b->15
+ * 4. return
+ *
+ * 4 7 a d 0 3
+ * abc/u0041/u0041/u
+ * C S L
+ */
+
+ if (count < 1) {
+ index.start = index.limit;
+ return; // Short circuit for empty compound transliterators
+ }
+
+ // compoundLimit is the limit value for the entire compound
+ // operation. We overwrite index.limit with the previous
+ // index.start. After each transliteration, we update
+ // compoundLimit for insertions or deletions that have happened.
+ int32_t compoundLimit = index.limit;
+
+ // compoundStart is the start for the entire compound
+ // operation.
+ int32_t compoundStart = index.start;
+
+ int32_t delta = 0; // delta in length
+
+ // Give each transliterator a crack at the run of characters.
+ // See comments at the top of the method for more detail.
+ for (int32_t i=0; i<count; ++i) {
+ index.start = compoundStart; // Reset start
+ int32_t limit = index.limit;
+
+ if (index.start == index.limit) {
+ // Short circuit for empty range
+ break;
+ }
+
+ trans[i]->filteredTransliterate(text, index, incremental);
+
+ // In a properly written transliterator, start == limit after
+ // handleTransliterate() returns when incremental is false.
+ // Catch cases where the subclass doesn't do this, and throw
+ // an exception. (Just pinning start to limit is a bad idea,
+ // because what's probably happening is that the subclass
+ // isn't transliterating all the way to the end, and it should
+ // in non-incremental mode.)
+ if (!incremental && index.start != index.limit) {
+ // We can't throw an exception, so just fudge things
+ index.start = index.limit;
+ }
+
+ // Cumulative delta for insertions/deletions
+ delta += index.limit - limit;
+
+ if (incremental) {
+ // In the incremental case, only allow subsequent
+ // transliterators to modify what has already been
+ // completely processed by prior transliterators. In the
+ // non-incrmental case, allow each transliterator to
+ // process the entire text.
+ index.limit = index.start;
+ }
+ }
+
+ compoundLimit += delta;
+
+ // Start is good where it is -- where the last transliterator left
+ // it. Limit needs to be put back where it was, modulo
+ // adjustments for deletions/insertions.
+ index.limit = compoundLimit;
+}
+
+/**
+ * Sets the length of the longest context required by this transliterator.
+ * This is <em>preceding</em> context.
+ */
+void CompoundTransliterator::computeMaximumContextLength() {
+ int32_t max = 0;
+ for (int32_t i=0; i<count; ++i) {
+ int32_t len = trans[i]->getMaximumContextLength();
+ if (len > max) {
+ max = len;
+ }
+ }
+ setMaximumContextLength(max);
+}
+
+U_NAMESPACE_END
+
+#endif /* #if !UCONFIG_NO_TRANSLITERATION */
+
+/* eof */