From 36d22d82aa202bb199967e9512281e9a53db42c9 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 7 Apr 2024 21:33:14 +0200 Subject: Adding upstream version 115.7.0esr. Signed-off-by: Daniel Baumann --- intl/icu/source/i18n/unicode/translit.h | 1593 +++++++++++++++++++++++++++++++ 1 file changed, 1593 insertions(+) create mode 100644 intl/icu/source/i18n/unicode/translit.h (limited to 'intl/icu/source/i18n/unicode/translit.h') diff --git a/intl/icu/source/i18n/unicode/translit.h b/intl/icu/source/i18n/unicode/translit.h new file mode 100644 index 0000000000..9ae32967fc --- /dev/null +++ b/intl/icu/source/i18n/unicode/translit.h @@ -0,0 +1,1593 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (C) 1999-2014, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* Date Name Description +* 11/17/99 aliu Creation. +********************************************************************** +*/ +#ifndef TRANSLIT_H +#define TRANSLIT_H + +#include "unicode/utypes.h" + +#if U_SHOW_CPLUSPLUS_API + +/** + * \file + * \brief C++ API: Transforms text from one format to another. + */ + +#if !UCONFIG_NO_TRANSLITERATION + +#include "unicode/uobject.h" +#include "unicode/unistr.h" +#include "unicode/parseerr.h" +#include "unicode/utrans.h" // UTransPosition, UTransDirection +#include "unicode/strenum.h" + +U_NAMESPACE_BEGIN + +class UnicodeFilter; +class UnicodeSet; +class TransliteratorParser; +class NormalizationTransliterator; +class TransliteratorIDParser; + +/** + * + * Transliterator is an abstract class that + * transliterates text from one format to another. The most common + * kind of transliterator is a script, or alphabet, transliterator. + * For example, a Russian to Latin transliterator changes Russian text + * written in Cyrillic characters to phonetically equivalent Latin + * characters. It does not translate Russian to English! + * Transliteration, unlike translation, operates on characters, without + * reference to the meanings of words and sentences. + * + *

Although script conversion is its most common use, a + * transliterator can actually perform a more general class of tasks. + * In fact, Transliterator defines a very general API + * which specifies only that a segment of the input text is replaced + * by new text. The particulars of this conversion are determined + * entirely by subclasses of Transliterator. + * + *

Transliterators are stateless + * + *

Transliterator objects are stateless; they + * retain no information between calls to + * transliterate(). (However, this does not + * mean that threads may share transliterators without synchronizing + * them. Transliterators are not immutable, so they must be + * synchronized when shared between threads.) This might seem to + * limit the complexity of the transliteration operation. In + * practice, subclasses perform complex transliterations by delaying + * the replacement of text until it is known that no other + * replacements are possible. In other words, although the + * Transliterator objects are stateless, the source text + * itself embodies all the needed information, and delayed operation + * allows arbitrary complexity. + * + *

Batch transliteration + * + *

The simplest way to perform transliteration is all at once, on a + * string of existing text. This is referred to as batch + * transliteration. For example, given a string input + * and a transliterator t, the call + * + * String result = t.transliterate(input); + * + * will transliterate it and return the result. Other methods allow + * the client to specify a substring to be transliterated and to use + * {@link Replaceable } objects instead of strings, in order to + * preserve out-of-band information (such as text styles). + * + *

Keyboard transliteration + * + *

Somewhat more involved is keyboard, or incremental + * transliteration. This is the transliteration of text that is + * arriving from some source (typically the user's keyboard) one + * character at a time, or in some other piecemeal fashion. + * + *

In keyboard transliteration, a Replaceable buffer + * stores the text. As text is inserted, as much as possible is + * transliterated on the fly. This means a GUI that displays the + * contents of the buffer may show text being modified as each new + * character arrives. + * + *

Consider the simple rule-based Transliterator: + * 

+ *     th>{theta}
+ *     t>{tau}
+ *
+ * + * When the user types 't', nothing will happen, since the + * transliterator is waiting to see if the next character is 'h'. To + * remedy this, we introduce the notion of a cursor, marked by a '|' + * in the output string: + * 
+ *     t>|{tau}
+ *     {tau}h>{theta}
+ *
+ * + * Now when the user types 't', tau appears, and if the next character + * is 'h', the tau changes to a theta. This is accomplished by + * maintaining a cursor position (independent of the insertion point, + * and invisible in the GUI) across calls to + * transliterate(). Typically, the cursor will + * be coincident with the insertion point, but in a case like the one + * above, it will precede the insertion point. + * + *

Keyboard transliteration methods maintain a set of three indices + * that are updated with each call to + * transliterate(), including the cursor, start, + * and limit. Since these indices are changed by the method, they are + * passed in an int[] array. The START index + * marks the beginning of the substring that the transliterator will + * look at. It is advanced as text becomes committed (but it is not + * the committed index; that's the CURSOR). The + * CURSOR index, described above, marks the point at + * which the transliterator last stopped, either because it reached + * the end, or because it required more characters to disambiguate + * between possible inputs. The CURSOR can also be + * explicitly set by rules in a rule-based Transliterator. + * Any characters before the CURSOR index are frozen; + * future keyboard transliteration calls within this input sequence + * will not change them. New text is inserted at the + * LIMIT index, which marks the end of the substring that + * the transliterator looks at. + * + *

Because keyboard transliteration assumes that more characters + * are to arrive, it is conservative in its operation. It only + * transliterates when it can do so unambiguously. Otherwise it waits + * for more characters to arrive. When the client code knows that no + * more characters are forthcoming, perhaps because the user has + * performed some input termination operation, then it should call + * finishTransliteration() to complete any + * pending transliterations. + * + *

Inverses + * + *

Pairs of transliterators may be inverses of one another. For + * example, if transliterator A transliterates characters by + * incrementing their Unicode value (so "abc" -> "def"), and + * transliterator B decrements character values, then A + * is an inverse of B and vice versa. If we compose A + * with B in a compound transliterator, the result is the + * identity transliterator, that is, a transliterator that does not + * change its input text. + * + * The Transliterator method getInverse() + * returns a transliterator's inverse, if one exists, or + * null otherwise. However, the result of + * getInverse() usually will not be a true + * mathematical inverse. This is because true inverse transliterators + * are difficult to formulate. For example, consider two + * transliterators: AB, which transliterates the character 'A' + * to 'B', and BA, which transliterates 'B' to 'A'. It might + * seem that these are exact inverses, since + * + * \htmlonly

\endhtmlonly"A" x AB -> "B"
+ * "B" x BA -> "A"\htmlonly
\endhtmlonly + * + * where 'x' represents transliteration. However, + * + * \htmlonly
\endhtmlonly"ABCD" x AB -> "BBCD"
+ * "BBCD" x BA -> "AACD"\htmlonly
\endhtmlonly + * + * so AB composed with BA is not the + * identity. Nonetheless, BA may be usefully considered to be + * AB's inverse, and it is on this basis that + * AB.getInverse() could legitimately return + * BA. + * + *

IDs and display names + * + *

A transliterator is designated by a short identifier string or + * ID. IDs follow the format source-destination, + * where source describes the entity being replaced, and + * destination describes the entity replacing + * source. The entities may be the names of scripts, + * particular sequences of characters, or whatever else it is that the + * transliterator converts to or from. For example, a transliterator + * from Russian to Latin might be named "Russian-Latin". A + * transliterator from keyboard escape sequences to Latin-1 characters + * might be named "KeyboardEscape-Latin1". By convention, system + * entity names are in English, with the initial letters of words + * capitalized; user entity names may follow any format so long as + * they do not contain dashes. + * + *

In addition to programmatic IDs, transliterator objects have + * display names for presentation in user interfaces, returned by + * {@link #getDisplayName }. + * + *

Factory methods and registration + * + *

In general, client code should use the factory method + * {@link #createInstance } to obtain an instance of a + * transliterator given its ID. Valid IDs may be enumerated using + * getAvailableIDs(). Since transliterators are mutable, + * multiple calls to {@link #createInstance } with the same ID will + * return distinct objects. + * + *

In addition to the system transliterators registered at startup, + * user transliterators may be registered by calling + * registerInstance() at run time. A registered instance + * acts a template; future calls to {@link #createInstance } with the ID + * of the registered object return clones of that object. Thus any + * object passed to registerInstance() must implement + * clone() properly. To register a transliterator subclass + * without instantiating it (until it is needed), users may call + * {@link #registerFactory }. In this case, the objects are + * instantiated by invoking the zero-argument public constructor of + * the class. + * + *

Subclassing + * + * Subclasses must implement the abstract method + * handleTransliterate().

Subclasses should override + * the transliterate() method taking a + * Replaceable and the transliterate() + * method taking a String and StringBuffer + * if the performance of these methods can be improved over the + * performance obtained by the default implementations in this class. + * + *

Rule syntax + * + *

A set of rules determines how to perform translations. + * Rules within a rule set are separated by semicolons (';'). + * To include a literal semicolon, prefix it with a backslash ('\'). + * Unicode Pattern_White_Space is ignored. + * If the first non-blank character on a line is '#', + * the entire line is ignored as a comment. + * + *

Each set of rules consists of two groups, one forward, and one + * reverse. This is a convention that is not enforced; rules for one + * direction may be omitted, with the result that translations in + * that direction will not modify the source text. In addition, + * bidirectional forward-reverse rules may be specified for + * symmetrical transformations. + * + *

Note: Another description of the Transliterator rule syntax is available in + * section + * Transform Rules Syntax of UTS #35: Unicode LDML. + * The rules are shown there using arrow symbols ← and → and ↔. + * ICU supports both those and the equivalent ASCII symbols < and > and <>. + * + *

Rule statements take one of the following forms: + * + *

+ *
$alefmadda=\\u0622;
+ *
Variable definition. The name on the + * left is assigned the text on the right. In this example, + * after this statement, instances of the left hand name, + * "$alefmadda", will be replaced by + * the Unicode character U+0622. Variable names must begin + * with a letter and consist only of letters, digits, and + * underscores. Case is significant. Duplicate names cause + * an exception to be thrown, that is, variables cannot be + * redefined. The right hand side may contain well-formed + * text of any length, including no text at all ("$empty=;"). + * The right hand side may contain embedded UnicodeSet + * patterns, for example, "$softvowel=[eiyEIY]".
+ *
ai>$alefmadda;
+ *
Forward translation rule. This rule + * states that the string on the left will be changed to the + * string on the right when performing forward + * transliteration.
+ *
ai<$alefmadda;
+ *
Reverse translation rule. This rule + * states that the string on the right will be changed to + * the string on the left when performing reverse + * transliteration.
+ *
+ * + *
+ *
ai<>$alefmadda;
+ *
Bidirectional translation rule. This + * rule states that the string on the right will be changed + * to the string on the left when performing forward + * transliteration, and vice versa when performing reverse + * transliteration.
+ *
+ * + *

Translation rules consist of a match pattern and an output + * string. The match pattern consists of literal characters, + * optionally preceded by context, and optionally followed by + * context. Context characters, like literal pattern characters, + * must be matched in the text being transliterated. However, unlike + * literal pattern characters, they are not replaced by the output + * text. For example, the pattern "abc{def}" + * indicates the characters "def" must be + * preceded by "abc" for a successful match. + * If there is a successful match, "def" will + * be replaced, but not "abc". The final '}' + * is optional, so "abc{def" is equivalent to + * "abc{def}". Another example is "{123}456" + * (or "123}456") in which the literal + * pattern "123" must be followed by "456". + * + *

The output string of a forward or reverse rule consists of + * characters to replace the literal pattern characters. If the + * output string contains the character '|', this is + * taken to indicate the location of the cursor after + * replacement. The cursor is the point in the text at which the + * next replacement, if any, will be applied. The cursor is usually + * placed within the replacement text; however, it can actually be + * placed into the preceding or following context by using the + * special character '@'. Examples: + * + * 

+ *     a {foo} z > | @ bar; # foo -> bar, move cursor before a
+ *     {foo} xyz > bar @@|; # foo -> bar, cursor between y and z
+ *
+ * + *

UnicodeSet + * + *

UnicodeSet patterns may appear anywhere that + * makes sense. They may appear in variable definitions. + * Contrariwise, UnicodeSet patterns may themselves + * contain variable references, such as "$a=[a-z];$not_a=[^$a]", + * or "$range=a-z;$ll=[$range]". + * + *

UnicodeSet patterns may also be embedded directly + * into rule strings. Thus, the following two rules are equivalent: + * + * 

+ *     $vowel=[aeiou]; $vowel>'*'; # One way to do this
+ *     [aeiou]>'*'; # Another way
+ *
+ * + *

See {@link UnicodeSet} for more documentation and examples. + * + *

Segments + * + *

Segments of the input string can be matched and copied to the + * output string. This makes certain sets of rules simpler and more + * general, and makes reordering possible. For example: + * + * 

+ *     ([a-z]) > $1 $1; # double lowercase letters
+ *     ([:Lu:]) ([:Ll:]) > $2 $1; # reverse order of Lu-Ll pairs
+ *
+ * + *

The segment of the input string to be copied is delimited by + * "(" and ")". Up to + * nine segments may be defined. Segments may not overlap. In the + * output string, "$1" through "$9" + * represent the input string segments, in left-to-right order of + * definition. + * + *

Anchors + * + *

Patterns can be anchored to the beginning or the end of the text. This is done with the + * special characters '^' and '$'. For example: + * + * 

+ *   ^ a   > 'BEG_A';   # match 'a' at start of text
+ *     a   > 'A'; # match other instances of 'a'
+ *     z $ > 'END_Z';   # match 'z' at end of text
+ *     z   > 'Z';       # match other instances of 'z'
+ *
+ * + *

It is also possible to match the beginning or the end of the text using a UnicodeSet. + * This is done by including a virtual anchor character '$' at the end of the + * set pattern. Although this is usually the match character for the end anchor, the set will + * match either the beginning or the end of the text, depending on its placement. For + * example: + * + * 

+ *   $x = [a-z$];   # match 'a' through 'z' OR anchor
+ *   $x 1    > 2;   # match '1' after a-z or at the start
+ *      3 $x > 4;   # match '3' before a-z or at the end
+ *
+ * + *

Example + * + *

The following example rules illustrate many of the features of + * the rule language. + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Rule 1.abc{def}>x|y
Rule 2.xyz>r
Rule 3.yz>q
+ * + *

Applying these rules to the string "adefabcdefz" + * yields the following results: + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
|adefabcdefzInitial state, no rules match. Advance + * cursor.
a|defabcdefzStill no match. Rule 1 does not match + * because the preceding context is not present.
ad|efabcdefzStill no match. Keep advancing until + * there is a match...
ade|fabcdefz...
adef|abcdefz...
adefa|bcdefz...
adefab|cdefz...
adefabc|defzRule 1 matches; replace "def" + * with "xy" and back up the cursor + * to before the 'y'.
adefabcx|yzAlthough "xyz" is + * present, rule 2 does not match because the cursor is + * before the 'y', not before the 'x'. + * Rule 3 does match. Replace "yz" + * with "q".
adefabcxq|The cursor is at the end; + * transliteration is complete.
+ * + *

The order of rules is significant. If multiple rules may match + * at some point, the first matching rule is applied. + * + *

Forward and reverse rules may have an empty output string. + * Otherwise, an empty left or right hand side of any statement is a + * syntax error. + * + *

Single quotes are used to quote any character other than a + * digit or letter. To specify a single quote itself, inside or + * outside of quotes, use two single quotes in a row. For example, + * the rule "'>'>o''clock" changes the + * string ">" to the string "o'clock". + * + *

Notes + * + *

While a Transliterator is being built from rules, it checks that + * the rules are added in proper order. For example, if the rule + * "a>x" is followed by the rule "ab>y", + * then the second rule will throw an exception. The reason is that + * the second rule can never be triggered, since the first rule + * always matches anything it matches. In other words, the first + * rule masks the second rule. + * + * @author Alan Liu + * @stable ICU 2.0 + */ +class U_I18N_API Transliterator : public UObject { + +private: + + /** + * Programmatic name, e.g., "Latin-Arabic". + */ + UnicodeString ID; + + /** + * This transliterator's filter. Any character for which + * filter.contains() returns false will not be + * altered by this transliterator. If filter is + * null then no filtering is applied. + */ + UnicodeFilter* filter; + + int32_t maximumContextLength; + + public: + + /** + * A context integer or pointer for a factory function, passed by + * value. + * @stable ICU 2.4 + */ + union Token { + /** + * This token, interpreted as a 32-bit integer. + * @stable ICU 2.4 + */ + int32_t integer; + /** + * This token, interpreted as a native pointer. + * @stable ICU 2.4 + */ + void* pointer; + }; + +#ifndef U_HIDE_INTERNAL_API + /** + * Return a token containing an integer. + * @return a token containing an integer. + * @internal + */ + inline static Token integerToken(int32_t); + + /** + * Return a token containing a pointer. + * @return a token containing a pointer. + * @internal + */ + inline static Token pointerToken(void*); +#endif /* U_HIDE_INTERNAL_API */ + + /** + * A function that creates and returns a Transliterator. When + * invoked, it will be passed the ID string that is being + * instantiated, together with the context pointer that was passed + * in when the factory function was first registered. Many + * factory functions will ignore both parameters, however, + * functions that are registered to more than one ID may use the + * ID or the context parameter to parameterize the transliterator + * they create. + * @param ID the string identifier for this transliterator + * @param context a context pointer that will be stored and + * later passed to the factory function when an ID matching + * the registration ID is being instantiated with this factory. + * @stable ICU 2.4 + */ + typedef Transliterator* (U_EXPORT2 *Factory)(const UnicodeString& ID, Token context); + +protected: + + /** + * Default constructor. + * @param ID the string identifier for this transliterator + * @param adoptedFilter the filter. Any character for which + * filter.contains() returns false will not be + * altered by this transliterator. If filter is + * null then no filtering is applied. + * @stable ICU 2.4 + */ + Transliterator(const UnicodeString& ID, UnicodeFilter* adoptedFilter); + + /** + * Copy constructor. + * @stable ICU 2.4 + */ + Transliterator(const Transliterator&); + + /** + * Assignment operator. + * @stable ICU 2.4 + */ + Transliterator& operator=(const Transliterator&); + + /** + * Create a transliterator from a basic ID. This is an ID + * containing only the forward direction source, target, and + * variant. + * @param id a basic ID of the form S-T or S-T/V. + * @param canon canonical ID to assign to the object, or + * nullptr to leave the ID unchanged + * @return a newly created Transliterator or null if the ID is + * invalid. + * @stable ICU 2.4 + */ + static Transliterator* createBasicInstance(const UnicodeString& id, + const UnicodeString* canon); + + friend class TransliteratorParser; // for parseID() + friend class TransliteratorIDParser; // for createBasicInstance() + friend class TransliteratorAlias; // for setID() + +public: + + /** + * Destructor. + * @stable ICU 2.0 + */ + virtual ~Transliterator(); + + /** + * Implements Cloneable. + * All subclasses are encouraged to implement this method if it is + * possible and reasonable to do so. Subclasses that are to be + * registered with the system using registerInstance() + * are required to implement this method. If a subclass does not + * implement clone() properly and is registered with the system + * using registerInstance(), then the default clone() implementation + * will return null, and calls to createInstance() will fail. + * + * @return a copy of the object. + * @see #registerInstance + * @stable ICU 2.0 + */ + virtual Transliterator* clone() const; + + /** + * Transliterates a segment of a string, with optional filtering. + * + * @param text the string to be transliterated + * @param start the beginning index, inclusive; 0 <= start + * <= limit. + * @param limit the ending index, exclusive; start <= limit + * <= text.length(). + * @return The new limit index. The text previously occupying [start, + * limit) has been transliterated, possibly to a string of a different + * length, at [start, new-limit), where + * new-limit is the return value. If the input offsets are out of bounds, + * the returned value is -1 and the input string remains unchanged. + * @stable ICU 2.0 + */ + virtual int32_t transliterate(Replaceable& text, + int32_t start, int32_t limit) const; + + /** + * Transliterates an entire string in place. Convenience method. + * @param text the string to be transliterated + * @stable ICU 2.0 + */ + virtual void transliterate(Replaceable& text) const; + + /** + * Transliterates the portion of the text buffer that can be + * transliterated unambiguosly after new text has been inserted, + * typically as a result of a keyboard event. The new text in + * insertion will be inserted into text + * at index.limit, advancing + * index.limit by insertion.length(). + * Then the transliterator will try to transliterate characters of + * text between index.cursor and + * index.limit. Characters before + * index.cursor will not be changed. + * + *

Upon return, values in index will be updated. + * index.start will be advanced to the first + * character that future calls to this method will read. + * index.cursor and index.limit will + * be adjusted to delimit the range of text that future calls to + * this method may change. + * + *

Typical usage of this method begins with an initial call + * with index.start and index.limit + * set to indicate the portion of text to be + * transliterated, and index.cursor == index.start. + * Thereafter, index can be used without + * modification in future calls, provided that all changes to + * text are made via this method. + * + *

This method assumes that future calls may be made that will + * insert new text into the buffer. As a result, it only performs + * unambiguous transliterations. After the last call to this + * method, there may be untransliterated text that is waiting for + * more input to resolve an ambiguity. In order to perform these + * pending transliterations, clients should call + * {@link #finishTransliteration } after the last call to this + * method has been made. + * + * @param text the buffer holding transliterated and untransliterated text + * @param index an array of three integers. + * + *

+ * + * @param insertion text to be inserted and possibly + * transliterated into the translation buffer at + * index.limit. If null then no text + * is inserted. + * @param status Output param to filled in with a success or an error. + * @see #handleTransliterate + * @exception IllegalArgumentException if index + * is invalid + * @see UTransPosition + * @stable ICU 2.0 + */ + virtual void transliterate(Replaceable& text, UTransPosition& index, + const UnicodeString& insertion, + UErrorCode& status) const; + + /** + * Transliterates the portion of the text buffer that can be + * transliterated unambiguosly after a new character has been + * inserted, typically as a result of a keyboard event. This is a + * convenience method. + * @param text the buffer holding transliterated and + * untransliterated text + * @param index an array of three integers. + * @param insertion text to be inserted and possibly + * transliterated into the translation buffer at + * index.limit. + * @param status Output param to filled in with a success or an error. + * @see #transliterate(Replaceable&, UTransPosition&, const UnicodeString&, UErrorCode&) const + * @stable ICU 2.0 + */ + virtual void transliterate(Replaceable& text, UTransPosition& index, + UChar32 insertion, + UErrorCode& status) const; + + /** + * Transliterates the portion of the text buffer that can be + * transliterated unambiguosly. This is a convenience method; see + * {@link #transliterate(Replaceable&, UTransPosition&, const UnicodeString&, UErrorCode&) const } + * for details. + * @param text the buffer holding transliterated and + * untransliterated text + * @param index an array of three integers. + * @param status Output param to filled in with a success or an error. + * @see #transliterate(Replaceable&, UTransPosition&, const UnicodeString&, UErrorCode &) const + * @stable ICU 2.0 + */ + virtual void transliterate(Replaceable& text, UTransPosition& index, + UErrorCode& status) const; + + /** + * Finishes any pending transliterations that were waiting for + * more characters. Clients should call this method as the last + * call after a sequence of one or more calls to + * transliterate(). + * @param text the buffer holding transliterated and + * untransliterated text. + * @param index the array of indices previously passed to {@link #transliterate } + * @stable ICU 2.0 + */ + virtual void finishTransliteration(Replaceable& text, + UTransPosition& index) const; + +private: + + /** + * This internal method does incremental transliteration. If the + * 'insertion' is non-null then we append it to 'text' before + * proceeding. This method calls through to the pure virtual + * framework method handleTransliterate() to do the actual + * work. + * @param text the buffer holding transliterated and + * untransliterated text + * @param index an array of three integers. See {@link + * #transliterate(Replaceable, int[], String)}. + * @param insertion text to be inserted and possibly + * transliterated into the translation buffer at + * index.limit. + * @param status Output param to filled in with a success or an error. + */ + void _transliterate(Replaceable& text, + UTransPosition& index, + const UnicodeString* insertion, + UErrorCode &status) const; + +protected: + + /** + * Abstract method that concrete subclasses define to implement + * their transliteration algorithm. This method handles both + * incremental and non-incremental transliteration. Let + * originalStart refer to the value of + * pos.start upon entry. + * + * + * + *

Implementations of this method should also obey the + * following invariants:

+ * + * + * + *

Subclasses may safely assume that all characters in + * [pos.start, pos.limit) are filtered. + * In other words, the filter has already been applied by the time + * this method is called. See + * filteredTransliterate(). + * + *

This method is not for public consumption. Calling + * this method directly will transliterate + * [pos.start, pos.limit) without + * applying the filter. End user code should call + * transliterate() instead of this method. Subclass code + * and wrapping transliterators should call + * filteredTransliterate() instead of this method.

+ * + * @param text the buffer holding transliterated and + * untransliterated text + * + * @param pos the indices indicating the start, limit, context + * start, and context limit of the text. + * + * @param incremental if true, assume more text may be inserted at + * pos.limit and act accordingly. Otherwise, + * transliterate all text between pos.start and + * pos.limit and move pos.start up to + * pos.limit. + * + * @see #transliterate + * @stable ICU 2.4 + */ + virtual void handleTransliterate(Replaceable& text, + UTransPosition& pos, + UBool incremental) const = 0; + +public: + /** + * Transliterate a substring of text, as specified by index, taking filters + * into account. This method is for subclasses that need to delegate to + * another transliterator. + * @param text the text to be transliterated + * @param index the position indices + * @param incremental if true, then assume more characters may be inserted + * at index.limit, and postpone processing to accommodate future incoming + * characters + * @stable ICU 2.4 + */ + virtual void filteredTransliterate(Replaceable& text, + UTransPosition& index, + UBool incremental) const; + +private: + + /** + * Top-level transliteration method, handling filtering, incremental and + * non-incremental transliteration, and rollback. All transliteration + * public API methods eventually call this method with a rollback argument + * of true. Other entities may call this method but rollback should be + * false. + * + *

If this transliterator has a filter, break up the input text into runs + * of unfiltered characters. Pass each run to + * subclass.handleTransliterate(). + * + *

In incremental mode, if rollback is true, perform a special + * incremental procedure in which several passes are made over the input + * text, adding one character at a time, and committing successful + * transliterations as they occur. Unsuccessful transliterations are rolled + * back and retried with additional characters to give correct results. + * + * @param text the text to be transliterated + * @param index the position indices + * @param incremental if true, then assume more characters may be inserted + * at index.limit, and postpone processing to accommodate future incoming + * characters + * @param rollback if true and if incremental is true, then perform special + * incremental processing, as described above, and undo partial + * transliterations where necessary. If incremental is false then this + * parameter is ignored. + */ + virtual void filteredTransliterate(Replaceable& text, + UTransPosition& index, + UBool incremental, + UBool rollback) const; + +public: + + /** + * Returns the length of the longest context required by this transliterator. + * This is preceding context. The default implementation supplied + * by Transliterator returns zero; subclasses + * that use preceding context should override this method to return the + * correct value. For example, if a transliterator translates "ddd" (where + * d is any digit) to "555" when preceded by "(ddd)", then the preceding + * context length is 5, the length of "(ddd)". + * + * @return The maximum number of preceding context characters this + * transliterator needs to examine + * @stable ICU 2.0 + */ + int32_t getMaximumContextLength(void) const; + +protected: + + /** + * Method for subclasses to use to set the maximum context length. + * @param maxContextLength the new value to be set. + * @see #getMaximumContextLength + * @stable ICU 2.4 + */ + void setMaximumContextLength(int32_t maxContextLength); + +public: + + /** + * Returns a programmatic identifier for this transliterator. + * If this identifier is passed to createInstance(), it + * will return this object, if it has been registered. + * @return a programmatic identifier for this transliterator. + * @see #registerInstance + * @see #registerFactory + * @see #getAvailableIDs + * @stable ICU 2.0 + */ + virtual const UnicodeString& getID(void) const; + + /** + * Returns a name for this transliterator that is appropriate for + * display to the user in the default locale. See {@link #getDisplayName } + * for details. + * @param ID the string identifier for this transliterator + * @param result Output param to receive the display name + * @return A reference to 'result'. + * @stable ICU 2.0 + */ + static UnicodeString& U_EXPORT2 getDisplayName(const UnicodeString& ID, + UnicodeString& result); + + /** + * Returns a name for this transliterator that is appropriate for + * display to the user in the given locale. This name is taken + * from the locale resource data in the standard manner of the + * java.text package. + * + *

If no localized names exist in the system resource bundles, + * a name is synthesized using a localized + * MessageFormat pattern from the resource data. The + * arguments to this pattern are an integer followed by one or two + * strings. The integer is the number of strings, either 1 or 2. + * The strings are formed by splitting the ID for this + * transliterator at the first '-'. If there is no '-', then the + * entire ID forms the only string. + * @param ID the string identifier for this transliterator + * @param inLocale the Locale in which the display name should be + * localized. + * @param result Output param to receive the display name + * @return A reference to 'result'. + * @stable ICU 2.0 + */ + static UnicodeString& U_EXPORT2 getDisplayName(const UnicodeString& ID, + const Locale& inLocale, + UnicodeString& result); + + /** + * Returns the filter used by this transliterator, or nullptr + * if this transliterator uses no filter. + * @return the filter used by this transliterator, or nullptr + * if this transliterator uses no filter. + * @stable ICU 2.0 + */ + const UnicodeFilter* getFilter(void) const; + + /** + * Returns the filter used by this transliterator, or nullptr if this + * transliterator uses no filter. The caller must eventually delete the + * result. After this call, this transliterator's filter is set to + * nullptr. + * @return the filter used by this transliterator, or nullptr if this + * transliterator uses no filter. + * @stable ICU 2.4 + */ + UnicodeFilter* orphanFilter(void); + + /** + * Changes the filter used by this transliterator. If the filter + * is set to null then no filtering will occur. + * + *

Callers must take care if a transliterator is in use by + * multiple threads. The filter should not be changed by one + * thread while another thread may be transliterating. + * @param adoptedFilter the new filter to be adopted. + * @stable ICU 2.0 + */ + void adoptFilter(UnicodeFilter* adoptedFilter); + + /** + * Returns this transliterator's inverse. See the class + * documentation for details. This implementation simply inverts + * the two entities in the ID and attempts to retrieve the + * resulting transliterator. That is, if getID() + * returns "A-B", then this method will return the result of + * createInstance("B-A"), or null if that + * call fails. + * + *

Subclasses with knowledge of their inverse may wish to + * override this method. + * + * @param status Output param to filled in with a success or an error. + * @return a transliterator that is an inverse, not necessarily + * exact, of this transliterator, or null if no such + * transliterator is registered. + * @see #registerInstance + * @stable ICU 2.0 + */ + Transliterator* createInverse(UErrorCode& status) const; + + /** + * Returns a Transliterator object given its ID. + * The ID must be either a system transliterator ID or a ID registered + * using registerInstance(). + * + * @param ID a valid ID, as enumerated by getAvailableIDs() + * @param dir either FORWARD or REVERSE. + * @param parseError Struct to receive information on position + * of error if an error is encountered + * @param status Output param to filled in with a success or an error. + * @return A Transliterator object with the given ID + * @see #registerInstance + * @see #getAvailableIDs + * @see #getID + * @stable ICU 2.0 + */ + static Transliterator* U_EXPORT2 createInstance(const UnicodeString& ID, + UTransDirection dir, + UParseError& parseError, + UErrorCode& status); + + /** + * Returns a Transliterator object given its ID. + * The ID must be either a system transliterator ID or a ID registered + * using registerInstance(). + * @param ID a valid ID, as enumerated by getAvailableIDs() + * @param dir either FORWARD or REVERSE. + * @param status Output param to filled in with a success or an error. + * @return A Transliterator object with the given ID + * @stable ICU 2.0 + */ + static Transliterator* U_EXPORT2 createInstance(const UnicodeString& ID, + UTransDirection dir, + UErrorCode& status); + + /** + * Returns a Transliterator object constructed from + * the given rule string. This will be a rule-based Transliterator, + * if the rule string contains only rules, or a + * compound Transliterator, if it contains ID blocks, or a + * null Transliterator, if it contains ID blocks which parse as + * empty for the given direction. + * + * @param ID the id for the transliterator. + * @param rules rules, separated by ';' + * @param dir either FORWARD or REVERSE. + * @param parseError Struct to receive information on position + * of error if an error is encountered + * @param status Output param set to success/failure code. + * @return a newly created Transliterator + * @stable ICU 2.0 + */ + static Transliterator* U_EXPORT2 createFromRules(const UnicodeString& ID, + const UnicodeString& rules, + UTransDirection dir, + UParseError& parseError, + UErrorCode& status); + + /** + * Create a rule string that can be passed to createFromRules() + * to recreate this transliterator. + * @param result the string to receive the rules. Previous + * contents will be deleted. + * @param escapeUnprintable if true then convert unprintable + * character to their hex escape representations, \\uxxxx or + * \\Uxxxxxxxx. Unprintable characters are those other than + * U+000A, U+0020..U+007E. + * @stable ICU 2.0 + */ + virtual UnicodeString& toRules(UnicodeString& result, + UBool escapeUnprintable) const; + + /** + * Return the number of elements that make up this transliterator. + * For example, if the transliterator "NFD;Jamo-Latin;Latin-Greek" + * were created, the return value of this method would be 3. + * + *

If this transliterator is not composed of other + * transliterators, then this method returns 1. + * @return the number of transliterators that compose this + * transliterator, or 1 if this transliterator is not composed of + * multiple transliterators + * @stable ICU 3.0 + */ + int32_t countElements() const; + + /** + * Return an element that makes up this transliterator. For + * example, if the transliterator "NFD;Jamo-Latin;Latin-Greek" + * were created, the return value of this method would be one + * of the three transliterator objects that make up that + * transliterator: [NFD, Jamo-Latin, Latin-Greek]. + * + *

If this transliterator is not composed of other + * transliterators, then this method will return a reference to + * this transliterator when given the index 0. + * @param index a value from 0..countElements()-1 indicating the + * transliterator to return + * @param ec input-output error code + * @return one of the transliterators that makes up this + * transliterator, if this transliterator is made up of multiple + * transliterators, otherwise a reference to this object if given + * an index of 0 + * @stable ICU 3.0 + */ + const Transliterator& getElement(int32_t index, UErrorCode& ec) const; + + /** + * Returns the set of all characters that may be modified in the + * input text by this Transliterator. This incorporates this + * object's current filter; if the filter is changed, the return + * value of this function will change. The default implementation + * returns an empty set. Some subclasses may override + * {@link #handleGetSourceSet } to return a more precise result. The + * return result is approximate in any case and is intended for + * use by tests, tools, or utilities. + * @param result receives result set; previous contents lost + * @return a reference to result + * @see #getTargetSet + * @see #handleGetSourceSet + * @stable ICU 2.4 + */ + UnicodeSet& getSourceSet(UnicodeSet& result) const; + + /** + * Framework method that returns the set of all characters that + * may be modified in the input text by this Transliterator, + * ignoring the effect of this object's filter. The base class + * implementation returns the empty set. Subclasses that wish to + * implement this should override this method. + * @return the set of characters that this transliterator may + * modify. The set may be modified, so subclasses should return a + * newly-created object. + * @param result receives result set; previous contents lost + * @see #getSourceSet + * @see #getTargetSet + * @stable ICU 2.4 + */ + virtual void handleGetSourceSet(UnicodeSet& result) const; + + /** + * Returns the set of all characters that may be generated as + * replacement text by this transliterator. The default + * implementation returns the empty set. Some subclasses may + * override this method to return a more precise result. The + * return result is approximate in any case and is intended for + * use by tests, tools, or utilities requiring such + * meta-information. + * @param result receives result set; previous contents lost + * @return a reference to result + * @see #getTargetSet + * @stable ICU 2.4 + */ + virtual UnicodeSet& getTargetSet(UnicodeSet& result) const; + +public: + + /** + * Registers a factory function that creates transliterators of + * a given ID. + * + * Because ICU may choose to cache Transliterators internally, this must + * be called at application startup, prior to any calls to + * Transliterator::createXXX to avoid undefined behavior. + * + * @param id the ID being registered + * @param factory a function pointer that will be copied and + * called later when the given ID is passed to createInstance() + * @param context a context pointer that will be stored and + * later passed to the factory function when an ID matching + * the registration ID is being instantiated with this factory. + * @stable ICU 2.0 + */ + static void U_EXPORT2 registerFactory(const UnicodeString& id, + Factory factory, + Token context); + + /** + * Registers an instance obj of a subclass of + * Transliterator with the system. When + * createInstance() is called with an ID string that is + * equal to obj->getID(), then obj->clone() is + * returned. + * + * After this call the Transliterator class owns the adoptedObj + * and will delete it. + * + * Because ICU may choose to cache Transliterators internally, this must + * be called at application startup, prior to any calls to + * Transliterator::createXXX to avoid undefined behavior. + * + * @param adoptedObj an instance of subclass of + * Transliterator that defines clone() + * @see #createInstance + * @see #registerFactory + * @see #unregister + * @stable ICU 2.0 + */ + static void U_EXPORT2 registerInstance(Transliterator* adoptedObj); + + /** + * Registers an ID string as an alias of another ID string. + * That is, after calling this function, createInstance(aliasID) + * will return the same thing as createInstance(realID). + * This is generally used to create shorter, more mnemonic aliases + * for long compound IDs. + * + * @param aliasID The new ID being registered. + * @param realID The ID that the new ID is to be an alias for. + * This can be a compound ID and can include filters and should + * refer to transliterators that have already been registered with + * the framework, although this isn't checked. + * @stable ICU 3.6 + */ + static void U_EXPORT2 registerAlias(const UnicodeString& aliasID, + const UnicodeString& realID); + +protected: + +#ifndef U_HIDE_INTERNAL_API + /** + * @param id the ID being registered + * @param factory a function pointer that will be copied and + * called later when the given ID is passed to createInstance() + * @param context a context pointer that will be stored and + * later passed to the factory function when an ID matching + * the registration ID is being instantiated with this factory. + * @internal + */ + static void _registerFactory(const UnicodeString& id, + Factory factory, + Token context); + + /** + * @internal + */ + static void _registerInstance(Transliterator* adoptedObj); + + /** + * @internal + */ + static void _registerAlias(const UnicodeString& aliasID, const UnicodeString& realID); + + /** + * Register two targets as being inverses of one another. For + * example, calling registerSpecialInverse("NFC", "NFD", true) causes + * Transliterator to form the following inverse relationships: + * + * 

NFC => NFD
+     * Any-NFC => Any-NFD
+     * NFD => NFC
+     * Any-NFD => Any-NFC
+ * + * (Without the special inverse registration, the inverse of NFC + * would be NFC-Any.) Note that NFD is shorthand for Any-NFD, but + * that the presence or absence of "Any-" is preserved. + * + *

The relationship is symmetrical; registering (a, b) is + * equivalent to registering (b, a). + * + *

The relevant IDs must still be registered separately as + * factories or classes. + * + *

Only the targets are specified. Special inverses always + * have the form Any-Target1 <=> Any-Target2. The target should + * have canonical casing (the casing desired to be produced when + * an inverse is formed) and should contain no whitespace or other + * extraneous characters. + * + * @param target the target against which to register the inverse + * @param inverseTarget the inverse of target, that is + * Any-target.getInverse() => Any-inverseTarget + * @param bidirectional if true, register the reverse relation + * as well, that is, Any-inverseTarget.getInverse() => Any-target + * @internal + */ + static void _registerSpecialInverse(const UnicodeString& target, + const UnicodeString& inverseTarget, + UBool bidirectional); +#endif /* U_HIDE_INTERNAL_API */ + +public: + + /** + * Unregisters a transliterator or class. This may be either + * a system transliterator or a user transliterator or class. + * Any attempt to construct an unregistered transliterator based + * on its ID will fail. + * + * Because ICU may choose to cache Transliterators internally, this should + * be called during application shutdown, after all calls to + * Transliterator::createXXX to avoid undefined behavior. + * + * @param ID the ID of the transliterator or class + * @return the Object that was registered with + * ID, or null if none was + * @see #registerInstance + * @see #registerFactory + * @stable ICU 2.0 + */ + static void U_EXPORT2 unregister(const UnicodeString& ID); + +public: + + /** + * Return a StringEnumeration over the IDs available at the time of the + * call, including user-registered IDs. + * @param ec input-output error code + * @return a newly-created StringEnumeration over the transliterators + * available at the time of the call. The caller should delete this object + * when done using it. + * @stable ICU 3.0 + */ + static StringEnumeration* U_EXPORT2 getAvailableIDs(UErrorCode& ec); + + /** + * Return the number of registered source specifiers. + * @return the number of registered source specifiers. + * @stable ICU 2.0 + */ + static int32_t U_EXPORT2 countAvailableSources(void); + + /** + * Return a registered source specifier. + * @param index which specifier to return, from 0 to n-1, where + * n = countAvailableSources() + * @param result fill-in parameter to receive the source specifier. + * If index is out of range, result will be empty. + * @return reference to result + * @stable ICU 2.0 + */ + static UnicodeString& U_EXPORT2 getAvailableSource(int32_t index, + UnicodeString& result); + + /** + * Return the number of registered target specifiers for a given + * source specifier. + * @param source the given source specifier. + * @return the number of registered target specifiers for a given + * source specifier. + * @stable ICU 2.0 + */ + static int32_t U_EXPORT2 countAvailableTargets(const UnicodeString& source); + + /** + * Return a registered target specifier for a given source. + * @param index which specifier to return, from 0 to n-1, where + * n = countAvailableTargets(source) + * @param source the source specifier + * @param result fill-in parameter to receive the target specifier. + * If source is invalid or if index is out of range, result will + * be empty. + * @return reference to result + * @stable ICU 2.0 + */ + static UnicodeString& U_EXPORT2 getAvailableTarget(int32_t index, + const UnicodeString& source, + UnicodeString& result); + + /** + * Return the number of registered variant specifiers for a given + * source-target pair. + * @param source the source specifiers. + * @param target the target specifiers. + * @stable ICU 2.0 + */ + static int32_t U_EXPORT2 countAvailableVariants(const UnicodeString& source, + const UnicodeString& target); + + /** + * Return a registered variant specifier for a given source-target + * pair. + * @param index which specifier to return, from 0 to n-1, where + * n = countAvailableVariants(source, target) + * @param source the source specifier + * @param target the target specifier + * @param result fill-in parameter to receive the variant + * specifier. If source is invalid or if target is invalid or if + * index is out of range, result will be empty. + * @return reference to result + * @stable ICU 2.0 + */ + static UnicodeString& U_EXPORT2 getAvailableVariant(int32_t index, + const UnicodeString& source, + const UnicodeString& target, + UnicodeString& result); + +protected: + +#ifndef U_HIDE_INTERNAL_API + /** + * Non-mutexed internal method + * @internal + */ + static int32_t _countAvailableSources(void); + + /** + * Non-mutexed internal method + * @internal + */ + static UnicodeString& _getAvailableSource(int32_t index, + UnicodeString& result); + + /** + * Non-mutexed internal method + * @internal + */ + static int32_t _countAvailableTargets(const UnicodeString& source); + + /** + * Non-mutexed internal method + * @internal + */ + static UnicodeString& _getAvailableTarget(int32_t index, + const UnicodeString& source, + UnicodeString& result); + + /** + * Non-mutexed internal method + * @internal + */ + static int32_t _countAvailableVariants(const UnicodeString& source, + const UnicodeString& target); + + /** + * Non-mutexed internal method + * @internal + */ + static UnicodeString& _getAvailableVariant(int32_t index, + const UnicodeString& source, + const UnicodeString& target, + UnicodeString& result); +#endif /* U_HIDE_INTERNAL_API */ + +protected: + + /** + * Set the ID of this transliterators. Subclasses shouldn't do + * this, unless the underlying script behavior has changed. + * @param id the new id t to be set. + * @stable ICU 2.4 + */ + void setID(const UnicodeString& id); + +public: + + /** + * Return the class ID for this class. This is useful only for + * comparing to a return value from getDynamicClassID(). + * Note that Transliterator is an abstract base class, and therefor + * no fully constructed object will have a dynamic + * UCLassID that equals the UClassID returned from + * TRansliterator::getStaticClassID(). + * @return The class ID for class Transliterator. + * @stable ICU 2.0 + */ + static UClassID U_EXPORT2 getStaticClassID(void); + + /** + * Returns a unique class ID polymorphically. This method + * is to implement a simple version of RTTI, since not all C++ + * compilers support genuine RTTI. Polymorphic operator==() and + * clone() methods call this method. + * + *

Concrete subclasses of Transliterator must use the + * UOBJECT_DEFINE_RTTI_IMPLEMENTATION macro from + * uobject.h to provide the RTTI functions. + * + * @return The class ID for this object. All objects of a given + * class have the same class ID. Objects of other classes have + * different class IDs. + * @stable ICU 2.0 + */ + virtual UClassID getDynamicClassID(void) const override = 0; + +private: + static UBool initializeRegistry(UErrorCode &status); + +public: +#ifndef U_HIDE_OBSOLETE_API + /** + * Return the number of IDs currently registered with the system. + * To retrieve the actual IDs, call getAvailableID(i) with + * i from 0 to countAvailableIDs() - 1. + * @return the number of IDs currently registered with the system. + * @obsolete ICU 3.4 use getAvailableIDs() instead + */ + static int32_t U_EXPORT2 countAvailableIDs(void); + + /** + * Return the index-th available ID. index must be between 0 + * and countAvailableIDs() - 1, inclusive. If index is out of + * range, the result of getAvailableID(0) is returned. + * @param index the given ID index. + * @return the index-th available ID. index must be between 0 + * and countAvailableIDs() - 1, inclusive. If index is out of + * range, the result of getAvailableID(0) is returned. + * @obsolete ICU 3.4 use getAvailableIDs() instead; this function + * is not thread safe, since it returns a reference to storage that + * may become invalid if another thread calls unregister + */ + static const UnicodeString& U_EXPORT2 getAvailableID(int32_t index); +#endif /* U_HIDE_OBSOLETE_API */ +}; + +inline int32_t Transliterator::getMaximumContextLength(void) const { + return maximumContextLength; +} + +inline void Transliterator::setID(const UnicodeString& id) { + ID = id; + // NUL-terminate the ID string, which is a non-aliased copy. + ID.append((char16_t)0); + ID.truncate(ID.length()-1); +} + +#ifndef U_HIDE_INTERNAL_API +inline Transliterator::Token Transliterator::integerToken(int32_t i) { + Token t; + t.integer = i; + return t; +} + +inline Transliterator::Token Transliterator::pointerToken(void* p) { + Token t; + t.pointer = p; + return t; +} +#endif /* U_HIDE_INTERNAL_API */ + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_TRANSLITERATION */ + +#endif /* U_SHOW_CPLUSPLUS_API */ + +#endif -- cgit v1.2.3