Adding upstream version 115.7.0esr.upstream/115.7.0esrupstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 1143 insertions, 0 deletions

diff --git a/intl/icu/source/common/ucnv_ext.cpp b/intl/icu/source/common/ucnv_ext.cpp
new file mode 100644
index 0000000000..10d00a62c2
--- /dev/null
+++ b/intl/icu/source/common/ucnv_ext.cpp
@@ -0,0 +1,1143 @@
+// © 2016 and later: Unicode, Inc. and others.
+// License & terms of use: http://www.unicode.org/copyright.html
+/*
+******************************************************************************
+*
+*   Copyright (C) 2003-2016, International Business Machines
+*   Corporation and others.  All Rights Reserved.
+*
+******************************************************************************
+*   file name:  ucnv_ext.cpp
+*   encoding:   UTF-8
+*   tab size:   8 (not used)
+*   indentation:4
+*
+*   created on: 2003jun13
+*   created by: Markus W. Scherer
+*
+*   Conversion extensions
+*/
+
+#include "unicode/utypes.h"
+
+#if !UCONFIG_NO_CONVERSION && !UCONFIG_NO_LEGACY_CONVERSION
+
+#include "unicode/uset.h"
+#include "unicode/ustring.h"
+#include "ucnv_bld.h"
+#include "ucnv_cnv.h"
+#include "ucnv_ext.h"
+#include "cmemory.h"
+#include "uassert.h"
+
+/* to Unicode --------------------------------------------------------------- */
+
+/*
+ * @return lookup value for the byte, if found; else 0
+ */
+static inline uint32_t
+ucnv_extFindToU(const uint32_t *toUSection, int32_t length, uint8_t byte) {
+    uint32_t word0, word;
+    int32_t i, start, limit;
+
+    /* check the input byte against the lowest and highest section bytes */
+    start=(int32_t)UCNV_EXT_TO_U_GET_BYTE(toUSection[0]);
+    limit=(int32_t)UCNV_EXT_TO_U_GET_BYTE(toUSection[length-1]);
+    if(byte<start || limit<byte) {
+        return 0; /* the byte is out of range */
+    }
+
+    if(length==((limit-start)+1)) {
+        /* direct access on a linear array */
+        return UCNV_EXT_TO_U_GET_VALUE(toUSection[byte-start]); /* could be 0 */
+    }
+
+    /* word0 is suitable for <=toUSection[] comparison, word for <toUSection[] */
+    word0=UCNV_EXT_TO_U_MAKE_WORD(byte, 0);
+
+    /*
+     * Shift byte once instead of each section word and add 0xffffff.
+     * We will compare the shifted/added byte (bbffffff) against
+     * section words which have byte values in the same bit position.
+     * If and only if byte bb < section byte ss then bbffffff<ssvvvvvv
+     * for all v=0..f
+     * so we need not mask off the lower 24 bits of each section word.
+     */
+    word=word0|UCNV_EXT_TO_U_VALUE_MASK;
+
+    /* binary search */
+    start=0;
+    limit=length;
+    for(;;) {
+        i=limit-start;
+        if(i<=1) {
+            break; /* done */
+        }
+        /* start<limit-1 */
+
+        if(i<=4) {
+            /* linear search for the last part */
+            if(word0<=toUSection[start]) {
+                break;
+            }
+            if(++start<limit && word0<=toUSection[start]) {
+                break;
+            }
+            if(++start<limit && word0<=toUSection[start]) {
+                break;
+            }
+            /* always break at start==limit-1 */
+            ++start;
+            break;
+        }
+
+        i=(start+limit)/2;
+        if(word<toUSection[i]) {
+            limit=i;
+        } else {
+            start=i;
+        }
+    }
+
+    /* did we really find it? */
+    if(start<limit && byte==UCNV_EXT_TO_U_GET_BYTE(word=toUSection[start])) {
+        return UCNV_EXT_TO_U_GET_VALUE(word); /* never 0 */
+    } else {
+        return 0; /* not found */
+    }
+}
+
+/*
+ * true if not an SI/SO stateful converter,
+ * or if the match length fits with the current converter state
+ */
+#define UCNV_EXT_TO_U_VERIFY_SISO_MATCH(sisoState, match) \
+    ((sisoState)<0 || ((sisoState)==0) == (match==1))
+
+/*
+ * this works like ucnv_extMatchFromU() except
+ * - the first character is in pre
+ * - no trie is used
+ * - the returned matchLength is not offset by 2
+ */
+static int32_t
+ucnv_extMatchToU(const int32_t *cx, int8_t sisoState,
+                 const char *pre, int32_t preLength,
+                 const char *src, int32_t srcLength,
+                 uint32_t *pMatchValue,
+                 UBool /*useFallback*/, UBool flush) {
+    const uint32_t *toUTable, *toUSection;
+
+    uint32_t value, matchValue;
+    int32_t i, j, idx, length, matchLength;
+    uint8_t b;
+
+    if(cx==nullptr || cx[UCNV_EXT_TO_U_LENGTH]<=0) {
+        return 0; /* no extension data, no match */
+    }
+
+    /* initialize */
+    toUTable=UCNV_EXT_ARRAY(cx, UCNV_EXT_TO_U_INDEX, uint32_t);
+    idx=0;
+
+    matchValue=0;
+    i=j=matchLength=0;
+
+    if(sisoState==0) {
+        /* SBCS state of an SI/SO stateful converter, look at only exactly 1 byte */
+        if(preLength>1) {
+            return 0; /* no match of a DBCS sequence in SBCS mode */
+        } else if(preLength==1) {
+            srcLength=0;
+        } else /* preLength==0 */ {
+            if(srcLength>1) {
+                srcLength=1;
+            }
+        }
+        flush=true;
+    }
+
+    /* we must not remember fallback matches when not using fallbacks */
+
+    /* match input units until there is a full match or the input is consumed */
+    for(;;) {
+        /* go to the next section */
+        toUSection=toUTable+idx;
+
+        /* read first pair of the section */
+        value=*toUSection++;
+        length=UCNV_EXT_TO_U_GET_BYTE(value);
+        value=UCNV_EXT_TO_U_GET_VALUE(value);
+        if( value!=0 &&
+            (UCNV_EXT_TO_U_IS_ROUNDTRIP(value) ||
+             TO_U_USE_FALLBACK(useFallback)) &&
+            UCNV_EXT_TO_U_VERIFY_SISO_MATCH(sisoState, i+j)
+        ) {
+            /* remember longest match so far */
+            matchValue=value;
+            matchLength=i+j;
+        }
+
+        /* match pre[] then src[] */
+        if(i<preLength) {
+            b=(uint8_t)pre[i++];
+        } else if(j<srcLength) {
+            b=(uint8_t)src[j++];
+        } else {
+            /* all input consumed, partial match */
+            if(flush || (length=(i+j))>UCNV_EXT_MAX_BYTES) {
+                /*
+                 * end of the entire input stream, stop with the longest match so far
+                 * or: partial match must not be longer than UCNV_EXT_MAX_BYTES
+                 * because it must fit into state buffers
+                 */
+                break;
+            } else {
+                /* continue with more input next time */
+                return -length;
+            }
+        }
+
+        /* search for the current char16_t */
+        value=ucnv_extFindToU(toUSection, length, b);
+        if(value==0) {
+            /* no match here, stop with the longest match so far */
+            break;
+        } else {
+            if(UCNV_EXT_TO_U_IS_PARTIAL(value)) {
+                /* partial match, continue */
+                idx=(int32_t)UCNV_EXT_TO_U_GET_PARTIAL_INDEX(value);
+            } else {
+                if( (UCNV_EXT_TO_U_IS_ROUNDTRIP(value) ||
+                     TO_U_USE_FALLBACK(useFallback)) &&
+                    UCNV_EXT_TO_U_VERIFY_SISO_MATCH(sisoState, i+j)
+                ) {
+                    /* full match, stop with result */
+                    matchValue=value;
+                    matchLength=i+j;
+                } else {
+                    /* full match on fallback not taken, stop with the longest match so far */
+                }
+                break;
+            }
+        }
+    }
+
+    if(matchLength==0) {
+        /* no match at all */
+        return 0;
+    }
+
+    /* return result */
+    *pMatchValue=UCNV_EXT_TO_U_MASK_ROUNDTRIP(matchValue);
+    return matchLength;
+}
+
+static inline void
+ucnv_extWriteToU(UConverter *cnv, const int32_t *cx,
+                 uint32_t value,
+                 char16_t **target, const char16_t *targetLimit,
+                 int32_t **offsets, int32_t srcIndex,
+                 UErrorCode *pErrorCode) {
+    /* output the result */
+    if(UCNV_EXT_TO_U_IS_CODE_POINT(value)) {
+        /* output a single code point */
+        ucnv_toUWriteCodePoint(
+            cnv, UCNV_EXT_TO_U_GET_CODE_POINT(value),
+            target, targetLimit,
+            offsets, srcIndex,
+            pErrorCode);
+    } else {
+        /* output a string - with correct data we have resultLength>0 */
+        ucnv_toUWriteUChars(
+            cnv,
+            UCNV_EXT_ARRAY(cx, UCNV_EXT_TO_U_UCHARS_INDEX, char16_t)+
+                UCNV_EXT_TO_U_GET_INDEX(value),
+            UCNV_EXT_TO_U_GET_LENGTH(value),
+            target, targetLimit,
+            offsets, srcIndex,
+            pErrorCode);
+    }
+}
+
+/*
+ * get the SI/SO toU state (state 0 is for SBCS, 1 for DBCS),
+ * or 1 for DBCS-only,
+ * or -1 if the converter is not SI/SO stateful
+ *
+ * Note: For SI/SO stateful converters getting here,
+ * cnv->mode==0 is equivalent to firstLength==1.
+ */
+#define UCNV_SISO_STATE(cnv) \
+    ((cnv)->sharedData->mbcs.outputType==MBCS_OUTPUT_2_SISO ? (int8_t)(cnv)->mode : \
+     (cnv)->sharedData->mbcs.outputType==MBCS_OUTPUT_DBCS_ONLY ? 1 : -1)
+
+/*
+ * target<targetLimit; set error code for overflow
+ */
+U_CFUNC UBool
+ucnv_extInitialMatchToU(UConverter *cnv, const int32_t *cx,
+                        int32_t firstLength,
+                        const char **src, const char *srcLimit,
+                        char16_t **target, const char16_t *targetLimit,
+                        int32_t **offsets, int32_t srcIndex,
+                        UBool flush,
+                        UErrorCode *pErrorCode) {
+    uint32_t value = 0;  /* initialize output-only param to 0 to silence gcc */
+    int32_t match;
+
+    /* try to match */
+    match=ucnv_extMatchToU(cx, (int8_t)UCNV_SISO_STATE(cnv),
+                           (const char *)cnv->toUBytes, firstLength,
+                           *src, (int32_t)(srcLimit-*src),
+                           &value,
+                           cnv->useFallback, flush);
+    if(match>0) {
+        /* advance src pointer for the consumed input */
+        *src+=match-firstLength;
+
+        /* write result to target */
+        ucnv_extWriteToU(cnv, cx,
+                         value,
+                         target, targetLimit,
+                         offsets, srcIndex,
+                         pErrorCode);
+        return true;
+    } else if(match<0) {
+        /* save state for partial match */
+        const char *s;
+        int32_t j;
+
+        /* copy the first code point */
+        s=(const char *)cnv->toUBytes;
+        cnv->preToUFirstLength=(int8_t)firstLength;
+        for(j=0; j<firstLength; ++j) {
+            cnv->preToU[j]=*s++;
+        }
+
+        /* now copy the newly consumed input */
+        s=*src;
+        match=-match;
+        for(; j<match; ++j) {
+            cnv->preToU[j]=*s++;
+        }
+        *src=s; /* same as *src=srcLimit; because we reached the end of input */
+        cnv->preToULength=(int8_t)match;
+        return true;
+    } else /* match==0 no match */ {
+        return false;
+    }
+}
+
+U_CFUNC UChar32
+ucnv_extSimpleMatchToU(const int32_t *cx,
+                       const char *source, int32_t length,
+                       UBool useFallback) {
+    uint32_t value = 0;  /* initialize output-only param to 0 to silence gcc */
+    int32_t match;
+
+    if(length<=0) {
+        return 0xffff;
+    }
+
+    /* try to match */
+    match=ucnv_extMatchToU(cx, -1,
+                           source, length,
+                           nullptr, 0,
+                           &value,
+                           useFallback, true);
+    if(match==length) {
+        /* write result for simple, single-character conversion */
+        if(UCNV_EXT_TO_U_IS_CODE_POINT(value)) {
+            return UCNV_EXT_TO_U_GET_CODE_POINT(value);
+        }
+    }
+
+    /*
+     * return no match because
+     * - match>0 && value points to string: simple conversion cannot handle multiple code points
+     * - match>0 && match!=length: not all input consumed, forbidden for this function
+     * - match==0: no match found in the first place
+     * - match<0: partial match, not supported for simple conversion (and flush==true)
+     */
+    return 0xfffe;
+}
+
+/*
+ * continue partial match with new input
+ * never called for simple, single-character conversion
+ */
+U_CFUNC void
+ucnv_extContinueMatchToU(UConverter *cnv,
+                         UConverterToUnicodeArgs *pArgs, int32_t srcIndex,
+                         UErrorCode *pErrorCode) {
+    uint32_t value = 0;  /* initialize output-only param to 0 to silence gcc */
+    int32_t match, length;
+
+    match=ucnv_extMatchToU(cnv->sharedData->mbcs.extIndexes, (int8_t)UCNV_SISO_STATE(cnv),
+                           cnv->preToU, cnv->preToULength,
+                           pArgs->source, (int32_t)(pArgs->sourceLimit-pArgs->source),
+                           &value,
+                           cnv->useFallback, pArgs->flush);
+    if(match>0) {
+        if(match>=cnv->preToULength) {
+            /* advance src pointer for the consumed input */
+            pArgs->source+=match-cnv->preToULength;
+            cnv->preToULength=0;
+        } else {
+            /* the match did not use all of preToU[] - keep the rest for replay */
+            length=cnv->preToULength-match;
+            uprv_memmove(cnv->preToU, cnv->preToU+match, length);
+            cnv->preToULength=(int8_t)-length;
+        }
+
+        /* write result */
+        ucnv_extWriteToU(cnv, cnv->sharedData->mbcs.extIndexes,
+                         value,
+                         &pArgs->target, pArgs->targetLimit,
+                         &pArgs->offsets, srcIndex,
+                         pErrorCode);
+    } else if(match<0) {
+        /* save state for partial match */
+        const char *s;
+        int32_t j;
+
+        /* just _append_ the newly consumed input to preToU[] */
+        s=pArgs->source;
+        match=-match;
+        for(j=cnv->preToULength; j<match; ++j) {
+            cnv->preToU[j]=*s++;
+        }
+        pArgs->source=s; /* same as *src=srcLimit; because we reached the end of input */
+        cnv->preToULength=(int8_t)match;
+    } else /* match==0 */ {
+        /*
+         * no match
+         *
+         * We need to split the previous input into two parts:
+         *
+         * 1. The first codepage character is unmappable - that's how we got into
+         *    trying the extension data in the first place.
+         *    We need to move it from the preToU buffer
+         *    to the error buffer, set an error code,
+         *    and prepare the rest of the previous input for 2.
+         *
+         * 2. The rest of the previous input must be converted once we
+         *    come back from the callback for the first character.
+         *    At that time, we have to try again from scratch to convert
+         *    these input characters.
+         *    The replay will be handled by the ucnv.c conversion code.
+         */
+
+        /* move the first codepage character to the error field */
+        uprv_memcpy(cnv->toUBytes, cnv->preToU, cnv->preToUFirstLength);
+        cnv->toULength=cnv->preToUFirstLength;
+
+        /* move the rest up inside the buffer */
+        length=cnv->preToULength-cnv->preToUFirstLength;
+        if(length>0) {
+            uprv_memmove(cnv->preToU, cnv->preToU+cnv->preToUFirstLength, length);
+        }
+
+        /* mark preToU for replay */
+        cnv->preToULength=(int8_t)-length;
+
+        /* set the error code for unassigned */
+        *pErrorCode=U_INVALID_CHAR_FOUND;
+    }
+}
+
+/* from Unicode ------------------------------------------------------------- */
+
+// Use roundtrips, "good one-way" mappings, and some normal fallbacks.
+static inline UBool
+extFromUUseMapping(UBool useFallback, uint32_t value, UChar32 firstCP) {
+    return
+        ((value&UCNV_EXT_FROM_U_STATUS_MASK)!=0 ||
+            FROM_U_USE_FALLBACK(useFallback, firstCP)) &&
+        (value&UCNV_EXT_FROM_U_RESERVED_MASK)==0;
+}
+
+/*
+ * @return index of the char16_t, if found; else <0
+ */
+static inline int32_t
+ucnv_extFindFromU(const char16_t *fromUSection, int32_t length, char16_t u) {
+    int32_t i, start, limit;
+
+    /* binary search */
+    start=0;
+    limit=length;
+    for(;;) {
+        i=limit-start;
+        if(i<=1) {
+            break; /* done */
+        }
+        /* start<limit-1 */
+
+        if(i<=4) {
+            /* linear search for the last part */
+            if(u<=fromUSection[start]) {
+                break;
+            }
+            if(++start<limit && u<=fromUSection[start]) {
+                break;
+            }
+            if(++start<limit && u<=fromUSection[start]) {
+                break;
+            }
+            /* always break at start==limit-1 */
+            ++start;
+            break;
+        }
+
+        i=(start+limit)/2;
+        if(u<fromUSection[i]) {
+            limit=i;
+        } else {
+            start=i;
+        }
+    }
+
+    /* did we really find it? */
+    if(start<limit && u==fromUSection[start]) {
+        return start;
+    } else {
+        return -1; /* not found */
+    }
+}
+
+/*
+ * @param cx pointer to extension data; if nullptr, returns 0
+ * @param firstCP the first code point before all the other UChars
+ * @param pre UChars that must match; !initialMatch: partial match with them
+ * @param preLength length of pre, >=0
+ * @param src UChars that can be used to complete a match
+ * @param srcLength length of src, >=0
+ * @param pMatchValue [out] output result value for the match from the data structure
+ * @param useFallback "use fallback" flag, usually from cnv->useFallback
+ * @param flush true if the end of the input stream is reached
+ * @return >1: matched, return value=total match length (number of input units matched)
+ *          1: matched, no mapping but request for <subchar1>
+ *             (only for the first code point)
+ *          0: no match
+ *         <0: partial match, return value=negative total match length
+ *             (partial matches are never returned for flush==true)
+ *             (partial matches are never returned as being longer than UCNV_EXT_MAX_UCHARS)
+ *         the matchLength is 2 if only firstCP matched, and >2 if firstCP and
+ *         further code units matched
+ */
+static int32_t
+ucnv_extMatchFromU(const int32_t *cx,
+                   UChar32 firstCP,
+                   const char16_t *pre, int32_t preLength,
+                   const char16_t *src, int32_t srcLength,
+                   uint32_t *pMatchValue,
+                   UBool useFallback, UBool flush) {
+    const uint16_t *stage12, *stage3;
+    const uint32_t *stage3b;
+
+    const char16_t *fromUTableUChars, *fromUSectionUChars;
+    const uint32_t *fromUTableValues, *fromUSectionValues;
+
+    uint32_t value, matchValue;
+    int32_t i, j, idx, length, matchLength;
+    char16_t c;
+
+    if(cx==nullptr) {
+        return 0; /* no extension data, no match */
+    }
+
+    /* trie lookup of firstCP */
+    idx=firstCP>>10; /* stage 1 index */
+    if(idx>=cx[UCNV_EXT_FROM_U_STAGE_1_LENGTH]) {
+        return 0; /* the first code point is outside the trie */
+    }
+
+    stage12=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_STAGE_12_INDEX, uint16_t);
+    stage3=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_STAGE_3_INDEX, uint16_t);
+    idx=UCNV_EXT_FROM_U(stage12, stage3, idx, firstCP);
+
+    stage3b=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_STAGE_3B_INDEX, uint32_t);
+    value=stage3b[idx];
+    if(value==0) {
+        return 0;
+    }
+
+    /*
+     * Tests for (value&UCNV_EXT_FROM_U_RESERVED_MASK)==0:
+     * Do not interpret values with reserved bits used, for forward compatibility,
+     * and do not even remember intermediate results with reserved bits used.
+     */
+
+    if(UCNV_EXT_TO_U_IS_PARTIAL(value)) {
+        /* partial match, enter the loop below */
+        idx=(int32_t)UCNV_EXT_FROM_U_GET_PARTIAL_INDEX(value);
+
+        /* initialize */
+        fromUTableUChars=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_UCHARS_INDEX, char16_t);
+        fromUTableValues=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_VALUES_INDEX, uint32_t);
+
+        matchValue=0;
+        i=j=matchLength=0;
+
+        /* we must not remember fallback matches when not using fallbacks */
+
+        /* match input units until there is a full match or the input is consumed */
+        for(;;) {
+            /* go to the next section */
+            fromUSectionUChars=fromUTableUChars+idx;
+            fromUSectionValues=fromUTableValues+idx;
+
+            /* read first pair of the section */
+            length=*fromUSectionUChars++;
+            value=*fromUSectionValues++;
+            if(value!=0 && extFromUUseMapping(useFallback, value, firstCP)) {
+                /* remember longest match so far */
+                matchValue=value;
+                matchLength=2+i+j;
+            }
+
+            /* match pre[] then src[] */
+            if(i<preLength) {
+                c=pre[i++];
+            } else if(j<srcLength) {
+                c=src[j++];
+            } else {
+                /* all input consumed, partial match */
+                if(flush || (length=(i+j))>UCNV_EXT_MAX_UCHARS) {
+                    /*
+                     * end of the entire input stream, stop with the longest match so far
+                     * or: partial match must not be longer than UCNV_EXT_MAX_UCHARS
+                     * because it must fit into state buffers
+                     */
+                    break;
+                } else {
+                    /* continue with more input next time */
+                    return -(2+length);
+                }
+            }
+
+            /* search for the current char16_t */
+            idx=ucnv_extFindFromU(fromUSectionUChars, length, c);
+            if(idx<0) {
+                /* no match here, stop with the longest match so far */
+                break;
+            } else {
+                value=fromUSectionValues[idx];
+                if(UCNV_EXT_FROM_U_IS_PARTIAL(value)) {
+                    /* partial match, continue */
+                    idx=(int32_t)UCNV_EXT_FROM_U_GET_PARTIAL_INDEX(value);
+                } else {
+                    if(extFromUUseMapping(useFallback, value, firstCP)) {
+                        /* full match, stop with result */
+                        matchValue=value;
+                        matchLength=2+i+j;
+                    } else {
+                        /* full match on fallback not taken, stop with the longest match so far */
+                    }
+                    break;
+                }
+            }
+        }
+
+        if(matchLength==0) {
+            /* no match at all */
+            return 0;
+        }
+    } else /* result from firstCP trie lookup */ {
+        if(extFromUUseMapping(useFallback, value, firstCP)) {
+            /* full match, stop with result */
+            matchValue=value;
+            matchLength=2;
+        } else {
+            /* fallback not taken */
+            return 0;
+        }
+    }
+
+    /* return result */
+    if(matchValue==UCNV_EXT_FROM_U_SUBCHAR1) {
+        return 1; /* assert matchLength==2 */
+    }
+
+    *pMatchValue=matchValue;
+    return matchLength;
+}
+
+/*
+ * @param value fromUnicode mapping table value; ignores roundtrip and reserved bits
+ */
+static inline void
+ucnv_extWriteFromU(UConverter *cnv, const int32_t *cx,
+                   uint32_t value,
+                   char **target, const char *targetLimit,
+                   int32_t **offsets, int32_t srcIndex,
+                   UErrorCode *pErrorCode) {
+    uint8_t buffer[1+UCNV_EXT_MAX_BYTES];
+    const uint8_t *result;
+    int32_t length, prevLength;
+
+    length=UCNV_EXT_FROM_U_GET_LENGTH(value);
+    value=(uint32_t)UCNV_EXT_FROM_U_GET_DATA(value);
+
+    /* output the result */
+    if(length<=UCNV_EXT_FROM_U_MAX_DIRECT_LENGTH) {
+        /*
+         * Generate a byte array and then write it below.
+         * This is not the fastest possible way, but it should be ok for
+         * extension mappings, and it is much simpler.
+         * Offset and overflow handling are only done once this way.
+         */
+        uint8_t *p=buffer+1; /* reserve buffer[0] for shiftByte below */
+        switch(length) {
+        case 3:
+            *p++=(uint8_t)(value>>16);
+            U_FALLTHROUGH;
+        case 2:
+            *p++=(uint8_t)(value>>8);
+            U_FALLTHROUGH;
+        case 1:
+            *p++=(uint8_t)value;
+            U_FALLTHROUGH;
+        default:
+            break; /* will never occur */
+        }
+        result=buffer+1;
+    } else {
+        result=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_BYTES_INDEX, uint8_t)+value;
+    }
+
+    /* with correct data we have length>0 */
+
+    if((prevLength=cnv->fromUnicodeStatus)!=0) {
+        /* handle SI/SO stateful output */
+        uint8_t shiftByte;
+
+        if(prevLength>1 && length==1) {
+            /* change from double-byte mode to single-byte */
+            shiftByte=(uint8_t)UCNV_SI;
+            cnv->fromUnicodeStatus=1;
+        } else if(prevLength==1 && length>1) {
+            /* change from single-byte mode to double-byte */
+            shiftByte=(uint8_t)UCNV_SO;
+            cnv->fromUnicodeStatus=2;
+        } else {
+            shiftByte=0;
+        }
+
+        if(shiftByte!=0) {
+            /* prepend the shift byte to the result bytes */
+            buffer[0]=shiftByte;
+            if(result!=buffer+1) {
+                uprv_memcpy(buffer+1, result, length);
+            }
+            result=buffer;
+            ++length;
+        }
+    }
+
+    ucnv_fromUWriteBytes(cnv, (const char *)result, length,
+                         target, targetLimit,
+                         offsets, srcIndex,
+                         pErrorCode);
+}
+
+/*
+ * target<targetLimit; set error code for overflow
+ */
+U_CFUNC UBool
+ucnv_extInitialMatchFromU(UConverter *cnv, const int32_t *cx,
+                          UChar32 cp,
+                          const char16_t **src, const char16_t *srcLimit,
+                          char **target, const char *targetLimit,
+                          int32_t **offsets, int32_t srcIndex,
+                          UBool flush,
+                          UErrorCode *pErrorCode) {
+    uint32_t value = 0;  /* initialize output-only param to 0 to silence gcc */
+    int32_t match;
+
+    /* try to match */
+    match=ucnv_extMatchFromU(cx, cp,
+                             nullptr, 0,
+                             *src, (int32_t)(srcLimit-*src),
+                             &value,
+                             cnv->useFallback, flush);
+
+    /* reject a match if the result is a single byte for DBCS-only */
+    if( match>=2 &&
+        !(UCNV_EXT_FROM_U_GET_LENGTH(value)==1 &&
+          cnv->sharedData->mbcs.outputType==MBCS_OUTPUT_DBCS_ONLY)
+    ) {
+        /* advance src pointer for the consumed input */
+        *src+=match-2; /* remove 2 for the initial code point */
+
+        /* write result to target */
+        ucnv_extWriteFromU(cnv, cx,
+                           value,
+                           target, targetLimit,
+                           offsets, srcIndex,
+                           pErrorCode);
+        return true;
+    } else if(match<0) {
+        /* save state for partial match */
+        const char16_t *s;
+        int32_t j;
+
+        /* copy the first code point */
+        cnv->preFromUFirstCP=cp;
+
+        /* now copy the newly consumed input */
+        s=*src;
+        match=-match-2; /* remove 2 for the initial code point */
+        for(j=0; j<match; ++j) {
+            cnv->preFromU[j]=*s++;
+        }
+        *src=s; /* same as *src=srcLimit; because we reached the end of input */
+        cnv->preFromULength=(int8_t)match;
+        return true;
+    } else if(match==1) {
+        /* matched, no mapping but request for <subchar1> */
+        cnv->useSubChar1=true;
+        return false;
+    } else /* match==0 no match */ {
+        return false;
+    }
+}
+
+/*
+ * Used by ISO 2022 implementation.
+ * @return number of bytes in *pValue; negative number if fallback; 0 for no mapping
+ */
+U_CFUNC int32_t
+ucnv_extSimpleMatchFromU(const int32_t *cx,
+                         UChar32 cp, uint32_t *pValue,
+                         UBool useFallback) {
+    uint32_t value;
+    int32_t match;
+
+    /* try to match */
+    match=ucnv_extMatchFromU(cx,
+                             cp,
+                             nullptr, 0,
+                             nullptr, 0,
+                             &value,
+                             useFallback, true);
+    if(match>=2) {
+        /* write result for simple, single-character conversion */
+        int32_t length;
+        int isRoundtrip;
+
+        isRoundtrip=UCNV_EXT_FROM_U_IS_ROUNDTRIP(value);
+        length=UCNV_EXT_FROM_U_GET_LENGTH(value);
+        value=(uint32_t)UCNV_EXT_FROM_U_GET_DATA(value);
+
+        if(length<=UCNV_EXT_FROM_U_MAX_DIRECT_LENGTH) {
+            *pValue=value;
+            return isRoundtrip ? length : -length;
+#if 0 /* not currently used */
+        } else if(length==4) {
+            /* de-serialize a 4-byte result */
+            const uint8_t *result=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_BYTES_INDEX, uint8_t)+value;
+            *pValue=
+                ((uint32_t)result[0]<<24)|
+                ((uint32_t)result[1]<<16)|
+                ((uint32_t)result[2]<<8)|
+                result[3];
+            return isRoundtrip ? 4 : -4;
+#endif
+        }
+    }
+
+    /*
+     * return no match because
+     * - match>1 && resultLength>4: result too long for simple conversion
+     * - match==1: no match found, <subchar1> preferred
+     * - match==0: no match found in the first place
+     * - match<0: partial match, not supported for simple conversion (and flush==true)
+     */
+    return 0;
+}
+
+/*
+ * continue partial match with new input, requires cnv->preFromUFirstCP>=0
+ * never called for simple, single-character conversion
+ */
+U_CFUNC void
+ucnv_extContinueMatchFromU(UConverter *cnv,
+                           UConverterFromUnicodeArgs *pArgs, int32_t srcIndex,
+                           UErrorCode *pErrorCode) {
+    uint32_t value = 0;  /* initialize output-only param to 0 to silence gcc */
+    int32_t match;
+
+    match=ucnv_extMatchFromU(cnv->sharedData->mbcs.extIndexes,
+                             cnv->preFromUFirstCP,
+                             cnv->preFromU, cnv->preFromULength,
+                             pArgs->source, (int32_t)(pArgs->sourceLimit-pArgs->source),
+                             &value,
+                             cnv->useFallback, pArgs->flush);
+    if(match>=2) {
+        match-=2; /* remove 2 for the initial code point */
+
+        if(match>=cnv->preFromULength) {
+            /* advance src pointer for the consumed input */
+            pArgs->source+=match-cnv->preFromULength;
+            cnv->preFromULength=0;
+        } else {
+            /* the match did not use all of preFromU[] - keep the rest for replay */
+            int32_t length=cnv->preFromULength-match;
+            u_memmove(cnv->preFromU, cnv->preFromU+match, length);
+            cnv->preFromULength=(int8_t)-length;
+        }
+
+        /* finish the partial match */
+        cnv->preFromUFirstCP=U_SENTINEL;
+
+        /* write result */
+        ucnv_extWriteFromU(cnv, cnv->sharedData->mbcs.extIndexes,
+                           value,
+                           &pArgs->target, pArgs->targetLimit,
+                           &pArgs->offsets, srcIndex,
+                           pErrorCode);
+    } else if(match<0) {
+        /* save state for partial match */
+        const char16_t *s;
+        int32_t j;
+
+        /* just _append_ the newly consumed input to preFromU[] */
+        s=pArgs->source;
+        match=-match-2; /* remove 2 for the initial code point */
+        for(j=cnv->preFromULength; j<match; ++j) {
+            U_ASSERT(j>=0);
+            cnv->preFromU[j]=*s++;
+        }
+        pArgs->source=s; /* same as *src=srcLimit; because we reached the end of input */
+        cnv->preFromULength=(int8_t)match;
+    } else /* match==0 or 1 */ {
+        /*
+         * no match
+         *
+         * We need to split the previous input into two parts:
+         *
+         * 1. The first code point is unmappable - that's how we got into
+         *    trying the extension data in the first place.
+         *    We need to move it from the preFromU buffer
+         *    to the error buffer, set an error code,
+         *    and prepare the rest of the previous input for 2.
+         *
+         * 2. The rest of the previous input must be converted once we
+         *    come back from the callback for the first code point.
+         *    At that time, we have to try again from scratch to convert
+         *    these input characters.
+         *    The replay will be handled by the ucnv.c conversion code.
+         */
+
+        if(match==1) {
+            /* matched, no mapping but request for <subchar1> */
+            cnv->useSubChar1=true;
+        }
+
+        /* move the first code point to the error field */
+        cnv->fromUChar32=cnv->preFromUFirstCP;
+        cnv->preFromUFirstCP=U_SENTINEL;
+
+        /* mark preFromU for replay */
+        cnv->preFromULength=-cnv->preFromULength;
+
+        /* set the error code for unassigned */
+        *pErrorCode=U_INVALID_CHAR_FOUND;
+    }
+}
+
+static UBool
+extSetUseMapping(UConverterUnicodeSet which, int32_t minLength, uint32_t value) {
+    if(which==UCNV_ROUNDTRIP_SET) {
+        // Add only code points for which the roundtrip flag is set.
+        // Do not add any fallbacks, even if ucnv_fromUnicode() would use them
+        // (fallbacks from PUA). See the API docs for ucnv_getUnicodeSet().
+        //
+        // By analogy, also do not add "good one-way" mappings.
+        //
+        // Do not add entries with reserved bits set.
+        if(((value&(UCNV_EXT_FROM_U_ROUNDTRIP_FLAG|UCNV_EXT_FROM_U_RESERVED_MASK))!=
+                UCNV_EXT_FROM_U_ROUNDTRIP_FLAG)) {
+            return false;
+        }
+    } else /* UCNV_ROUNDTRIP_AND_FALLBACK_SET */ {
+        // Do not add entries with reserved bits set.
+        if((value&UCNV_EXT_FROM_U_RESERVED_MASK)!=0) {
+            return false;
+        }
+    }
+    // Do not add <subchar1> entries or other (future?) pseudo-entries
+    // with an output length of 0.
+    return UCNV_EXT_FROM_U_GET_LENGTH(value)>=minLength;
+}
+
+static void
+ucnv_extGetUnicodeSetString(const UConverterSharedData *sharedData,
+                            const int32_t *cx,
+                            const USetAdder *sa,
+                            UConverterUnicodeSet which,
+                            int32_t minLength,
+                            UChar32 firstCP,
+                            char16_t s[UCNV_EXT_MAX_UCHARS], int32_t length,
+                            int32_t sectionIndex,
+                            UErrorCode *pErrorCode) {
+    const char16_t *fromUSectionUChars;
+    const uint32_t *fromUSectionValues;
+
+    uint32_t value;
+    int32_t i, count;
+
+    fromUSectionUChars=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_UCHARS_INDEX, char16_t)+sectionIndex;
+    fromUSectionValues=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_VALUES_INDEX, uint32_t)+sectionIndex;
+
+    /* read first pair of the section */
+    count=*fromUSectionUChars++;
+    value=*fromUSectionValues++;
+
+    if(extSetUseMapping(which, minLength, value)) {
+        if(length==U16_LENGTH(firstCP)) {
+            /* add the initial code point */
+            sa->add(sa->set, firstCP);
+        } else {
+            /* add the string so far */
+            sa->addString(sa->set, s, length);
+        }
+    }
+
+    for(i=0; i<count; ++i) {
+        /* append this code unit and recurse or add the string */
+        s[length]=fromUSectionUChars[i];
+        value=fromUSectionValues[i];
+
+        if(value==0) {
+            /* no mapping, do nothing */
+        } else if(UCNV_EXT_FROM_U_IS_PARTIAL(value)) {
+            ucnv_extGetUnicodeSetString(
+                sharedData, cx, sa, which, minLength,
+                firstCP, s, length+1,
+                (int32_t)UCNV_EXT_FROM_U_GET_PARTIAL_INDEX(value),
+                pErrorCode);
+        } else if(extSetUseMapping(which, minLength, value)) {
+            sa->addString(sa->set, s, length+1);
+        }
+    }
+}
+
+U_CFUNC void
+ucnv_extGetUnicodeSet(const UConverterSharedData *sharedData,
+                      const USetAdder *sa,
+                      UConverterUnicodeSet which,
+                      UConverterSetFilter filter,
+                      UErrorCode *pErrorCode) {
+    const int32_t *cx;
+    const uint16_t *stage12, *stage3, *ps2, *ps3;
+    const uint32_t *stage3b;
+
+    uint32_t value;
+    int32_t st1, stage1Length, st2, st3, minLength;
+
+    char16_t s[UCNV_EXT_MAX_UCHARS];
+    UChar32 c;
+    int32_t length;
+
+    cx=sharedData->mbcs.extIndexes;
+    if(cx==nullptr) {
+        return;
+    }
+
+    stage12=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_STAGE_12_INDEX, uint16_t);
+    stage3=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_STAGE_3_INDEX, uint16_t);
+    stage3b=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_STAGE_3B_INDEX, uint32_t);
+
+    stage1Length=cx[UCNV_EXT_FROM_U_STAGE_1_LENGTH];
+
+    /* enumerate the from-Unicode trie table */
+    c=0; /* keep track of the current code point while enumerating */
+
+    if(filter==UCNV_SET_FILTER_2022_CN) {
+        minLength=3;
+    } else if( sharedData->mbcs.outputType==MBCS_OUTPUT_DBCS_ONLY ||
+               filter!=UCNV_SET_FILTER_NONE
+    ) {
+        /* DBCS-only, ignore single-byte results */
+        minLength=2;
+    } else {
+        minLength=1;
+    }
+
+    /*
+     * the trie enumeration is almost the same as
+     * in MBCSGetUnicodeSet() for MBCS_OUTPUT_1
+     */
+    for(st1=0; st1<stage1Length; ++st1) {
+        st2=stage12[st1];
+        if(st2>stage1Length) {
+            ps2=stage12+st2;
+            for(st2=0; st2<64; ++st2) {
+                if((st3=(int32_t)ps2[st2]<<UCNV_EXT_STAGE_2_LEFT_SHIFT)!=0) {
+                    /* read the stage 3 block */
+                    ps3=stage3+st3;
+
+                    do {
+                        value=stage3b[*ps3++];
+                        if(value==0) {
+                            /* no mapping, do nothing */
+                        } else if(UCNV_EXT_FROM_U_IS_PARTIAL(value)) {
+                            // Recurse for partial results.
+                            length=0;
+                            U16_APPEND_UNSAFE(s, length, c);
+                            ucnv_extGetUnicodeSetString(
+                                sharedData, cx, sa, which, minLength,
+                                c, s, length,
+                                (int32_t)UCNV_EXT_FROM_U_GET_PARTIAL_INDEX(value),
+                                pErrorCode);
+                        } else if(extSetUseMapping(which, minLength, value)) {
+                            switch(filter) {
+                            case UCNV_SET_FILTER_2022_CN:
+                                if(!(UCNV_EXT_FROM_U_GET_LENGTH(value)==3 && UCNV_EXT_FROM_U_GET_DATA(value)<=0x82ffff)) {
+                                    continue;
+                                }
+                                break;
+                            case UCNV_SET_FILTER_SJIS:
+                                if(!(UCNV_EXT_FROM_U_GET_LENGTH(value)==2 && (value=UCNV_EXT_FROM_U_GET_DATA(value))>=0x8140 && value<=0xeffc)) {
+                                    continue;
+                                }
+                                break;
+                            case UCNV_SET_FILTER_GR94DBCS:
+                                if(!(UCNV_EXT_FROM_U_GET_LENGTH(value)==2 &&
+                                     (uint16_t)((value=UCNV_EXT_FROM_U_GET_DATA(value))-0xa1a1)<=(0xfefe - 0xa1a1) &&
+                                     (uint8_t)(value-0xa1)<=(0xfe - 0xa1))) {
+                                    continue;
+                                }
+                                break;
+                            case UCNV_SET_FILTER_HZ:
+                                if(!(UCNV_EXT_FROM_U_GET_LENGTH(value)==2 &&
+                                     (uint16_t)((value=UCNV_EXT_FROM_U_GET_DATA(value))-0xa1a1)<=(0xfdfe - 0xa1a1) &&
+                                     (uint8_t)(value-0xa1)<=(0xfe - 0xa1))) {
+                                    continue;
+                                }
+                                break;
+                            default:
+                                /*
+                                 * UCNV_SET_FILTER_NONE,
+                                 * or UCNV_SET_FILTER_DBCS_ONLY which is handled via minLength
+                                 */
+                                break;
+                            }
+                            sa->add(sa->set, c);
+                        }
+                    } while((++c&0xf)!=0);
+                } else {
+                    c+=16; /* empty stage 3 block */
+                }
+            }
+        } else {
+            c+=1024; /* empty stage 2 block */
+        }
+    }
+}
+
+#endif /* #if !UCONFIG_NO_LEGACY_CONVERSION */