Adding upstream version 115.7.0esr.upstream/115.7.0esrupstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 1397 insertions, 0 deletions

diff --git a/intl/icu/source/extra/uconv/uconv.cpp b/intl/icu/source/extra/uconv/uconv.cpp
new file mode 100644
index 0000000000..0f4af65663
--- /dev/null
+++ b/intl/icu/source/extra/uconv/uconv.cpp
@@ -0,0 +1,1397 @@
+// © 2016 and later: Unicode, Inc. and others.
+// License & terms of use: http://www.unicode.org/copyright.html
+/*****************************************************************************
+*
+*   Copyright (C) 1999-2016, International Business Machines
+*   Corporation and others.  All Rights Reserved.
+*
+******************************************************************************/
+
+/*
+ * uconv(1): an iconv(1)-like converter using ICU.
+ *
+ * Original code by Jonas Utterstr&#x00F6;m <jonas.utterstrom@vittran.norrnod.se>
+ * contributed in 1999.
+ *
+ * Conversion to the C conversion API and many improvements by
+ * Yves Arrouye <yves@realnames.com>, current maintainer.
+ *
+ * Markus Scherer maintainer from 2003.
+ * See source code repository history for changes.
+ */
+
+#include <unicode/utypes.h>
+#include <unicode/putil.h>
+#include <unicode/ucnv.h>
+#include <unicode/uenum.h>
+#include <unicode/unistr.h>
+#include <unicode/translit.h>
+#include <unicode/uset.h>
+#include <unicode/uclean.h>
+#include <unicode/utf16.h>
+
+#include <stdio.h>
+#include <errno.h>
+#include <string.h>
+#include <stdlib.h>
+
+#include "cmemory.h"
+#include "cstring.h"
+#include "ustrfmt.h"
+
+#include "unicode/uwmsg.h"
+
+U_NAMESPACE_USE
+
+#if U_PLATFORM_USES_ONLY_WIN32_API && !defined(__STRICT_ANSI__)
+#include <io.h>
+#include <fcntl.h>
+#if U_PLATFORM_USES_ONLY_WIN32_API
+#define USE_FILENO_BINARY_MODE 1
+/* Windows likes to rename Unix-like functions */
+#ifndef fileno
+#define fileno _fileno
+#endif
+#ifndef setmode
+#define setmode _setmode
+#endif
+#ifndef O_BINARY
+#define O_BINARY _O_BINARY
+#endif
+#endif
+#endif
+
+#ifdef UCONVMSG_LINK
+/* below from the README */
+#include "unicode/utypes.h"
+#include "unicode/udata.h"
+U_CFUNC char uconvmsg_dat[];
+#endif
+
+#define DEFAULT_BUFSZ   4096
+#define UCONVMSG "uconvmsg"
+
+static UResourceBundle *gBundle = 0;    /* Bundle containing messages. */
+
+/*
+ * Initialize the message bundle so that message strings can be fetched
+ * by u_wmsg().
+ *
+ */
+
+static void initMsg(const char *pname) {
+    static int ps = 0;
+
+    if (!ps) {
+        char dataPath[2048];        /* XXX Sloppy: should be PATH_MAX. */
+        UErrorCode err = U_ZERO_ERROR;
+
+        ps = 1;
+
+        /* Set up our static data - if any */
+#if defined(UCONVMSG_LINK) && U_PLATFORM != U_PF_OS390 /* On z/OS, this is failing. */
+        udata_setAppData(UCONVMSG, (const void*) uconvmsg_dat, &err);
+        if (U_FAILURE(err)) {
+          fprintf(stderr, "%s: warning, problem installing our static resource bundle data uconvmsg: %s - trying anyways.\n",
+                  pname, u_errorName(err));
+          err = U_ZERO_ERROR; /* It may still fail */
+        }
+#endif
+
+        /* Get messages. */
+        gBundle = u_wmsg_setPath(UCONVMSG, &err);
+        if (U_FAILURE(err)) {
+            fprintf(stderr,
+                    "%s: warning: couldn't open bundle %s: %s\n",
+                    pname, UCONVMSG, u_errorName(err));
+#ifdef UCONVMSG_LINK
+            fprintf(stderr,
+                    "%s: setAppData was called, internal data %s failed to load\n",
+                        pname, UCONVMSG);
+#endif
+ 
+            err = U_ZERO_ERROR;
+            /* that was try #1, try again with a path */
+            uprv_strcpy(dataPath, u_getDataDirectory());
+            uprv_strcat(dataPath, U_FILE_SEP_STRING);
+            uprv_strcat(dataPath, UCONVMSG);
+
+            gBundle = u_wmsg_setPath(dataPath, &err);
+            if (U_FAILURE(err)) {
+                fprintf(stderr,
+                    "%s: warning: still couldn't open bundle %s: %s\n",
+                    pname, dataPath, u_errorName(err));
+                fprintf(stderr, "%s: warning: messages will not be displayed\n", pname);
+            }
+        }
+    }
+}
+
+/* Mapping of callback names to the callbacks passed to the converter
+   API. */
+
+static struct callback_ent {
+    const char *name;
+    UConverterFromUCallback fromu;
+    const void *fromuctxt;
+    UConverterToUCallback tou;
+    const void *touctxt;
+} transcode_callbacks[] = {
+    { "substitute",
+      UCNV_FROM_U_CALLBACK_SUBSTITUTE, 0,
+      UCNV_TO_U_CALLBACK_SUBSTITUTE, 0 },
+    { "skip",
+      UCNV_FROM_U_CALLBACK_SKIP, 0,
+      UCNV_TO_U_CALLBACK_SKIP, 0 },
+    { "stop",
+      UCNV_FROM_U_CALLBACK_STOP, 0,
+      UCNV_TO_U_CALLBACK_STOP, 0 },
+    { "escape",
+      UCNV_FROM_U_CALLBACK_ESCAPE, 0,
+      UCNV_TO_U_CALLBACK_ESCAPE, 0},
+    { "escape-icu",
+      UCNV_FROM_U_CALLBACK_ESCAPE, UCNV_ESCAPE_ICU,
+      UCNV_TO_U_CALLBACK_ESCAPE, UCNV_ESCAPE_ICU },
+    { "escape-java",
+      UCNV_FROM_U_CALLBACK_ESCAPE, UCNV_ESCAPE_JAVA,
+      UCNV_TO_U_CALLBACK_ESCAPE, UCNV_ESCAPE_JAVA },
+    { "escape-c",
+      UCNV_FROM_U_CALLBACK_ESCAPE, UCNV_ESCAPE_C,
+      UCNV_TO_U_CALLBACK_ESCAPE, UCNV_ESCAPE_C },
+    { "escape-xml",
+      UCNV_FROM_U_CALLBACK_ESCAPE, UCNV_ESCAPE_XML_HEX,
+      UCNV_TO_U_CALLBACK_ESCAPE, UCNV_ESCAPE_XML_HEX },
+    { "escape-xml-hex",
+      UCNV_FROM_U_CALLBACK_ESCAPE, UCNV_ESCAPE_XML_HEX,
+      UCNV_TO_U_CALLBACK_ESCAPE, UCNV_ESCAPE_XML_HEX },
+    { "escape-xml-dec",
+      UCNV_FROM_U_CALLBACK_ESCAPE, UCNV_ESCAPE_XML_DEC,
+      UCNV_TO_U_CALLBACK_ESCAPE, UCNV_ESCAPE_XML_DEC },
+    { "escape-unicode", UCNV_FROM_U_CALLBACK_ESCAPE, UCNV_ESCAPE_UNICODE,
+      UCNV_TO_U_CALLBACK_ESCAPE, UCNV_ESCAPE_UNICODE }
+};
+
+/* Return a pointer to a callback record given its name. */
+
+static const struct callback_ent *findCallback(const char *name) {
+    int i, count =
+        UPRV_LENGTHOF(transcode_callbacks);
+
+    /* We'll do a linear search, there aren't many of them and bsearch()
+       may not be that portable. */
+
+    for (i = 0; i < count; ++i) {
+        if (!uprv_stricmp(name, transcode_callbacks[i].name)) {
+            return &transcode_callbacks[i];
+        }
+    }
+
+    return 0;
+}
+
+/* Print converter information. If lookfor is set, only that converter will
+   be printed, otherwise all converters will be printed. If canon is non
+   zero, tags and aliases for each converter are printed too, in the format
+   expected for convrters.txt(5). */
+
+static int printConverters(const char *pname, const char *lookfor,
+    UBool canon)
+{
+    UErrorCode err = U_ZERO_ERROR;
+    int32_t num;
+    uint16_t num_stds;
+    const char **stds;
+
+    /* If there is a specified name, just handle that now. */
+
+    if (lookfor) {
+        if (!canon) {
+            printf("%s\n", lookfor);
+            return 0;
+        } else {
+        /*  Because we are printing a canonical name, we need the
+            true converter name. We've done that already except for
+            the default name (because we want to print the exact
+            name one would get when calling ucnv_getDefaultName()
+            in non-canon mode). But since we do not know at this
+            point if we have the default name or something else, we
+            need to normalize again to the canonical converter
+            name. */
+
+            const char *truename = ucnv_getAlias(lookfor, 0, &err);
+            if (U_SUCCESS(err)) {
+                lookfor = truename;
+            } else {
+                err = U_ZERO_ERROR;
+            }
+        }
+    }
+
+    /* Print converter names. We come here for one of two reasons: we
+       are printing all the names (lookfor was null), or we have a
+       single converter to print but in canon mode, hence we need to
+       get to it in order to print everything. */
+
+    num = ucnv_countAvailable();
+    if (num <= 0) {
+        initMsg(pname);
+        u_wmsg(stderr, "cantGetNames");
+        return -1;
+    }
+    if (lookfor) {
+        num = 1;                /* We know where we want to be. */
+    }
+
+    num_stds = ucnv_countStandards();
+    stds = (const char **) uprv_malloc(num_stds * sizeof(*stds));
+    if (!stds) {
+        u_wmsg(stderr, "cantGetTag", u_wmsg_errorName(U_MEMORY_ALLOCATION_ERROR));
+        return -1;
+    } else {
+        uint16_t s;
+
+        if (canon) {
+            printf("{ ");
+        }
+        for (s = 0; s < num_stds; ++s) {
+            stds[s] = ucnv_getStandard(s, &err);
+            if (canon) {
+                printf("%s ", stds[s]);
+            }
+            if (U_FAILURE(err)) {
+                u_wmsg(stderr, "cantGetTag", u_wmsg_errorName(err));
+                goto error_cleanup;
+            }
+        }
+        if (canon) {
+            puts("}");
+        }
+    }
+
+    for (int32_t i = 0; i < num; i++) {
+        const char *name;
+        uint16_t num_aliases;
+
+        /* Set the name either to what we are looking for, or
+        to the current converter name. */
+
+        if (lookfor) {
+            name = lookfor;
+        } else {
+            name = ucnv_getAvailableName(i);
+        }
+
+        /* Get all the aliases associated to the name. */
+
+        err = U_ZERO_ERROR;
+        num_aliases = ucnv_countAliases(name, &err);
+        if (U_FAILURE(err)) {
+            printf("%s", name);
+
+            UnicodeString str(name, "");
+            putchar('\t');
+            u_wmsg(stderr, "cantGetAliases", str.getTerminatedBuffer(),
+                u_wmsg_errorName(err));
+            goto error_cleanup;
+        } else {
+            uint16_t a, s, t;
+
+            /* Write all the aliases and their tags. */
+
+            for (a = 0; a < num_aliases; ++a) {
+                const char *alias = ucnv_getAlias(name, a, &err);
+
+                if (U_FAILURE(err)) {
+                    UnicodeString str(name, "");
+                    putchar('\t');
+                    u_wmsg(stderr, "cantGetAliases", str.getTerminatedBuffer(),
+                        u_wmsg_errorName(err));
+                    goto error_cleanup;
+                }
+
+                /* Print the current alias so that it looks right. */
+                printf("%s%s%s", (canon ? (a == 0? "" : "\t" ) : "") ,
+                                 alias,
+                                 (canon ? "" : " "));
+
+                /* Look (slowly, linear searching) for a tag. */
+
+                if (canon) {
+                    /* -1 to skip the last standard */
+                    for (s = t = 0; s < num_stds-1; ++s) {
+                        UEnumeration *nameEnum = ucnv_openStandardNames(name, stds[s], &err);
+                        if (U_SUCCESS(err)) {
+                            /* List the standard tags */
+                            const char *standardName;
+                            UBool isFirst = true;
+                            UErrorCode enumError = U_ZERO_ERROR;
+                            while ((standardName = uenum_next(nameEnum, nullptr, &enumError))) {
+                                /* See if this alias is supported by this standard. */
+                                if (!strcmp(standardName, alias)) {
+                                    if (!t) {
+                                        printf(" {");
+                                        t = 1;
+                                    }
+                                    /* Print a * after the default standard name */
+                                    printf(" %s%s", stds[s], (isFirst ? "*" : ""));
+                                }
+                                isFirst = false;
+                            }
+                        }
+                    }
+                    if (t) {
+                        printf(" }");
+                    }
+                }
+                /* Terminate this entry. */
+                if (canon) {
+                    puts("");
+                }
+
+                /* Move on. */
+            }
+            /* Terminate this entry. */
+            if (!canon) {
+                puts("");
+            }
+        }
+    }
+
+    /* Free temporary data. */
+
+    uprv_free(stds);
+
+    /* Success. */
+
+    return 0;
+error_cleanup:
+    uprv_free(stds);
+    return -1;
+}
+
+/* Print all available transliterators. If canon is non zero, print
+   one transliterator per line. */
+
+static int printTransliterators(UBool canon)
+{
+#if UCONFIG_NO_TRANSLITERATION
+    printf("no transliterators available because of UCONFIG_NO_TRANSLITERATION, see uconfig.h\n");
+    return 1;
+#else
+    UErrorCode status = U_ZERO_ERROR;
+    UEnumeration *ids = utrans_openIDs(&status);
+    int32_t i, numtrans = uenum_count(ids, &status);
+
+    char sepchar = canon ? '\n' : ' ';
+
+    for (i = 0; U_SUCCESS(status)&& (i < numtrans); ++i) {
+    	int32_t len;
+    	const char *nextTrans = uenum_next(ids, &len, &status);
+
+        printf("%s", nextTrans);
+        if (i < numtrans - 1) {
+            putchar(sepchar);
+        }
+    }
+
+    uenum_close(ids);
+
+    /* Add a terminating newline if needed. */
+
+    if (sepchar != '\n') {
+        putchar('\n');
+    }
+
+    /* Success. */
+
+    return 0;
+#endif
+}
+
+enum {
+    uSP = 0x20,         // space
+    uCR = 0xd,          // carriage return
+    uLF = 0xa,          // line feed
+    uNL = 0x85,         // newline
+    uLS = 0x2028,       // line separator
+    uPS = 0x2029,       // paragraph separator
+    uSig = 0xfeff       // signature/BOM character
+};
+
+static inline int32_t
+getChunkLimit(const UnicodeString &prev, const UnicodeString &s) {
+    // find one of
+    // CR, LF, CRLF, NL, LS, PS
+    // for paragraph ends (see UAX #13/Unicode 4)
+    // and include it in the chunk
+    // all of these characters are on the BMP
+    // do not include FF or VT in case they are part of a paragraph
+    // (important for bidi contexts)
+    static const char16_t paraEnds[] = {
+        0xd, 0xa, 0x85, 0x2028, 0x2029
+    };
+    enum {
+        iCR, iLF, iNL, iLS, iPS, iCount
+    };
+
+    // first, see if there is a CRLF split between prev and s
+    if (prev.endsWith(paraEnds + iCR, 1)) {
+        if (s.startsWith(paraEnds + iLF, 1)) {
+            return 1; // split CRLF, include the LF
+        } else if (!s.isEmpty()) {
+            return 0; // complete the last chunk
+        } else {
+            return -1; // wait for actual further contents to arrive
+        }
+    }
+
+    const char16_t *u = s.getBuffer(), *limit = u + s.length();
+    char16_t c;
+
+    while (u < limit) {
+        c = *u++;
+        if (
+            ((c < uSP) && (c == uCR || c == uLF)) ||
+            (c == uNL) ||
+            ((c & uLS) == uLS)
+        ) {
+            if (c == uCR) {
+                // check for CRLF
+                if (u == limit) {
+                    return -1; // LF may be in the next chunk
+                } else if (*u == uLF) {
+                    ++u; // include the LF in this chunk
+                }
+            }
+            return (int32_t)(u - s.getBuffer());
+        }
+    }
+
+    return -1; // continue collecting the chunk
+}
+
+enum {
+    CNV_NO_FEFF,    // cannot convert the U+FEFF Unicode signature character (BOM)
+    CNV_WITH_FEFF,  // can convert the U+FEFF signature character
+    CNV_ADDS_FEFF   // automatically adds/detects the U+FEFF signature character
+};
+
+static inline char16_t
+nibbleToHex(uint8_t n) {
+    n &= 0xf;
+    return
+        n <= 9 ?
+            (char16_t)(0x30 + n) :
+            (char16_t)((0x61 - 10) + n);
+}
+
+// check the converter's Unicode signature properties;
+// the fromUnicode side of the converter must be in its initial state
+// and will be reset again if it was used
+static int32_t
+cnvSigType(UConverter *cnv) {
+    UErrorCode err;
+    int32_t result;
+
+    // test if the output charset can convert U+FEFF
+    USet *set = uset_open(1, 0);
+    err = U_ZERO_ERROR;
+    ucnv_getUnicodeSet(cnv, set, UCNV_ROUNDTRIP_SET, &err);
+    if (U_SUCCESS(err) && uset_contains(set, uSig)) {
+        result = CNV_WITH_FEFF;
+    } else {
+        result = CNV_NO_FEFF; // an error occurred or U+FEFF cannot be converted
+    }
+    uset_close(set);
+
+    if (result == CNV_WITH_FEFF) {
+        // test if the output charset emits a signature anyway
+        const char16_t a[1] = { 0x61 }; // "a"
+        const char16_t *in;
+
+        char buffer[20];
+        char *out;
+
+        in = a;
+        out = buffer;
+        err = U_ZERO_ERROR;
+        ucnv_fromUnicode(cnv,
+            &out, buffer + sizeof(buffer),
+            &in, a + 1,
+            nullptr, true, &err);
+        ucnv_resetFromUnicode(cnv);
+
+        if (nullptr != ucnv_detectUnicodeSignature(buffer, (int32_t)(out - buffer), nullptr, &err) &&
+            U_SUCCESS(err)
+        ) {
+            result = CNV_ADDS_FEFF;
+        }
+    }
+
+    return result;
+}
+
+class ConvertFile {
+public:
+    ConvertFile() :
+        buf(nullptr), outbuf(nullptr), fromoffsets(nullptr),
+        bufsz(0), signature(0) {}
+
+    void
+    setBufferSize(size_t bufferSize) {
+        bufsz = bufferSize;
+
+        buf = new char[2 * bufsz];
+        outbuf = buf + bufsz;
+
+        // +1 for an added U+FEFF in the intermediate Unicode buffer
+        fromoffsets = new int32_t[bufsz + 1];
+    }
+
+    ~ConvertFile() {
+        delete [] buf;
+        delete [] fromoffsets;
+    }
+
+    UBool convertFile(const char *pname,
+                      const char *fromcpage,
+                      UConverterToUCallback toucallback,
+                      const void *touctxt,
+                      const char *tocpage,
+                      UConverterFromUCallback fromucallback,
+                      const void *fromuctxt,
+                      UBool fallback,
+                      const char *translit,
+                      const char *infilestr,
+                      FILE * outfile, int verbose);
+private:
+    friend int main(int argc, char **argv);
+
+    char *buf, *outbuf;
+    int32_t *fromoffsets;
+
+    size_t bufsz;
+    int8_t signature; // add (1) or remove (-1) a U+FEFF Unicode signature character
+};
+
+// Convert a file from one encoding to another
+UBool
+ConvertFile::convertFile(const char *pname,
+                         const char *fromcpage,
+                         UConverterToUCallback toucallback,
+                         const void *touctxt,
+                         const char *tocpage,
+                         UConverterFromUCallback fromucallback,
+                         const void *fromuctxt,
+                         UBool fallback,
+                         const char *translit,
+                         const char *infilestr,
+                         FILE * outfile, int verbose)
+{
+    FILE *infile;
+    UBool ret = true;
+    UConverter *convfrom = 0;
+    UConverter *convto = 0;
+    UErrorCode err = U_ZERO_ERROR;
+    UBool flush;
+    UBool closeFile = false;
+    const char *cbufp, *prevbufp;
+    char *bufp;
+
+    uint32_t infoffset = 0, outfoffset = 0;   /* Where we are in the file, for error reporting. */
+
+    const char16_t *unibuf, *unibufbp;
+    char16_t *unibufp;
+
+    size_t rd, wr;
+
+#if !UCONFIG_NO_TRANSLITERATION
+    Transliterator *t = 0;      // Transliterator acting on Unicode data.
+    UnicodeString chunk;        // One chunk of the text being collected for transformation.
+#endif
+    UnicodeString u;            // String to do the transliteration.
+    int32_t ulen;
+
+    // use conversion offsets for error messages
+    // unless a transliterator is used -
+    // a text transformation will reorder characters in unpredictable ways
+    UBool useOffsets = true;
+
+    // Open the correct input file or connect to stdin for reading input
+
+    if (infilestr != 0 && strcmp(infilestr, "-")) {
+        infile = fopen(infilestr, "rb");
+        if (infile == 0) {
+            UnicodeString str1(infilestr, "");
+            str1.append((UChar32) 0);
+            UnicodeString str2(strerror(errno), "");
+            str2.append((UChar32) 0);
+            initMsg(pname);
+            u_wmsg(stderr, "cantOpenInputF", str1.getBuffer(), str2.getBuffer());
+            return false;
+        }
+        closeFile = true;
+    } else {
+        infilestr = "-";
+        infile = stdin;
+#ifdef USE_FILENO_BINARY_MODE
+        if (setmode(fileno(stdin), O_BINARY) == -1) {
+            initMsg(pname);
+            u_wmsg(stderr, "cantSetInBinMode");
+            return false;
+        }
+#endif
+    }
+
+    if (verbose) {
+        fprintf(stderr, "%s:\n", infilestr);
+    }
+
+#if !UCONFIG_NO_TRANSLITERATION
+    // Create transliterator as needed.
+
+    if (translit != nullptr && *translit) {
+        UParseError parse;
+        UnicodeString str(translit), pestr;
+
+        /* Create from rules or by ID as needed. */
+
+        parse.line = -1;
+
+        if (uprv_strchr(translit, ':') || uprv_strchr(translit, '>') || uprv_strchr(translit, '<') || uprv_strchr(translit, '>')) {
+            t = Transliterator::createFromRules(UNICODE_STRING_SIMPLE("Uconv"), str, UTRANS_FORWARD, parse, err);
+        } else {
+            t = Transliterator::createInstance(UnicodeString(translit, -1, US_INV), UTRANS_FORWARD, err);
+        }
+
+        if (U_FAILURE(err)) {
+            str.append((UChar32) 0);
+            initMsg(pname);
+
+            if (parse.line >= 0) {
+                char16_t linebuf[20], offsetbuf[20];
+                uprv_itou(linebuf, 20, parse.line, 10, 0);
+                uprv_itou(offsetbuf, 20, parse.offset, 10, 0);
+                u_wmsg(stderr, "cantCreateTranslitParseErr", str.getTerminatedBuffer(),
+                    u_wmsg_errorName(err), linebuf, offsetbuf);
+            } else {
+                u_wmsg(stderr, "cantCreateTranslit", str.getTerminatedBuffer(),
+                    u_wmsg_errorName(err));
+            }
+
+            if (t) {
+                delete t;
+                t = 0;
+            }
+            goto error_exit;
+        }
+
+        useOffsets = false;
+    }
+#endif
+
+    // Create codepage converter. If the codepage or its aliases weren't
+    // available, it returns nullptr and a failure code. We also set the
+    // callbacks, and return errors in the same way.
+
+    convfrom = ucnv_open(fromcpage, &err);
+    if (U_FAILURE(err)) {
+        UnicodeString str(fromcpage, "");
+        initMsg(pname);
+        u_wmsg(stderr, "cantOpenFromCodeset", str.getTerminatedBuffer(),
+            u_wmsg_errorName(err));
+        goto error_exit;
+    }
+    ucnv_setToUCallBack(convfrom, toucallback, touctxt, 0, 0, &err);
+    if (U_FAILURE(err)) {
+        initMsg(pname);
+        u_wmsg(stderr, "cantSetCallback", u_wmsg_errorName(err));
+        goto error_exit;
+    }
+
+    convto = ucnv_open(tocpage, &err);
+    if (U_FAILURE(err)) {
+        UnicodeString str(tocpage, "");
+        initMsg(pname);
+        u_wmsg(stderr, "cantOpenToCodeset", str.getTerminatedBuffer(),
+            u_wmsg_errorName(err));
+        goto error_exit;
+    }
+    ucnv_setFromUCallBack(convto, fromucallback, fromuctxt, 0, 0, &err);
+    if (U_FAILURE(err)) {
+        initMsg(pname);
+        u_wmsg(stderr, "cantSetCallback", u_wmsg_errorName(err));
+        goto error_exit;
+    }
+    ucnv_setFallback(convto, fallback);
+
+    UBool willexit, fromSawEndOfBytes, toSawEndOfUnicode;
+    int8_t sig;
+
+    // OK, we can convert now.
+    sig = signature;
+    rd = 0;
+
+    do {
+        willexit = false;
+
+        // input file offset at the beginning of the next buffer
+        infoffset += static_cast<uint32_t>(rd);
+
+        rd = fread(buf, 1, bufsz, infile);
+        if (ferror(infile) != 0) {
+            UnicodeString str(strerror(errno));
+            initMsg(pname);
+            u_wmsg(stderr, "cantRead", str.getTerminatedBuffer());
+            goto error_exit;
+        }
+
+        // Convert the read buffer into the new encoding via Unicode.
+        // After the call 'unibufp' will be placed behind the last
+        // character that was converted in the 'unibuf'.
+        // Also the 'cbufp' is positioned behind the last converted
+        // character.
+        // At the last conversion in the file, flush should be set to
+        // true so that we get all characters converted.
+        //
+        // The converter must be flushed at the end of conversion so
+        // that characters on hold also will be written.
+
+        cbufp = buf;
+        flush = (UBool)(rd != bufsz);
+
+        // convert until the input is consumed
+        do {
+            // remember the start of the current byte-to-Unicode conversion
+            prevbufp = cbufp;
+
+            unibuf = unibufp = u.getBuffer((int32_t)bufsz);
+
+            // Use bufsz instead of u.getCapacity() for the targetLimit
+            // so that we don't overflow fromoffsets[].
+            ucnv_toUnicode(convfrom, &unibufp, unibuf + bufsz, &cbufp,
+                buf + rd, useOffsets ? fromoffsets : nullptr, flush, &err);
+
+            ulen = (int32_t)(unibufp - unibuf);
+            u.releaseBuffer(U_SUCCESS(err) ? ulen : 0);
+
+            // fromSawEndOfBytes indicates that ucnv_toUnicode() is done
+            // converting all of the input bytes.
+            // It works like this because ucnv_toUnicode() returns only under the
+            // following conditions:
+            // - an error occurred during conversion (an error code is set)
+            // - the target buffer is filled (the error code indicates an overflow)
+            // - the source is consumed
+            // That is, if the error code does not indicate a failure,
+            // not even an overflow, then the source must be consumed entirely.
+            fromSawEndOfBytes = (UBool)U_SUCCESS(err);
+
+            if (err == U_BUFFER_OVERFLOW_ERROR) {
+                err = U_ZERO_ERROR;
+            } else if (U_FAILURE(err)) {
+                char pos[32], errorBytes[32];
+                int8_t i, length, errorLength;
+
+                UErrorCode localError = U_ZERO_ERROR;
+                errorLength = (int8_t)sizeof(errorBytes);
+                ucnv_getInvalidChars(convfrom, errorBytes, &errorLength, &localError);
+                if (U_FAILURE(localError) || errorLength == 0) {
+                    errorLength = 1;
+                }
+
+                // print the input file offset of the start of the error bytes:
+                // input file offset of the current byte buffer +
+                // length of the just consumed bytes -
+                // length of the error bytes
+                length =
+                    (int8_t)snprintf(pos, sizeof(pos), "%d",
+                        (int)(infoffset + (cbufp - buf) - errorLength));
+
+                // output the bytes that caused the error
+                UnicodeString str;
+                for (i = 0; i < errorLength; ++i) {
+                    if (i > 0) {
+                        str.append((char16_t)uSP);
+                    }
+                    str.append(nibbleToHex((uint8_t)errorBytes[i] >> 4));
+                    str.append(nibbleToHex((uint8_t)errorBytes[i]));
+                }
+
+                initMsg(pname);
+                u_wmsg(stderr, "problemCvtToU",
+                        UnicodeString(pos, length, "").getTerminatedBuffer(),
+                        str.getTerminatedBuffer(),
+                        u_wmsg_errorName(err));
+
+                willexit = true;
+                err = U_ZERO_ERROR; /* reset the error for the rest of the conversion. */
+            }
+
+            // Replaced a check for whether the input was consumed by
+            // looping until it is; message key "premEndInput" now obsolete.
+
+            if (ulen == 0) {
+                continue;
+            }
+
+            // remove a U+FEFF Unicode signature character if requested
+            if (sig < 0) {
+                if (u.charAt(0) == uSig) {
+                    u.remove(0, 1);
+
+                    // account for the removed char16_t and offset
+                    --ulen;
+
+                    if (useOffsets) {
+                        // remove an offset from fromoffsets[] as well
+                        // to keep the array parallel with the UChars
+                        memmove(fromoffsets, fromoffsets + 1, ulen * 4);
+                    }
+
+                }
+                sig = 0;
+            }
+
+#if !UCONFIG_NO_TRANSLITERATION
+            // Transliterate/transform if needed.
+
+            // For transformation, we use chunking code -
+            // collect Unicode input until, for example, an end-of-line,
+            // then transform and output-convert that and continue collecting.
+            // This makes the transformation result independent of the buffer size
+            // while avoiding the slower keyboard mode.
+            // The end-of-chunk characters are completely included in the
+            // transformed string in case they are to be transformed themselves.
+            if (t != nullptr) {
+                UnicodeString out;
+                int32_t chunkLimit;
+
+                do {
+                    chunkLimit = getChunkLimit(chunk, u);
+                    if (chunkLimit < 0 && flush && fromSawEndOfBytes) {
+                        // use all of the rest at the end of the text
+                        chunkLimit = u.length();
+                    }
+                    if (chunkLimit >= 0) {
+                        // complete the chunk and transform it
+                        chunk.append(u, 0, chunkLimit);
+                        u.remove(0, chunkLimit);
+                        t->transliterate(chunk);
+
+                        // append the transformation result to the result and empty the chunk
+                        out.append(chunk);
+                        chunk.remove();
+                    } else {
+                        // continue collecting the chunk
+                        chunk.append(u);
+                        break;
+                    }
+                } while (!u.isEmpty());
+
+                u = out;
+                ulen = u.length();
+            }
+#endif
+
+            // add a U+FEFF Unicode signature character if requested
+            // and possible/necessary
+            if (sig > 0) {
+                if (u.charAt(0) != uSig && cnvSigType(convto) == CNV_WITH_FEFF) {
+                    u.insert(0, (char16_t)uSig);
+
+                    if (useOffsets) {
+                        // insert a pseudo-offset into fromoffsets[] as well
+                        // to keep the array parallel with the UChars
+                        memmove(fromoffsets + 1, fromoffsets, ulen * 4);
+                        fromoffsets[0] = -1;
+                    }
+
+                    // account for the additional char16_t and offset
+                    ++ulen;
+                }
+                sig = 0;
+            }
+
+            // Convert the Unicode buffer into the destination codepage
+            // Again 'bufp' will be placed behind the last converted character
+            // And 'unibufp' will be placed behind the last converted unicode character
+            // At the last conversion flush should be set to true to ensure that
+            // all characters left get converted
+
+            unibuf = unibufbp = u.getBuffer();
+
+            do {
+                bufp = outbuf;
+
+                // Use fromSawEndOfBytes in addition to the flush flag -
+                // it indicates whether the intermediate Unicode string
+                // contains the very last UChars for the very last input bytes.
+                ucnv_fromUnicode(convto, &bufp, outbuf + bufsz,
+                                 &unibufbp,
+                                 unibuf + ulen,
+                                 nullptr, (UBool)(flush && fromSawEndOfBytes), &err);
+
+                // toSawEndOfUnicode indicates that ucnv_fromUnicode() is done
+                // converting all of the intermediate UChars.
+                // See comment for fromSawEndOfBytes.
+                toSawEndOfUnicode = (UBool)U_SUCCESS(err);
+
+                if (err == U_BUFFER_OVERFLOW_ERROR) {
+                    err = U_ZERO_ERROR;
+                } else if (U_FAILURE(err)) {
+                    char16_t errorUChars[4];
+                    const char *errtag;
+                    char pos[32];
+                    UChar32 c;
+                    int8_t i, length, errorLength;
+
+                    UErrorCode localError = U_ZERO_ERROR;
+                    errorLength = UPRV_LENGTHOF(errorUChars);
+                    ucnv_getInvalidUChars(convto, errorUChars, &errorLength, &localError);
+                    if (U_FAILURE(localError) || errorLength == 0) {
+                        // need at least 1 so that we don't access beyond the length of fromoffsets[]
+                        errorLength = 1;
+                    }
+
+                    int32_t ferroffset;
+
+                    if (useOffsets) {
+                        // Unicode buffer offset of the start of the error UChars
+                        ferroffset = (int32_t)((unibufbp - unibuf) - errorLength);
+                        if (ferroffset < 0) {
+                            // approximation - the character started in the previous Unicode buffer
+                            ferroffset = 0;
+                        }
+
+                        // get the corresponding byte offset out of fromoffsets[]
+                        // go back if the offset is not known for some of the UChars
+                        int32_t fromoffset;
+                        do {
+                            fromoffset = fromoffsets[ferroffset];
+                        } while (fromoffset < 0 && --ferroffset >= 0);
+
+                        // total input file offset =
+                        // input file offset of the current byte buffer +
+                        // byte buffer offset of where the current Unicode buffer is converted from +
+                        // fromoffsets[Unicode offset]
+                        ferroffset = static_cast<int32_t>(infoffset + (prevbufp - buf) + fromoffset);
+                        errtag = "problemCvtFromU";
+                    } else {
+                        // Do not use fromoffsets if (t != nullptr) because the Unicode text may
+                        // be different from what the offsets refer to.
+
+                        // output file offset
+                        ferroffset = (int32_t)(outfoffset + (bufp - outbuf));
+                        errtag = "problemCvtFromUOut";
+                    }
+
+                    length = (int8_t)snprintf(pos, sizeof(pos), "%u", (int)ferroffset);
+
+                    // output the code points that caused the error
+                    UnicodeString str;
+                    for (i = 0; i < errorLength;) {
+                        if (i > 0) {
+                            str.append((char16_t)uSP);
+                        }
+                        U16_NEXT(errorUChars, i, errorLength, c);
+                        if (c >= 0x100000) {
+                            str.append(nibbleToHex((uint8_t)(c >> 20)));
+                        }
+                        if (c >= 0x10000) {
+                            str.append(nibbleToHex((uint8_t)(c >> 16)));
+                        }
+                        str.append(nibbleToHex((uint8_t)(c >> 12)));
+                        str.append(nibbleToHex((uint8_t)(c >> 8)));
+                        str.append(nibbleToHex((uint8_t)(c >> 4)));
+                        str.append(nibbleToHex((uint8_t)c));
+                    }
+
+                    initMsg(pname);
+                    u_wmsg(stderr, errtag,
+                            UnicodeString(pos, length, "").getTerminatedBuffer(),
+                            str.getTerminatedBuffer(),
+                           u_wmsg_errorName(err));
+                    u_wmsg(stderr, "errorUnicode", str.getTerminatedBuffer());
+
+                    willexit = true;
+                    err = U_ZERO_ERROR; /* reset the error for the rest of the conversion. */
+                }
+
+                // Replaced a check for whether the intermediate Unicode characters were all consumed by
+                // looping until they are; message key "premEnd" now obsolete.
+
+                // Finally, write the converted buffer to the output file
+                size_t outlen = (size_t) (bufp - outbuf);
+                outfoffset += (int32_t)(wr = fwrite(outbuf, 1, outlen, outfile));
+                if (wr != outlen) {
+                    UnicodeString str(strerror(errno));
+                    initMsg(pname);
+                    u_wmsg(stderr, "cantWrite", str.getTerminatedBuffer());
+                    willexit = true;
+                }
+
+                if (willexit) {
+                    goto error_exit;
+                }
+            } while (!toSawEndOfUnicode);
+        } while (!fromSawEndOfBytes);
+    } while (!flush);           // Stop when we have flushed the
+                                // converters (this means that it's
+                                // the end of output)
+
+    goto normal_exit;
+
+error_exit:
+    ret = false;
+
+normal_exit:
+    // Cleanup.
+
+    ucnv_close(convfrom);
+    ucnv_close(convto);
+
+#if !UCONFIG_NO_TRANSLITERATION
+    delete t;
+#endif
+
+    if (closeFile) {
+        fclose(infile);
+    }
+
+    return ret;
+}
+
+static void usage(const char *pname, int ecode) {
+    const char16_t *msg;
+    int32_t msgLen;
+    UErrorCode err = U_ZERO_ERROR;
+    FILE *fp = ecode ? stderr : stdout;
+    int res;
+
+    initMsg(pname);
+    msg =
+        ures_getStringByKey(gBundle, ecode ? "lcUsageWord" : "ucUsageWord",
+                            &msgLen, &err);
+    UnicodeString upname(pname, (int32_t)(uprv_strlen(pname) + 1));
+    UnicodeString mname(msg, msgLen + 1);
+
+    res = u_wmsg(fp, "usage", mname.getBuffer(), upname.getBuffer());
+    if (!ecode) {
+        if (!res) {
+            fputc('\n', fp);
+        }
+        if (!u_wmsg(fp, "help")) {
+            /* Now dump callbacks and finish. */
+
+            int i, count =
+                UPRV_LENGTHOF(transcode_callbacks);
+            for (i = 0; i < count; ++i) {
+                fprintf(fp, " %s", transcode_callbacks[i].name);
+            }
+            fputc('\n', fp);
+        }
+    }
+
+    exit(ecode);
+}
+
+extern int
+main(int argc, char **argv)
+{
+    FILE *outfile;
+    int ret = 0;
+
+    size_t bufsz = DEFAULT_BUFSZ;
+
+    const char *fromcpage = 0;
+    const char *tocpage = 0;
+    const char *translit = 0;
+    const char *outfilestr = 0;
+    UBool fallback = false;
+
+    UConverterFromUCallback fromucallback = UCNV_FROM_U_CALLBACK_STOP;
+    const void *fromuctxt = 0;
+    UConverterToUCallback toucallback = UCNV_TO_U_CALLBACK_STOP;
+    const void *touctxt = 0;
+
+    char **iter, **remainArgv, **remainArgvLimit;
+    char **end = argv + argc;
+
+    const char *pname;
+
+    UBool printConvs = false, printCanon = false, printTranslits = false;
+    const char *printName = 0;
+
+    UBool verbose = false;
+    UErrorCode status = U_ZERO_ERROR;
+
+    ConvertFile cf;
+
+    /* Initialize ICU */
+    u_init(&status);
+    if (U_FAILURE(status)) {
+        fprintf(stderr, "%s: can not initialize ICU.  status = %s\n",
+            argv[0], u_errorName(status));
+        exit(1);
+    }
+
+    // Get and prettify pname.
+    pname = uprv_strrchr(*argv, U_FILE_SEP_CHAR);
+#if U_PLATFORM_USES_ONLY_WIN32_API
+    if (!pname) {
+        pname = uprv_strrchr(*argv, '/');
+    }
+#endif
+    if (!pname) {
+        pname = *argv;
+    } else {
+        ++pname;
+    }
+
+    // First, get the arguments from command-line
+    // to know the codepages to convert between
+
+    remainArgv = remainArgvLimit = argv + 1;
+    for (iter = argv + 1; iter != end; iter++) {
+        // Check for from charset
+        if (strcmp("-f", *iter) == 0 || !strcmp("--from-code", *iter)) {
+            iter++;
+            if (iter != end)
+                fromcpage = *iter;
+            else
+                usage(pname, 1);
+        } else if (strcmp("-t", *iter) == 0 || !strcmp("--to-code", *iter)) {
+            iter++;
+            if (iter != end)
+                tocpage = *iter;
+            else
+                usage(pname, 1);
+        } else if (strcmp("-x", *iter) == 0) {
+            iter++;
+            if (iter != end)
+                translit = *iter;
+            else
+                usage(pname, 1);
+        } else if (!strcmp("--fallback", *iter)) {
+            fallback = true;
+        } else if (!strcmp("--no-fallback", *iter)) {
+            fallback = false;
+        } else if (strcmp("-b", *iter) == 0 || !strcmp("--block-size", *iter)) {
+            iter++;
+            if (iter != end) {
+                bufsz = atoi(*iter);
+                if ((int) bufsz <= 0) {
+                    initMsg(pname);
+                    UnicodeString str(*iter);
+                    initMsg(pname);
+                    u_wmsg(stderr, "badBlockSize", str.getTerminatedBuffer());
+                    return 3;
+                }
+            } else {
+                usage(pname, 1);
+            }
+        } else if (strcmp("-l", *iter) == 0 || !strcmp("--list", *iter)) {
+            if (printTranslits) {
+                usage(pname, 1);
+            }
+            printConvs = true;
+        } else if (strcmp("--default-code", *iter) == 0) {
+            if (printTranslits) {
+                usage(pname, 1);
+            }
+            printName = ucnv_getDefaultName();
+        } else if (strcmp("--list-code", *iter) == 0) {
+            if (printTranslits) {
+                usage(pname, 1);
+            }
+
+            iter++;
+            if (iter != end) {
+                UErrorCode e = U_ZERO_ERROR;
+                printName = ucnv_getAlias(*iter, 0, &e);
+                if (U_FAILURE(e) || !printName) {
+                    UnicodeString str(*iter);
+                    initMsg(pname);
+                    u_wmsg(stderr, "noSuchCodeset", str.getTerminatedBuffer());
+                    return 2;
+                }
+            } else
+                usage(pname, 1);
+        } else if (strcmp("--canon", *iter) == 0) {
+            printCanon = true;
+        } else if (strcmp("-L", *iter) == 0
+            || !strcmp("--list-transliterators", *iter)) {
+            if (printConvs) {
+                usage(pname, 1);
+            }
+            printTranslits = true;
+        } else if (strcmp("-h", *iter) == 0 || !strcmp("-?", *iter)
+            || !strcmp("--help", *iter)) {
+            usage(pname, 0);
+        } else if (!strcmp("-c", *iter)) {
+            fromucallback = UCNV_FROM_U_CALLBACK_SKIP;
+        } else if (!strcmp("--to-callback", *iter)) {
+            iter++;
+            if (iter != end) {
+                const struct callback_ent *cbe = findCallback(*iter);
+                if (cbe) {
+                    fromucallback = cbe->fromu;
+                    fromuctxt = cbe->fromuctxt;
+                } else {
+                    UnicodeString str(*iter);
+                    initMsg(pname);
+                    u_wmsg(stderr, "unknownCallback", str.getTerminatedBuffer());
+                    return 4;
+                }
+            } else {
+                usage(pname, 1);
+            }
+        } else if (!strcmp("--from-callback", *iter)) {
+            iter++;
+            if (iter != end) {
+                const struct callback_ent *cbe = findCallback(*iter);
+                if (cbe) {
+                    toucallback = cbe->tou;
+                    touctxt = cbe->touctxt;
+                } else {
+                    UnicodeString str(*iter);
+                    initMsg(pname);
+                    u_wmsg(stderr, "unknownCallback", str.getTerminatedBuffer());
+                    return 4;
+                }
+            } else {
+                usage(pname, 1);
+            }
+        } else if (!strcmp("-i", *iter)) {
+            toucallback = UCNV_TO_U_CALLBACK_SKIP;
+        } else if (!strcmp("--callback", *iter)) {
+            iter++;
+            if (iter != end) {
+                const struct callback_ent *cbe = findCallback(*iter);
+                if (cbe) {
+                    fromucallback = cbe->fromu;
+                    fromuctxt = cbe->fromuctxt;
+                    toucallback = cbe->tou;
+                    touctxt = cbe->touctxt;
+                } else {
+                    UnicodeString str(*iter);
+                    initMsg(pname);
+                    u_wmsg(stderr, "unknownCallback", str.getTerminatedBuffer());
+                    return 4;
+                }
+            } else {
+                usage(pname, 1);
+            }
+        } else if (!strcmp("-s", *iter) || !strcmp("--silent", *iter)) {
+            verbose = false;
+        } else if (!strcmp("-v", *iter) || !strcmp("--verbose", *iter)) {
+            verbose = true;
+        } else if (!strcmp("-V", *iter) || !strcmp("--version", *iter)) {
+            printf("%s v2.1  ICU " U_ICU_VERSION "\n", pname);
+            return 0;
+        } else if (!strcmp("-o", *iter) || !strcmp("--output", *iter)) {
+            ++iter;
+            if (iter != end && !outfilestr) {
+                outfilestr = *iter;
+            } else {
+                usage(pname, 1);
+            }
+        } else if (0 == strcmp("--add-signature", *iter)) {
+            cf.signature = 1;
+        } else if (0 == strcmp("--remove-signature", *iter)) {
+            cf.signature = -1;
+        } else if (**iter == '-' && (*iter)[1]) {
+            usage(pname, 1);
+        } else {
+            // move a non-option up in argv[]
+            *remainArgvLimit++ = *iter;
+        }
+    }
+
+    if (printConvs || printName) {
+        return printConverters(pname, printName, printCanon) ? 2 : 0;
+    } else if (printTranslits) {
+        return printTransliterators(printCanon) ? 3 : 0;
+    }
+
+    if (!fromcpage || !uprv_strcmp(fromcpage, "-")) {
+        fromcpage = ucnv_getDefaultName();
+    }
+    if (!tocpage || !uprv_strcmp(tocpage, "-")) {
+        tocpage = ucnv_getDefaultName();
+    }
+
+    // Open the correct output file or connect to stdout for reading input
+    if (outfilestr != 0 && strcmp(outfilestr, "-")) {
+        outfile = fopen(outfilestr, "wb");
+        if (outfile == 0) {
+            UnicodeString str1(outfilestr, "");
+            UnicodeString str2(strerror(errno), "");
+            initMsg(pname);
+            u_wmsg(stderr, "cantCreateOutputF",
+                str1.getBuffer(), str2.getBuffer());
+            return 1;
+        }
+    } else {
+        outfilestr = "-";
+        outfile = stdout;
+#ifdef USE_FILENO_BINARY_MODE
+        if (setmode(fileno(outfile), O_BINARY) == -1) {
+            u_wmsg(stderr, "cantSetOutBinMode");
+            exit(-1);
+        }
+#endif
+    }
+
+    /* Loop again on the arguments to find all the input files, and
+    convert them. */
+
+    cf.setBufferSize(bufsz);
+
+    if(remainArgv < remainArgvLimit) {
+        for (iter = remainArgv; iter != remainArgvLimit; iter++) {
+            if (!cf.convertFile(
+                    pname, fromcpage, toucallback, touctxt, tocpage,
+                    fromucallback, fromuctxt, fallback, translit, *iter,
+                    outfile, verbose)
+            ) {
+                goto error_exit;
+            }
+        }
+    } else {
+        if (!cf.convertFile(
+                pname, fromcpage, toucallback, touctxt, tocpage,
+                fromucallback, fromuctxt, fallback, translit, 0,
+                outfile, verbose)
+        ) {
+            goto error_exit;
+        }
+    }
+
+    goto normal_exit;
+error_exit:
+#if !UCONFIG_NO_LEGACY_CONVERSION
+    ret = 1;
+#else 
+    fprintf(stderr, "uconv error: UCONFIG_NO_LEGACY_CONVERSION is on. See uconfig.h\n");
+#endif
+normal_exit:
+
+    if (outfile != stdout) {
+        fclose(outfile);
+    }
+
+    u_cleanup();
+
+    return ret;
+}
+
+
+/*
+ * Hey, Emacs, please set the following:
+ *
+ * Local Variables:
+ * indent-tabs-mode: nil
+ * End:
+ *
+ */