summaryrefslogtreecommitdiffstats
path: root/intl/icu/source/extra/uconv/uconv.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'intl/icu/source/extra/uconv/uconv.cpp')
-rw-r--r--intl/icu/source/extra/uconv/uconv.cpp1397
1 files changed, 1397 insertions, 0 deletions
diff --git a/intl/icu/source/extra/uconv/uconv.cpp b/intl/icu/source/extra/uconv/uconv.cpp
new file mode 100644
index 0000000000..0f4af65663
--- /dev/null
+++ b/intl/icu/source/extra/uconv/uconv.cpp
@@ -0,0 +1,1397 @@
+// © 2016 and later: Unicode, Inc. and others.
+// License & terms of use: http://www.unicode.org/copyright.html
+/*****************************************************************************
+*
+* Copyright (C) 1999-2016, International Business Machines
+* Corporation and others. All Rights Reserved.
+*
+******************************************************************************/
+
+/*
+ * uconv(1): an iconv(1)-like converter using ICU.
+ *
+ * Original code by Jonas Utterstr&#x00F6;m <jonas.utterstrom@vittran.norrnod.se>
+ * contributed in 1999.
+ *
+ * Conversion to the C conversion API and many improvements by
+ * Yves Arrouye <yves@realnames.com>, current maintainer.
+ *
+ * Markus Scherer maintainer from 2003.
+ * See source code repository history for changes.
+ */
+
+#include <unicode/utypes.h>
+#include <unicode/putil.h>
+#include <unicode/ucnv.h>
+#include <unicode/uenum.h>
+#include <unicode/unistr.h>
+#include <unicode/translit.h>
+#include <unicode/uset.h>
+#include <unicode/uclean.h>
+#include <unicode/utf16.h>
+
+#include <stdio.h>
+#include <errno.h>
+#include <string.h>
+#include <stdlib.h>
+
+#include "cmemory.h"
+#include "cstring.h"
+#include "ustrfmt.h"
+
+#include "unicode/uwmsg.h"
+
+U_NAMESPACE_USE
+
+#if U_PLATFORM_USES_ONLY_WIN32_API && !defined(__STRICT_ANSI__)
+#include <io.h>
+#include <fcntl.h>
+#if U_PLATFORM_USES_ONLY_WIN32_API
+#define USE_FILENO_BINARY_MODE 1
+/* Windows likes to rename Unix-like functions */
+#ifndef fileno
+#define fileno _fileno
+#endif
+#ifndef setmode
+#define setmode _setmode
+#endif
+#ifndef O_BINARY
+#define O_BINARY _O_BINARY
+#endif
+#endif
+#endif
+
+#ifdef UCONVMSG_LINK
+/* below from the README */
+#include "unicode/utypes.h"
+#include "unicode/udata.h"
+U_CFUNC char uconvmsg_dat[];
+#endif
+
+#define DEFAULT_BUFSZ 4096
+#define UCONVMSG "uconvmsg"
+
+static UResourceBundle *gBundle = 0; /* Bundle containing messages. */
+
+/*
+ * Initialize the message bundle so that message strings can be fetched
+ * by u_wmsg().
+ *
+ */
+
+static void initMsg(const char *pname) {
+ static int ps = 0;
+
+ if (!ps) {
+ char dataPath[2048]; /* XXX Sloppy: should be PATH_MAX. */
+ UErrorCode err = U_ZERO_ERROR;
+
+ ps = 1;
+
+ /* Set up our static data - if any */
+#if defined(UCONVMSG_LINK) && U_PLATFORM != U_PF_OS390 /* On z/OS, this is failing. */
+ udata_setAppData(UCONVMSG, (const void*) uconvmsg_dat, &err);
+ if (U_FAILURE(err)) {
+ fprintf(stderr, "%s: warning, problem installing our static resource bundle data uconvmsg: %s - trying anyways.\n",
+ pname, u_errorName(err));
+ err = U_ZERO_ERROR; /* It may still fail */
+ }
+#endif
+
+ /* Get messages. */
+ gBundle = u_wmsg_setPath(UCONVMSG, &err);
+ if (U_FAILURE(err)) {
+ fprintf(stderr,
+ "%s: warning: couldn't open bundle %s: %s\n",
+ pname, UCONVMSG, u_errorName(err));
+#ifdef UCONVMSG_LINK
+ fprintf(stderr,
+ "%s: setAppData was called, internal data %s failed to load\n",
+ pname, UCONVMSG);
+#endif
+
+ err = U_ZERO_ERROR;
+ /* that was try #1, try again with a path */
+ uprv_strcpy(dataPath, u_getDataDirectory());
+ uprv_strcat(dataPath, U_FILE_SEP_STRING);
+ uprv_strcat(dataPath, UCONVMSG);
+
+ gBundle = u_wmsg_setPath(dataPath, &err);
+ if (U_FAILURE(err)) {
+ fprintf(stderr,
+ "%s: warning: still couldn't open bundle %s: %s\n",
+ pname, dataPath, u_errorName(err));
+ fprintf(stderr, "%s: warning: messages will not be displayed\n", pname);
+ }
+ }
+ }
+}
+
+/* Mapping of callback names to the callbacks passed to the converter
+ API. */
+
+static struct callback_ent {
+ const char *name;
+ UConverterFromUCallback fromu;
+ const void *fromuctxt;
+ UConverterToUCallback tou;
+ const void *touctxt;
+} transcode_callbacks[] = {
+ { "substitute",
+ UCNV_FROM_U_CALLBACK_SUBSTITUTE, 0,
+ UCNV_TO_U_CALLBACK_SUBSTITUTE, 0 },
+ { "skip",
+ UCNV_FROM_U_CALLBACK_SKIP, 0,
+ UCNV_TO_U_CALLBACK_SKIP, 0 },
+ { "stop",
+ UCNV_FROM_U_CALLBACK_STOP, 0,
+ UCNV_TO_U_CALLBACK_STOP, 0 },
+ { "escape",
+ UCNV_FROM_U_CALLBACK_ESCAPE, 0,
+ UCNV_TO_U_CALLBACK_ESCAPE, 0},
+ { "escape-icu",
+ UCNV_FROM_U_CALLBACK_ESCAPE, UCNV_ESCAPE_ICU,
+ UCNV_TO_U_CALLBACK_ESCAPE, UCNV_ESCAPE_ICU },
+ { "escape-java",
+ UCNV_FROM_U_CALLBACK_ESCAPE, UCNV_ESCAPE_JAVA,
+ UCNV_TO_U_CALLBACK_ESCAPE, UCNV_ESCAPE_JAVA },
+ { "escape-c",
+ UCNV_FROM_U_CALLBACK_ESCAPE, UCNV_ESCAPE_C,
+ UCNV_TO_U_CALLBACK_ESCAPE, UCNV_ESCAPE_C },
+ { "escape-xml",
+ UCNV_FROM_U_CALLBACK_ESCAPE, UCNV_ESCAPE_XML_HEX,
+ UCNV_TO_U_CALLBACK_ESCAPE, UCNV_ESCAPE_XML_HEX },
+ { "escape-xml-hex",
+ UCNV_FROM_U_CALLBACK_ESCAPE, UCNV_ESCAPE_XML_HEX,
+ UCNV_TO_U_CALLBACK_ESCAPE, UCNV_ESCAPE_XML_HEX },
+ { "escape-xml-dec",
+ UCNV_FROM_U_CALLBACK_ESCAPE, UCNV_ESCAPE_XML_DEC,
+ UCNV_TO_U_CALLBACK_ESCAPE, UCNV_ESCAPE_XML_DEC },
+ { "escape-unicode", UCNV_FROM_U_CALLBACK_ESCAPE, UCNV_ESCAPE_UNICODE,
+ UCNV_TO_U_CALLBACK_ESCAPE, UCNV_ESCAPE_UNICODE }
+};
+
+/* Return a pointer to a callback record given its name. */
+
+static const struct callback_ent *findCallback(const char *name) {
+ int i, count =
+ UPRV_LENGTHOF(transcode_callbacks);
+
+ /* We'll do a linear search, there aren't many of them and bsearch()
+ may not be that portable. */
+
+ for (i = 0; i < count; ++i) {
+ if (!uprv_stricmp(name, transcode_callbacks[i].name)) {
+ return &transcode_callbacks[i];
+ }
+ }
+
+ return 0;
+}
+
+/* Print converter information. If lookfor is set, only that converter will
+ be printed, otherwise all converters will be printed. If canon is non
+ zero, tags and aliases for each converter are printed too, in the format
+ expected for convrters.txt(5). */
+
+static int printConverters(const char *pname, const char *lookfor,
+ UBool canon)
+{
+ UErrorCode err = U_ZERO_ERROR;
+ int32_t num;
+ uint16_t num_stds;
+ const char **stds;
+
+ /* If there is a specified name, just handle that now. */
+
+ if (lookfor) {
+ if (!canon) {
+ printf("%s\n", lookfor);
+ return 0;
+ } else {
+ /* Because we are printing a canonical name, we need the
+ true converter name. We've done that already except for
+ the default name (because we want to print the exact
+ name one would get when calling ucnv_getDefaultName()
+ in non-canon mode). But since we do not know at this
+ point if we have the default name or something else, we
+ need to normalize again to the canonical converter
+ name. */
+
+ const char *truename = ucnv_getAlias(lookfor, 0, &err);
+ if (U_SUCCESS(err)) {
+ lookfor = truename;
+ } else {
+ err = U_ZERO_ERROR;
+ }
+ }
+ }
+
+ /* Print converter names. We come here for one of two reasons: we
+ are printing all the names (lookfor was null), or we have a
+ single converter to print but in canon mode, hence we need to
+ get to it in order to print everything. */
+
+ num = ucnv_countAvailable();
+ if (num <= 0) {
+ initMsg(pname);
+ u_wmsg(stderr, "cantGetNames");
+ return -1;
+ }
+ if (lookfor) {
+ num = 1; /* We know where we want to be. */
+ }
+
+ num_stds = ucnv_countStandards();
+ stds = (const char **) uprv_malloc(num_stds * sizeof(*stds));
+ if (!stds) {
+ u_wmsg(stderr, "cantGetTag", u_wmsg_errorName(U_MEMORY_ALLOCATION_ERROR));
+ return -1;
+ } else {
+ uint16_t s;
+
+ if (canon) {
+ printf("{ ");
+ }
+ for (s = 0; s < num_stds; ++s) {
+ stds[s] = ucnv_getStandard(s, &err);
+ if (canon) {
+ printf("%s ", stds[s]);
+ }
+ if (U_FAILURE(err)) {
+ u_wmsg(stderr, "cantGetTag", u_wmsg_errorName(err));
+ goto error_cleanup;
+ }
+ }
+ if (canon) {
+ puts("}");
+ }
+ }
+
+ for (int32_t i = 0; i < num; i++) {
+ const char *name;
+ uint16_t num_aliases;
+
+ /* Set the name either to what we are looking for, or
+ to the current converter name. */
+
+ if (lookfor) {
+ name = lookfor;
+ } else {
+ name = ucnv_getAvailableName(i);
+ }
+
+ /* Get all the aliases associated to the name. */
+
+ err = U_ZERO_ERROR;
+ num_aliases = ucnv_countAliases(name, &err);
+ if (U_FAILURE(err)) {
+ printf("%s", name);
+
+ UnicodeString str(name, "");
+ putchar('\t');
+ u_wmsg(stderr, "cantGetAliases", str.getTerminatedBuffer(),
+ u_wmsg_errorName(err));
+ goto error_cleanup;
+ } else {
+ uint16_t a, s, t;
+
+ /* Write all the aliases and their tags. */
+
+ for (a = 0; a < num_aliases; ++a) {
+ const char *alias = ucnv_getAlias(name, a, &err);
+
+ if (U_FAILURE(err)) {
+ UnicodeString str(name, "");
+ putchar('\t');
+ u_wmsg(stderr, "cantGetAliases", str.getTerminatedBuffer(),
+ u_wmsg_errorName(err));
+ goto error_cleanup;
+ }
+
+ /* Print the current alias so that it looks right. */
+ printf("%s%s%s", (canon ? (a == 0? "" : "\t" ) : "") ,
+ alias,
+ (canon ? "" : " "));
+
+ /* Look (slowly, linear searching) for a tag. */
+
+ if (canon) {
+ /* -1 to skip the last standard */
+ for (s = t = 0; s < num_stds-1; ++s) {
+ UEnumeration *nameEnum = ucnv_openStandardNames(name, stds[s], &err);
+ if (U_SUCCESS(err)) {
+ /* List the standard tags */
+ const char *standardName;
+ UBool isFirst = true;
+ UErrorCode enumError = U_ZERO_ERROR;
+ while ((standardName = uenum_next(nameEnum, nullptr, &enumError))) {
+ /* See if this alias is supported by this standard. */
+ if (!strcmp(standardName, alias)) {
+ if (!t) {
+ printf(" {");
+ t = 1;
+ }
+ /* Print a * after the default standard name */
+ printf(" %s%s", stds[s], (isFirst ? "*" : ""));
+ }
+ isFirst = false;
+ }
+ }
+ }
+ if (t) {
+ printf(" }");
+ }
+ }
+ /* Terminate this entry. */
+ if (canon) {
+ puts("");
+ }
+
+ /* Move on. */
+ }
+ /* Terminate this entry. */
+ if (!canon) {
+ puts("");
+ }
+ }
+ }
+
+ /* Free temporary data. */
+
+ uprv_free(stds);
+
+ /* Success. */
+
+ return 0;
+error_cleanup:
+ uprv_free(stds);
+ return -1;
+}
+
+/* Print all available transliterators. If canon is non zero, print
+ one transliterator per line. */
+
+static int printTransliterators(UBool canon)
+{
+#if UCONFIG_NO_TRANSLITERATION
+ printf("no transliterators available because of UCONFIG_NO_TRANSLITERATION, see uconfig.h\n");
+ return 1;
+#else
+ UErrorCode status = U_ZERO_ERROR;
+ UEnumeration *ids = utrans_openIDs(&status);
+ int32_t i, numtrans = uenum_count(ids, &status);
+
+ char sepchar = canon ? '\n' : ' ';
+
+ for (i = 0; U_SUCCESS(status)&& (i < numtrans); ++i) {
+ int32_t len;
+ const char *nextTrans = uenum_next(ids, &len, &status);
+
+ printf("%s", nextTrans);
+ if (i < numtrans - 1) {
+ putchar(sepchar);
+ }
+ }
+
+ uenum_close(ids);
+
+ /* Add a terminating newline if needed. */
+
+ if (sepchar != '\n') {
+ putchar('\n');
+ }
+
+ /* Success. */
+
+ return 0;
+#endif
+}
+
+enum {
+ uSP = 0x20, // space
+ uCR = 0xd, // carriage return
+ uLF = 0xa, // line feed
+ uNL = 0x85, // newline
+ uLS = 0x2028, // line separator
+ uPS = 0x2029, // paragraph separator
+ uSig = 0xfeff // signature/BOM character
+};
+
+static inline int32_t
+getChunkLimit(const UnicodeString &prev, const UnicodeString &s) {
+ // find one of
+ // CR, LF, CRLF, NL, LS, PS
+ // for paragraph ends (see UAX #13/Unicode 4)
+ // and include it in the chunk
+ // all of these characters are on the BMP
+ // do not include FF or VT in case they are part of a paragraph
+ // (important for bidi contexts)
+ static const char16_t paraEnds[] = {
+ 0xd, 0xa, 0x85, 0x2028, 0x2029
+ };
+ enum {
+ iCR, iLF, iNL, iLS, iPS, iCount
+ };
+
+ // first, see if there is a CRLF split between prev and s
+ if (prev.endsWith(paraEnds + iCR, 1)) {
+ if (s.startsWith(paraEnds + iLF, 1)) {
+ return 1; // split CRLF, include the LF
+ } else if (!s.isEmpty()) {
+ return 0; // complete the last chunk
+ } else {
+ return -1; // wait for actual further contents to arrive
+ }
+ }
+
+ const char16_t *u = s.getBuffer(), *limit = u + s.length();
+ char16_t c;
+
+ while (u < limit) {
+ c = *u++;
+ if (
+ ((c < uSP) && (c == uCR || c == uLF)) ||
+ (c == uNL) ||
+ ((c & uLS) == uLS)
+ ) {
+ if (c == uCR) {
+ // check for CRLF
+ if (u == limit) {
+ return -1; // LF may be in the next chunk
+ } else if (*u == uLF) {
+ ++u; // include the LF in this chunk
+ }
+ }
+ return (int32_t)(u - s.getBuffer());
+ }
+ }
+
+ return -1; // continue collecting the chunk
+}
+
+enum {
+ CNV_NO_FEFF, // cannot convert the U+FEFF Unicode signature character (BOM)
+ CNV_WITH_FEFF, // can convert the U+FEFF signature character
+ CNV_ADDS_FEFF // automatically adds/detects the U+FEFF signature character
+};
+
+static inline char16_t
+nibbleToHex(uint8_t n) {
+ n &= 0xf;
+ return
+ n <= 9 ?
+ (char16_t)(0x30 + n) :
+ (char16_t)((0x61 - 10) + n);
+}
+
+// check the converter's Unicode signature properties;
+// the fromUnicode side of the converter must be in its initial state
+// and will be reset again if it was used
+static int32_t
+cnvSigType(UConverter *cnv) {
+ UErrorCode err;
+ int32_t result;
+
+ // test if the output charset can convert U+FEFF
+ USet *set = uset_open(1, 0);
+ err = U_ZERO_ERROR;
+ ucnv_getUnicodeSet(cnv, set, UCNV_ROUNDTRIP_SET, &err);
+ if (U_SUCCESS(err) && uset_contains(set, uSig)) {
+ result = CNV_WITH_FEFF;
+ } else {
+ result = CNV_NO_FEFF; // an error occurred or U+FEFF cannot be converted
+ }
+ uset_close(set);
+
+ if (result == CNV_WITH_FEFF) {
+ // test if the output charset emits a signature anyway
+ const char16_t a[1] = { 0x61 }; // "a"
+ const char16_t *in;
+
+ char buffer[20];
+ char *out;
+
+ in = a;
+ out = buffer;
+ err = U_ZERO_ERROR;
+ ucnv_fromUnicode(cnv,
+ &out, buffer + sizeof(buffer),
+ &in, a + 1,
+ nullptr, true, &err);
+ ucnv_resetFromUnicode(cnv);
+
+ if (nullptr != ucnv_detectUnicodeSignature(buffer, (int32_t)(out - buffer), nullptr, &err) &&
+ U_SUCCESS(err)
+ ) {
+ result = CNV_ADDS_FEFF;
+ }
+ }
+
+ return result;
+}
+
+class ConvertFile {
+public:
+ ConvertFile() :
+ buf(nullptr), outbuf(nullptr), fromoffsets(nullptr),
+ bufsz(0), signature(0) {}
+
+ void
+ setBufferSize(size_t bufferSize) {
+ bufsz = bufferSize;
+
+ buf = new char[2 * bufsz];
+ outbuf = buf + bufsz;
+
+ // +1 for an added U+FEFF in the intermediate Unicode buffer
+ fromoffsets = new int32_t[bufsz + 1];
+ }
+
+ ~ConvertFile() {
+ delete [] buf;
+ delete [] fromoffsets;
+ }
+
+ UBool convertFile(const char *pname,
+ const char *fromcpage,
+ UConverterToUCallback toucallback,
+ const void *touctxt,
+ const char *tocpage,
+ UConverterFromUCallback fromucallback,
+ const void *fromuctxt,
+ UBool fallback,
+ const char *translit,
+ const char *infilestr,
+ FILE * outfile, int verbose);
+private:
+ friend int main(int argc, char **argv);
+
+ char *buf, *outbuf;
+ int32_t *fromoffsets;
+
+ size_t bufsz;
+ int8_t signature; // add (1) or remove (-1) a U+FEFF Unicode signature character
+};
+
+// Convert a file from one encoding to another
+UBool
+ConvertFile::convertFile(const char *pname,
+ const char *fromcpage,
+ UConverterToUCallback toucallback,
+ const void *touctxt,
+ const char *tocpage,
+ UConverterFromUCallback fromucallback,
+ const void *fromuctxt,
+ UBool fallback,
+ const char *translit,
+ const char *infilestr,
+ FILE * outfile, int verbose)
+{
+ FILE *infile;
+ UBool ret = true;
+ UConverter *convfrom = 0;
+ UConverter *convto = 0;
+ UErrorCode err = U_ZERO_ERROR;
+ UBool flush;
+ UBool closeFile = false;
+ const char *cbufp, *prevbufp;
+ char *bufp;
+
+ uint32_t infoffset = 0, outfoffset = 0; /* Where we are in the file, for error reporting. */
+
+ const char16_t *unibuf, *unibufbp;
+ char16_t *unibufp;
+
+ size_t rd, wr;
+
+#if !UCONFIG_NO_TRANSLITERATION
+ Transliterator *t = 0; // Transliterator acting on Unicode data.
+ UnicodeString chunk; // One chunk of the text being collected for transformation.
+#endif
+ UnicodeString u; // String to do the transliteration.
+ int32_t ulen;
+
+ // use conversion offsets for error messages
+ // unless a transliterator is used -
+ // a text transformation will reorder characters in unpredictable ways
+ UBool useOffsets = true;
+
+ // Open the correct input file or connect to stdin for reading input
+
+ if (infilestr != 0 && strcmp(infilestr, "-")) {
+ infile = fopen(infilestr, "rb");
+ if (infile == 0) {
+ UnicodeString str1(infilestr, "");
+ str1.append((UChar32) 0);
+ UnicodeString str2(strerror(errno), "");
+ str2.append((UChar32) 0);
+ initMsg(pname);
+ u_wmsg(stderr, "cantOpenInputF", str1.getBuffer(), str2.getBuffer());
+ return false;
+ }
+ closeFile = true;
+ } else {
+ infilestr = "-";
+ infile = stdin;
+#ifdef USE_FILENO_BINARY_MODE
+ if (setmode(fileno(stdin), O_BINARY) == -1) {
+ initMsg(pname);
+ u_wmsg(stderr, "cantSetInBinMode");
+ return false;
+ }
+#endif
+ }
+
+ if (verbose) {
+ fprintf(stderr, "%s:\n", infilestr);
+ }
+
+#if !UCONFIG_NO_TRANSLITERATION
+ // Create transliterator as needed.
+
+ if (translit != nullptr && *translit) {
+ UParseError parse;
+ UnicodeString str(translit), pestr;
+
+ /* Create from rules or by ID as needed. */
+
+ parse.line = -1;
+
+ if (uprv_strchr(translit, ':') || uprv_strchr(translit, '>') || uprv_strchr(translit, '<') || uprv_strchr(translit, '>')) {
+ t = Transliterator::createFromRules(UNICODE_STRING_SIMPLE("Uconv"), str, UTRANS_FORWARD, parse, err);
+ } else {
+ t = Transliterator::createInstance(UnicodeString(translit, -1, US_INV), UTRANS_FORWARD, err);
+ }
+
+ if (U_FAILURE(err)) {
+ str.append((UChar32) 0);
+ initMsg(pname);
+
+ if (parse.line >= 0) {
+ char16_t linebuf[20], offsetbuf[20];
+ uprv_itou(linebuf, 20, parse.line, 10, 0);
+ uprv_itou(offsetbuf, 20, parse.offset, 10, 0);
+ u_wmsg(stderr, "cantCreateTranslitParseErr", str.getTerminatedBuffer(),
+ u_wmsg_errorName(err), linebuf, offsetbuf);
+ } else {
+ u_wmsg(stderr, "cantCreateTranslit", str.getTerminatedBuffer(),
+ u_wmsg_errorName(err));
+ }
+
+ if (t) {
+ delete t;
+ t = 0;
+ }
+ goto error_exit;
+ }
+
+ useOffsets = false;
+ }
+#endif
+
+ // Create codepage converter. If the codepage or its aliases weren't
+ // available, it returns nullptr and a failure code. We also set the
+ // callbacks, and return errors in the same way.
+
+ convfrom = ucnv_open(fromcpage, &err);
+ if (U_FAILURE(err)) {
+ UnicodeString str(fromcpage, "");
+ initMsg(pname);
+ u_wmsg(stderr, "cantOpenFromCodeset", str.getTerminatedBuffer(),
+ u_wmsg_errorName(err));
+ goto error_exit;
+ }
+ ucnv_setToUCallBack(convfrom, toucallback, touctxt, 0, 0, &err);
+ if (U_FAILURE(err)) {
+ initMsg(pname);
+ u_wmsg(stderr, "cantSetCallback", u_wmsg_errorName(err));
+ goto error_exit;
+ }
+
+ convto = ucnv_open(tocpage, &err);
+ if (U_FAILURE(err)) {
+ UnicodeString str(tocpage, "");
+ initMsg(pname);
+ u_wmsg(stderr, "cantOpenToCodeset", str.getTerminatedBuffer(),
+ u_wmsg_errorName(err));
+ goto error_exit;
+ }
+ ucnv_setFromUCallBack(convto, fromucallback, fromuctxt, 0, 0, &err);
+ if (U_FAILURE(err)) {
+ initMsg(pname);
+ u_wmsg(stderr, "cantSetCallback", u_wmsg_errorName(err));
+ goto error_exit;
+ }
+ ucnv_setFallback(convto, fallback);
+
+ UBool willexit, fromSawEndOfBytes, toSawEndOfUnicode;
+ int8_t sig;
+
+ // OK, we can convert now.
+ sig = signature;
+ rd = 0;
+
+ do {
+ willexit = false;
+
+ // input file offset at the beginning of the next buffer
+ infoffset += static_cast<uint32_t>(rd);
+
+ rd = fread(buf, 1, bufsz, infile);
+ if (ferror(infile) != 0) {
+ UnicodeString str(strerror(errno));
+ initMsg(pname);
+ u_wmsg(stderr, "cantRead", str.getTerminatedBuffer());
+ goto error_exit;
+ }
+
+ // Convert the read buffer into the new encoding via Unicode.
+ // After the call 'unibufp' will be placed behind the last
+ // character that was converted in the 'unibuf'.
+ // Also the 'cbufp' is positioned behind the last converted
+ // character.
+ // At the last conversion in the file, flush should be set to
+ // true so that we get all characters converted.
+ //
+ // The converter must be flushed at the end of conversion so
+ // that characters on hold also will be written.
+
+ cbufp = buf;
+ flush = (UBool)(rd != bufsz);
+
+ // convert until the input is consumed
+ do {
+ // remember the start of the current byte-to-Unicode conversion
+ prevbufp = cbufp;
+
+ unibuf = unibufp = u.getBuffer((int32_t)bufsz);
+
+ // Use bufsz instead of u.getCapacity() for the targetLimit
+ // so that we don't overflow fromoffsets[].
+ ucnv_toUnicode(convfrom, &unibufp, unibuf + bufsz, &cbufp,
+ buf + rd, useOffsets ? fromoffsets : nullptr, flush, &err);
+
+ ulen = (int32_t)(unibufp - unibuf);
+ u.releaseBuffer(U_SUCCESS(err) ? ulen : 0);
+
+ // fromSawEndOfBytes indicates that ucnv_toUnicode() is done
+ // converting all of the input bytes.
+ // It works like this because ucnv_toUnicode() returns only under the
+ // following conditions:
+ // - an error occurred during conversion (an error code is set)
+ // - the target buffer is filled (the error code indicates an overflow)
+ // - the source is consumed
+ // That is, if the error code does not indicate a failure,
+ // not even an overflow, then the source must be consumed entirely.
+ fromSawEndOfBytes = (UBool)U_SUCCESS(err);
+
+ if (err == U_BUFFER_OVERFLOW_ERROR) {
+ err = U_ZERO_ERROR;
+ } else if (U_FAILURE(err)) {
+ char pos[32], errorBytes[32];
+ int8_t i, length, errorLength;
+
+ UErrorCode localError = U_ZERO_ERROR;
+ errorLength = (int8_t)sizeof(errorBytes);
+ ucnv_getInvalidChars(convfrom, errorBytes, &errorLength, &localError);
+ if (U_FAILURE(localError) || errorLength == 0) {
+ errorLength = 1;
+ }
+
+ // print the input file offset of the start of the error bytes:
+ // input file offset of the current byte buffer +
+ // length of the just consumed bytes -
+ // length of the error bytes
+ length =
+ (int8_t)snprintf(pos, sizeof(pos), "%d",
+ (int)(infoffset + (cbufp - buf) - errorLength));
+
+ // output the bytes that caused the error
+ UnicodeString str;
+ for (i = 0; i < errorLength; ++i) {
+ if (i > 0) {
+ str.append((char16_t)uSP);
+ }
+ str.append(nibbleToHex((uint8_t)errorBytes[i] >> 4));
+ str.append(nibbleToHex((uint8_t)errorBytes[i]));
+ }
+
+ initMsg(pname);
+ u_wmsg(stderr, "problemCvtToU",
+ UnicodeString(pos, length, "").getTerminatedBuffer(),
+ str.getTerminatedBuffer(),
+ u_wmsg_errorName(err));
+
+ willexit = true;
+ err = U_ZERO_ERROR; /* reset the error for the rest of the conversion. */
+ }
+
+ // Replaced a check for whether the input was consumed by
+ // looping until it is; message key "premEndInput" now obsolete.
+
+ if (ulen == 0) {
+ continue;
+ }
+
+ // remove a U+FEFF Unicode signature character if requested
+ if (sig < 0) {
+ if (u.charAt(0) == uSig) {
+ u.remove(0, 1);
+
+ // account for the removed char16_t and offset
+ --ulen;
+
+ if (useOffsets) {
+ // remove an offset from fromoffsets[] as well
+ // to keep the array parallel with the UChars
+ memmove(fromoffsets, fromoffsets + 1, ulen * 4);
+ }
+
+ }
+ sig = 0;
+ }
+
+#if !UCONFIG_NO_TRANSLITERATION
+ // Transliterate/transform if needed.
+
+ // For transformation, we use chunking code -
+ // collect Unicode input until, for example, an end-of-line,
+ // then transform and output-convert that and continue collecting.
+ // This makes the transformation result independent of the buffer size
+ // while avoiding the slower keyboard mode.
+ // The end-of-chunk characters are completely included in the
+ // transformed string in case they are to be transformed themselves.
+ if (t != nullptr) {
+ UnicodeString out;
+ int32_t chunkLimit;
+
+ do {
+ chunkLimit = getChunkLimit(chunk, u);
+ if (chunkLimit < 0 && flush && fromSawEndOfBytes) {
+ // use all of the rest at the end of the text
+ chunkLimit = u.length();
+ }
+ if (chunkLimit >= 0) {
+ // complete the chunk and transform it
+ chunk.append(u, 0, chunkLimit);
+ u.remove(0, chunkLimit);
+ t->transliterate(chunk);
+
+ // append the transformation result to the result and empty the chunk
+ out.append(chunk);
+ chunk.remove();
+ } else {
+ // continue collecting the chunk
+ chunk.append(u);
+ break;
+ }
+ } while (!u.isEmpty());
+
+ u = out;
+ ulen = u.length();
+ }
+#endif
+
+ // add a U+FEFF Unicode signature character if requested
+ // and possible/necessary
+ if (sig > 0) {
+ if (u.charAt(0) != uSig && cnvSigType(convto) == CNV_WITH_FEFF) {
+ u.insert(0, (char16_t)uSig);
+
+ if (useOffsets) {
+ // insert a pseudo-offset into fromoffsets[] as well
+ // to keep the array parallel with the UChars
+ memmove(fromoffsets + 1, fromoffsets, ulen * 4);
+ fromoffsets[0] = -1;
+ }
+
+ // account for the additional char16_t and offset
+ ++ulen;
+ }
+ sig = 0;
+ }
+
+ // Convert the Unicode buffer into the destination codepage
+ // Again 'bufp' will be placed behind the last converted character
+ // And 'unibufp' will be placed behind the last converted unicode character
+ // At the last conversion flush should be set to true to ensure that
+ // all characters left get converted
+
+ unibuf = unibufbp = u.getBuffer();
+
+ do {
+ bufp = outbuf;
+
+ // Use fromSawEndOfBytes in addition to the flush flag -
+ // it indicates whether the intermediate Unicode string
+ // contains the very last UChars for the very last input bytes.
+ ucnv_fromUnicode(convto, &bufp, outbuf + bufsz,
+ &unibufbp,
+ unibuf + ulen,
+ nullptr, (UBool)(flush && fromSawEndOfBytes), &err);
+
+ // toSawEndOfUnicode indicates that ucnv_fromUnicode() is done
+ // converting all of the intermediate UChars.
+ // See comment for fromSawEndOfBytes.
+ toSawEndOfUnicode = (UBool)U_SUCCESS(err);
+
+ if (err == U_BUFFER_OVERFLOW_ERROR) {
+ err = U_ZERO_ERROR;
+ } else if (U_FAILURE(err)) {
+ char16_t errorUChars[4];
+ const char *errtag;
+ char pos[32];
+ UChar32 c;
+ int8_t i, length, errorLength;
+
+ UErrorCode localError = U_ZERO_ERROR;
+ errorLength = UPRV_LENGTHOF(errorUChars);
+ ucnv_getInvalidUChars(convto, errorUChars, &errorLength, &localError);
+ if (U_FAILURE(localError) || errorLength == 0) {
+ // need at least 1 so that we don't access beyond the length of fromoffsets[]
+ errorLength = 1;
+ }
+
+ int32_t ferroffset;
+
+ if (useOffsets) {
+ // Unicode buffer offset of the start of the error UChars
+ ferroffset = (int32_t)((unibufbp - unibuf) - errorLength);
+ if (ferroffset < 0) {
+ // approximation - the character started in the previous Unicode buffer
+ ferroffset = 0;
+ }
+
+ // get the corresponding byte offset out of fromoffsets[]
+ // go back if the offset is not known for some of the UChars
+ int32_t fromoffset;
+ do {
+ fromoffset = fromoffsets[ferroffset];
+ } while (fromoffset < 0 && --ferroffset >= 0);
+
+ // total input file offset =
+ // input file offset of the current byte buffer +
+ // byte buffer offset of where the current Unicode buffer is converted from +
+ // fromoffsets[Unicode offset]
+ ferroffset = static_cast<int32_t>(infoffset + (prevbufp - buf) + fromoffset);
+ errtag = "problemCvtFromU";
+ } else {
+ // Do not use fromoffsets if (t != nullptr) because the Unicode text may
+ // be different from what the offsets refer to.
+
+ // output file offset
+ ferroffset = (int32_t)(outfoffset + (bufp - outbuf));
+ errtag = "problemCvtFromUOut";
+ }
+
+ length = (int8_t)snprintf(pos, sizeof(pos), "%u", (int)ferroffset);
+
+ // output the code points that caused the error
+ UnicodeString str;
+ for (i = 0; i < errorLength;) {
+ if (i > 0) {
+ str.append((char16_t)uSP);
+ }
+ U16_NEXT(errorUChars, i, errorLength, c);
+ if (c >= 0x100000) {
+ str.append(nibbleToHex((uint8_t)(c >> 20)));
+ }
+ if (c >= 0x10000) {
+ str.append(nibbleToHex((uint8_t)(c >> 16)));
+ }
+ str.append(nibbleToHex((uint8_t)(c >> 12)));
+ str.append(nibbleToHex((uint8_t)(c >> 8)));
+ str.append(nibbleToHex((uint8_t)(c >> 4)));
+ str.append(nibbleToHex((uint8_t)c));
+ }
+
+ initMsg(pname);
+ u_wmsg(stderr, errtag,
+ UnicodeString(pos, length, "").getTerminatedBuffer(),
+ str.getTerminatedBuffer(),
+ u_wmsg_errorName(err));
+ u_wmsg(stderr, "errorUnicode", str.getTerminatedBuffer());
+
+ willexit = true;
+ err = U_ZERO_ERROR; /* reset the error for the rest of the conversion. */
+ }
+
+ // Replaced a check for whether the intermediate Unicode characters were all consumed by
+ // looping until they are; message key "premEnd" now obsolete.
+
+ // Finally, write the converted buffer to the output file
+ size_t outlen = (size_t) (bufp - outbuf);
+ outfoffset += (int32_t)(wr = fwrite(outbuf, 1, outlen, outfile));
+ if (wr != outlen) {
+ UnicodeString str(strerror(errno));
+ initMsg(pname);
+ u_wmsg(stderr, "cantWrite", str.getTerminatedBuffer());
+ willexit = true;
+ }
+
+ if (willexit) {
+ goto error_exit;
+ }
+ } while (!toSawEndOfUnicode);
+ } while (!fromSawEndOfBytes);
+ } while (!flush); // Stop when we have flushed the
+ // converters (this means that it's
+ // the end of output)
+
+ goto normal_exit;
+
+error_exit:
+ ret = false;
+
+normal_exit:
+ // Cleanup.
+
+ ucnv_close(convfrom);
+ ucnv_close(convto);
+
+#if !UCONFIG_NO_TRANSLITERATION
+ delete t;
+#endif
+
+ if (closeFile) {
+ fclose(infile);
+ }
+
+ return ret;
+}
+
+static void usage(const char *pname, int ecode) {
+ const char16_t *msg;
+ int32_t msgLen;
+ UErrorCode err = U_ZERO_ERROR;
+ FILE *fp = ecode ? stderr : stdout;
+ int res;
+
+ initMsg(pname);
+ msg =
+ ures_getStringByKey(gBundle, ecode ? "lcUsageWord" : "ucUsageWord",
+ &msgLen, &err);
+ UnicodeString upname(pname, (int32_t)(uprv_strlen(pname) + 1));
+ UnicodeString mname(msg, msgLen + 1);
+
+ res = u_wmsg(fp, "usage", mname.getBuffer(), upname.getBuffer());
+ if (!ecode) {
+ if (!res) {
+ fputc('\n', fp);
+ }
+ if (!u_wmsg(fp, "help")) {
+ /* Now dump callbacks and finish. */
+
+ int i, count =
+ UPRV_LENGTHOF(transcode_callbacks);
+ for (i = 0; i < count; ++i) {
+ fprintf(fp, " %s", transcode_callbacks[i].name);
+ }
+ fputc('\n', fp);
+ }
+ }
+
+ exit(ecode);
+}
+
+extern int
+main(int argc, char **argv)
+{
+ FILE *outfile;
+ int ret = 0;
+
+ size_t bufsz = DEFAULT_BUFSZ;
+
+ const char *fromcpage = 0;
+ const char *tocpage = 0;
+ const char *translit = 0;
+ const char *outfilestr = 0;
+ UBool fallback = false;
+
+ UConverterFromUCallback fromucallback = UCNV_FROM_U_CALLBACK_STOP;
+ const void *fromuctxt = 0;
+ UConverterToUCallback toucallback = UCNV_TO_U_CALLBACK_STOP;
+ const void *touctxt = 0;
+
+ char **iter, **remainArgv, **remainArgvLimit;
+ char **end = argv + argc;
+
+ const char *pname;
+
+ UBool printConvs = false, printCanon = false, printTranslits = false;
+ const char *printName = 0;
+
+ UBool verbose = false;
+ UErrorCode status = U_ZERO_ERROR;
+
+ ConvertFile cf;
+
+ /* Initialize ICU */
+ u_init(&status);
+ if (U_FAILURE(status)) {
+ fprintf(stderr, "%s: can not initialize ICU. status = %s\n",
+ argv[0], u_errorName(status));
+ exit(1);
+ }
+
+ // Get and prettify pname.
+ pname = uprv_strrchr(*argv, U_FILE_SEP_CHAR);
+#if U_PLATFORM_USES_ONLY_WIN32_API
+ if (!pname) {
+ pname = uprv_strrchr(*argv, '/');
+ }
+#endif
+ if (!pname) {
+ pname = *argv;
+ } else {
+ ++pname;
+ }
+
+ // First, get the arguments from command-line
+ // to know the codepages to convert between
+
+ remainArgv = remainArgvLimit = argv + 1;
+ for (iter = argv + 1; iter != end; iter++) {
+ // Check for from charset
+ if (strcmp("-f", *iter) == 0 || !strcmp("--from-code", *iter)) {
+ iter++;
+ if (iter != end)
+ fromcpage = *iter;
+ else
+ usage(pname, 1);
+ } else if (strcmp("-t", *iter) == 0 || !strcmp("--to-code", *iter)) {
+ iter++;
+ if (iter != end)
+ tocpage = *iter;
+ else
+ usage(pname, 1);
+ } else if (strcmp("-x", *iter) == 0) {
+ iter++;
+ if (iter != end)
+ translit = *iter;
+ else
+ usage(pname, 1);
+ } else if (!strcmp("--fallback", *iter)) {
+ fallback = true;
+ } else if (!strcmp("--no-fallback", *iter)) {
+ fallback = false;
+ } else if (strcmp("-b", *iter) == 0 || !strcmp("--block-size", *iter)) {
+ iter++;
+ if (iter != end) {
+ bufsz = atoi(*iter);
+ if ((int) bufsz <= 0) {
+ initMsg(pname);
+ UnicodeString str(*iter);
+ initMsg(pname);
+ u_wmsg(stderr, "badBlockSize", str.getTerminatedBuffer());
+ return 3;
+ }
+ } else {
+ usage(pname, 1);
+ }
+ } else if (strcmp("-l", *iter) == 0 || !strcmp("--list", *iter)) {
+ if (printTranslits) {
+ usage(pname, 1);
+ }
+ printConvs = true;
+ } else if (strcmp("--default-code", *iter) == 0) {
+ if (printTranslits) {
+ usage(pname, 1);
+ }
+ printName = ucnv_getDefaultName();
+ } else if (strcmp("--list-code", *iter) == 0) {
+ if (printTranslits) {
+ usage(pname, 1);
+ }
+
+ iter++;
+ if (iter != end) {
+ UErrorCode e = U_ZERO_ERROR;
+ printName = ucnv_getAlias(*iter, 0, &e);
+ if (U_FAILURE(e) || !printName) {
+ UnicodeString str(*iter);
+ initMsg(pname);
+ u_wmsg(stderr, "noSuchCodeset", str.getTerminatedBuffer());
+ return 2;
+ }
+ } else
+ usage(pname, 1);
+ } else if (strcmp("--canon", *iter) == 0) {
+ printCanon = true;
+ } else if (strcmp("-L", *iter) == 0
+ || !strcmp("--list-transliterators", *iter)) {
+ if (printConvs) {
+ usage(pname, 1);
+ }
+ printTranslits = true;
+ } else if (strcmp("-h", *iter) == 0 || !strcmp("-?", *iter)
+ || !strcmp("--help", *iter)) {
+ usage(pname, 0);
+ } else if (!strcmp("-c", *iter)) {
+ fromucallback = UCNV_FROM_U_CALLBACK_SKIP;
+ } else if (!strcmp("--to-callback", *iter)) {
+ iter++;
+ if (iter != end) {
+ const struct callback_ent *cbe = findCallback(*iter);
+ if (cbe) {
+ fromucallback = cbe->fromu;
+ fromuctxt = cbe->fromuctxt;
+ } else {
+ UnicodeString str(*iter);
+ initMsg(pname);
+ u_wmsg(stderr, "unknownCallback", str.getTerminatedBuffer());
+ return 4;
+ }
+ } else {
+ usage(pname, 1);
+ }
+ } else if (!strcmp("--from-callback", *iter)) {
+ iter++;
+ if (iter != end) {
+ const struct callback_ent *cbe = findCallback(*iter);
+ if (cbe) {
+ toucallback = cbe->tou;
+ touctxt = cbe->touctxt;
+ } else {
+ UnicodeString str(*iter);
+ initMsg(pname);
+ u_wmsg(stderr, "unknownCallback", str.getTerminatedBuffer());
+ return 4;
+ }
+ } else {
+ usage(pname, 1);
+ }
+ } else if (!strcmp("-i", *iter)) {
+ toucallback = UCNV_TO_U_CALLBACK_SKIP;
+ } else if (!strcmp("--callback", *iter)) {
+ iter++;
+ if (iter != end) {
+ const struct callback_ent *cbe = findCallback(*iter);
+ if (cbe) {
+ fromucallback = cbe->fromu;
+ fromuctxt = cbe->fromuctxt;
+ toucallback = cbe->tou;
+ touctxt = cbe->touctxt;
+ } else {
+ UnicodeString str(*iter);
+ initMsg(pname);
+ u_wmsg(stderr, "unknownCallback", str.getTerminatedBuffer());
+ return 4;
+ }
+ } else {
+ usage(pname, 1);
+ }
+ } else if (!strcmp("-s", *iter) || !strcmp("--silent", *iter)) {
+ verbose = false;
+ } else if (!strcmp("-v", *iter) || !strcmp("--verbose", *iter)) {
+ verbose = true;
+ } else if (!strcmp("-V", *iter) || !strcmp("--version", *iter)) {
+ printf("%s v2.1 ICU " U_ICU_VERSION "\n", pname);
+ return 0;
+ } else if (!strcmp("-o", *iter) || !strcmp("--output", *iter)) {
+ ++iter;
+ if (iter != end && !outfilestr) {
+ outfilestr = *iter;
+ } else {
+ usage(pname, 1);
+ }
+ } else if (0 == strcmp("--add-signature", *iter)) {
+ cf.signature = 1;
+ } else if (0 == strcmp("--remove-signature", *iter)) {
+ cf.signature = -1;
+ } else if (**iter == '-' && (*iter)[1]) {
+ usage(pname, 1);
+ } else {
+ // move a non-option up in argv[]
+ *remainArgvLimit++ = *iter;
+ }
+ }
+
+ if (printConvs || printName) {
+ return printConverters(pname, printName, printCanon) ? 2 : 0;
+ } else if (printTranslits) {
+ return printTransliterators(printCanon) ? 3 : 0;
+ }
+
+ if (!fromcpage || !uprv_strcmp(fromcpage, "-")) {
+ fromcpage = ucnv_getDefaultName();
+ }
+ if (!tocpage || !uprv_strcmp(tocpage, "-")) {
+ tocpage = ucnv_getDefaultName();
+ }
+
+ // Open the correct output file or connect to stdout for reading input
+ if (outfilestr != 0 && strcmp(outfilestr, "-")) {
+ outfile = fopen(outfilestr, "wb");
+ if (outfile == 0) {
+ UnicodeString str1(outfilestr, "");
+ UnicodeString str2(strerror(errno), "");
+ initMsg(pname);
+ u_wmsg(stderr, "cantCreateOutputF",
+ str1.getBuffer(), str2.getBuffer());
+ return 1;
+ }
+ } else {
+ outfilestr = "-";
+ outfile = stdout;
+#ifdef USE_FILENO_BINARY_MODE
+ if (setmode(fileno(outfile), O_BINARY) == -1) {
+ u_wmsg(stderr, "cantSetOutBinMode");
+ exit(-1);
+ }
+#endif
+ }
+
+ /* Loop again on the arguments to find all the input files, and
+ convert them. */
+
+ cf.setBufferSize(bufsz);
+
+ if(remainArgv < remainArgvLimit) {
+ for (iter = remainArgv; iter != remainArgvLimit; iter++) {
+ if (!cf.convertFile(
+ pname, fromcpage, toucallback, touctxt, tocpage,
+ fromucallback, fromuctxt, fallback, translit, *iter,
+ outfile, verbose)
+ ) {
+ goto error_exit;
+ }
+ }
+ } else {
+ if (!cf.convertFile(
+ pname, fromcpage, toucallback, touctxt, tocpage,
+ fromucallback, fromuctxt, fallback, translit, 0,
+ outfile, verbose)
+ ) {
+ goto error_exit;
+ }
+ }
+
+ goto normal_exit;
+error_exit:
+#if !UCONFIG_NO_LEGACY_CONVERSION
+ ret = 1;
+#else
+ fprintf(stderr, "uconv error: UCONFIG_NO_LEGACY_CONVERSION is on. See uconfig.h\n");
+#endif
+normal_exit:
+
+ if (outfile != stdout) {
+ fclose(outfile);
+ }
+
+ u_cleanup();
+
+ return ret;
+}
+
+
+/*
+ * Hey, Emacs, please set the following:
+ *
+ * Local Variables:
+ * indent-tabs-mode: nil
+ * End:
+ *
+ */
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-19 15:19:27 +0000