Adding upstream version 115.7.0esr.upstream/115.7.0esrupstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 862 insertions, 0 deletions

diff --git a/intl/icu/source/tools/makeconv/makeconv.cpp b/intl/icu/source/tools/makeconv/makeconv.cpp
new file mode 100644
index 0000000000..b14b4316f3
--- /dev/null
+++ b/intl/icu/source/tools/makeconv/makeconv.cpp
@@ -0,0 +1,862 @@
+// © 2016 and later: Unicode, Inc. and others.
+// License & terms of use: http://www.unicode.org/copyright.html
+/*
+ ********************************************************************************
+ *
+ *   Copyright (C) 1998-2015, International Business Machines
+ *   Corporation and others.  All Rights Reserved.
+ *
+ ********************************************************************************
+ *
+ *
+ *  makeconv.cpp:
+ *  tool creating a binary (compressed) representation of the conversion mapping
+ *  table (IBM NLTC ucmap format).
+ *
+ *  05/04/2000    helena     Added fallback mapping into the picture...
+ *  06/29/2000  helena      Major rewrite of the callback APIs.
+ */
+
+#include <stdio.h>
+#include "unicode/putil.h"
+#include "unicode/ucnv_err.h"
+#include "charstr.h"
+#include "ucnv_bld.h"
+#include "ucnv_imp.h"
+#include "ucnv_cnv.h"
+#include "cstring.h"
+#include "cmemory.h"
+#include "uinvchar.h"
+#include "filestrm.h"
+#include "toolutil.h"
+#include "uoptions.h"
+#include "unicode/udata.h"
+#include "unewdata.h"
+#include "uparse.h"
+#include "ucm.h"
+#include "makeconv.h"
+#include "genmbcs.h"
+
+#define DEBUG 0
+
+typedef struct ConvData {
+    UCMFile *ucm;
+    NewConverter *cnvData, *extData;
+    UConverterSharedData sharedData;
+    UConverterStaticData staticData;
+} ConvData;
+
+static void
+initConvData(ConvData *data) {
+    uprv_memset(data, 0, sizeof(ConvData));
+    data->sharedData.structSize=sizeof(UConverterSharedData);
+    data->staticData.structSize=sizeof(UConverterStaticData);
+    data->sharedData.staticData=&data->staticData;
+}
+
+static void
+cleanupConvData(ConvData *data) {
+    if(data!=nullptr) {
+        if(data->cnvData!=nullptr) {
+            data->cnvData->close(data->cnvData);
+            data->cnvData=nullptr;
+        }
+        if(data->extData!=nullptr) {
+            data->extData->close(data->extData);
+            data->extData=nullptr;
+        }
+        ucm_close(data->ucm);
+        data->ucm=nullptr;
+    }
+}
+
+/*
+ * from ucnvstat.c - static prototypes of data-based converters
+ */
+U_CAPI const UConverterStaticData * ucnv_converterStaticData[UCNV_NUMBER_OF_SUPPORTED_CONVERTER_TYPES];
+
+/*
+ * Global - verbosity
+ */
+UBool VERBOSE = false;
+UBool QUIET = false;
+UBool SMALL = false;
+UBool IGNORE_SISO_CHECK = false;
+
+static void
+createConverter(ConvData *data, const char* converterName, UErrorCode *pErrorCode);
+
+/*
+ * Set up the UNewData and write the converter..
+ */
+static void
+writeConverterData(ConvData *data, const char *cnvName, const char *cnvDir, UErrorCode *status);
+
+UBool haveCopyright=true;
+
+static UDataInfo dataInfo={
+    sizeof(UDataInfo),
+    0,
+
+    U_IS_BIG_ENDIAN,
+    U_CHARSET_FAMILY,
+    sizeof(char16_t),
+    0,
+
+    {0x63, 0x6e, 0x76, 0x74},     /* dataFormat="cnvt" */
+    {6, 2, 0, 0},                 /* formatVersion */
+    {0, 0, 0, 0}                  /* dataVersion (calculated at runtime) */
+};
+
+static void
+writeConverterData(ConvData *data, const char *cnvName, const char *cnvDir, UErrorCode *status)
+{
+    UNewDataMemory *mem = nullptr;
+    uint32_t sz2;
+    uint32_t size = 0;
+    int32_t tableType;
+
+    if(U_FAILURE(*status))
+      {
+        return;
+      }
+
+    tableType=TABLE_NONE;
+    if(data->cnvData!=nullptr) {
+        tableType|=TABLE_BASE;
+    }
+    if(data->extData!=nullptr) {
+        tableType|=TABLE_EXT;
+    }
+
+    mem = udata_create(cnvDir, "cnv", cnvName, &dataInfo, haveCopyright ? U_COPYRIGHT_STRING : nullptr, status);
+
+    if(U_FAILURE(*status))
+      {
+        fprintf(stderr, "Couldn't create the udata %s.%s: %s\n",
+                cnvName,
+                "cnv",
+                u_errorName(*status));
+        return;
+      }
+
+    if(VERBOSE)
+      {
+        printf("- Opened udata %s.%s\n", cnvName, "cnv");
+      }
+
+
+    /* all read only, clean, platform independent data.  Mmmm. :)  */
+    udata_writeBlock(mem, &data->staticData, sizeof(UConverterStaticData));
+    size += sizeof(UConverterStaticData); /* Is 4-aligned  - by size */
+    /* Now, write the table */
+    if(tableType&TABLE_BASE) {
+        size += data->cnvData->write(data->cnvData, &data->staticData, mem, tableType);
+    }
+    if(tableType&TABLE_EXT) {
+        size += data->extData->write(data->extData, &data->staticData, mem, tableType);
+    }
+
+    sz2 = udata_finish(mem, status);
+    if(size != sz2)
+    {
+        fprintf(stderr, "error: wrote %u bytes to the .cnv file but counted %u bytes\n", (int)sz2, (int)size);
+        *status=U_INTERNAL_PROGRAM_ERROR;
+    }
+    if(VERBOSE)
+    {
+      printf("- Wrote %u bytes to the udata.\n", (int)sz2);
+    }
+}
+
+enum {
+    OPT_HELP_H,
+    OPT_HELP_QUESTION_MARK,
+    OPT_COPYRIGHT,
+    OPT_VERSION,
+    OPT_DESTDIR,
+    OPT_VERBOSE,
+    OPT_SMALL,
+    OPT_IGNORE_SISO_CHECK,
+    OPT_QUIET,
+    OPT_SOURCEDIR,
+
+    OPT_COUNT
+};
+
+static UOption options[]={
+    UOPTION_HELP_H,
+    UOPTION_HELP_QUESTION_MARK,
+    UOPTION_COPYRIGHT,
+    UOPTION_VERSION,
+    UOPTION_DESTDIR,
+    UOPTION_VERBOSE,
+    { "small", nullptr, nullptr, nullptr, '\1', UOPT_NO_ARG, 0 },
+    { "ignore-siso-check", nullptr, nullptr, nullptr, '\1', UOPT_NO_ARG, 0 },
+    UOPTION_QUIET,
+    UOPTION_SOURCEDIR,
+};
+
+int main(int argc, char* argv[])
+{
+    ConvData data;
+    char cnvName[UCNV_MAX_FULL_FILE_NAME_LENGTH];
+
+    U_MAIN_INIT_ARGS(argc, argv);
+
+    /* Set up the ICU version number */
+    UVersionInfo icuVersion;
+    u_getVersion(icuVersion);
+    uprv_memcpy(&dataInfo.dataVersion, &icuVersion, sizeof(UVersionInfo));
+
+    /* preset then read command line options */
+    options[OPT_DESTDIR].value=u_getDataDirectory();
+    argc=u_parseArgs(argc, argv, UPRV_LENGTHOF(options), options);
+
+    if(options[OPT_VERSION].doesOccur) {
+        printf("makeconv version %u.%u, ICU tool to read .ucm codepage mapping files and write .cnv files\n",
+               dataInfo.formatVersion[0], dataInfo.formatVersion[1]);
+        printf("%s\n", U_COPYRIGHT_STRING);
+        exit(0);
+    }
+
+    /* error handling, printing usage message */
+    if(argc<0) {
+        fprintf(stderr,
+            "error in command line argument \"%s\"\n",
+            argv[-argc]);
+    } else if(argc<2) {
+        argc=-1;
+    }
+    if(argc<0 || options[OPT_HELP_H].doesOccur || options[OPT_HELP_QUESTION_MARK].doesOccur) {
+        FILE *stdfile=argc<0 ? stderr : stdout;
+        fprintf(stdfile,
+            "usage: %s [-options] files...\n"
+            "\tread .ucm codepage mapping files and write .cnv files\n"
+            "options:\n"
+            "\t-h or -? or --help  this usage text\n"
+            "\t-V or --version     show a version message\n"
+            "\t-c or --copyright   include a copyright notice\n"
+            "\t-d or --destdir     destination directory, followed by the path\n"
+            "\t-v or --verbose     Turn on verbose output\n"
+            "\t-q or --quiet       do not display warnings and progress\n"
+            "\t-s or --sourcedir   source directory, followed by the path\n",
+            argv[0]);
+        fprintf(stdfile,
+            "\t      --small       Generate smaller .cnv files. They will be\n"
+            "\t                    significantly smaller but may not be compatible with\n"
+            "\t                    older versions of ICU and will require heap memory\n"
+            "\t                    allocation when loaded.\n"
+            "\t      --ignore-siso-check         Use SI/SO other than 0xf/0xe.\n");
+        return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR;
+    }
+
+    /* get the options values */
+    haveCopyright = options[OPT_COPYRIGHT].doesOccur;
+    const char *destdir = options[OPT_DESTDIR].value;
+    VERBOSE = options[OPT_VERBOSE].doesOccur;
+    QUIET = options[OPT_QUIET].doesOccur;
+    SMALL = options[OPT_SMALL].doesOccur;
+
+    if (options[OPT_IGNORE_SISO_CHECK].doesOccur) {
+        IGNORE_SISO_CHECK = true;
+    }
+
+    icu::CharString outFileName;
+    UErrorCode err = U_ZERO_ERROR;
+    if (destdir != nullptr && *destdir != 0) {
+        outFileName.append(destdir, err).ensureEndsWithFileSeparator(err);
+        if (U_FAILURE(err)) {
+            return err;
+        }
+    }
+    int32_t outBasenameStart = outFileName.length();
+
+#if DEBUG
+    {
+      int i;
+      printf("makeconv: processing %d files...\n", argc - 1);
+      for(i=1; i<argc; ++i) {
+        printf("%s ", argv[i]);
+      }
+      printf("\n");
+      fflush(stdout);
+    }
+#endif
+
+    UBool printFilename = (UBool) (argc > 2 || VERBOSE);
+    icu::CharString pathBuf;
+    for (++argv; --argc; ++argv)
+    {
+        UErrorCode localError = U_ZERO_ERROR;
+        const char *arg = getLongPathname(*argv);
+
+        const char* sourcedir = options[OPT_SOURCEDIR].value;
+        if (sourcedir != nullptr && *sourcedir != 0 && uprv_strcmp(sourcedir, ".") != 0) {
+            pathBuf.clear();
+            pathBuf.appendPathPart(sourcedir, localError);
+            pathBuf.appendPathPart(arg, localError);
+            arg = pathBuf.data();
+        }
+
+        /*produces the right destination path for display*/
+        outFileName.truncate(outBasenameStart);
+        if (outBasenameStart != 0)
+        {
+            /* find the last file sepator */
+            const char *basename = findBasename(arg);
+            outFileName.append(basename, localError);
+        }
+        else
+        {
+            outFileName.append(arg, localError);
+        }
+        if (U_FAILURE(localError)) {
+            return localError;
+        }
+
+        /*removes the extension if any is found*/
+        int32_t lastDotIndex = outFileName.lastIndexOf('.');
+        if (lastDotIndex >= outBasenameStart) {
+            outFileName.truncate(lastDotIndex);
+        }
+
+        /* the basename without extension is the converter name */
+        if ((outFileName.length() - outBasenameStart) >= UPRV_LENGTHOF(cnvName)) {
+            fprintf(stderr, "converter name %s too long\n", outFileName.data() + outBasenameStart);
+            return U_BUFFER_OVERFLOW_ERROR;
+        }
+        uprv_strcpy(cnvName, outFileName.data() + outBasenameStart);
+
+        /*Adds the target extension*/
+        outFileName.append(CONVERTER_FILE_EXTENSION, localError);
+        if (U_FAILURE(localError)) {
+            return localError;
+        }
+
+#if DEBUG
+        printf("makeconv: processing %s  ...\n", arg);
+        fflush(stdout);
+#endif
+        initConvData(&data);
+        createConverter(&data, arg, &localError);
+
+        if (U_FAILURE(localError))
+        {
+            /* if an error is found, print out an error msg and keep going */
+            fprintf(stderr, "Error creating converter for \"%s\" file for \"%s\" (%s)\n",
+                    outFileName.data(), arg, u_errorName(localError));
+            if(U_SUCCESS(err)) {
+                err = localError;
+            }
+        }
+        else
+        {
+            /* Insure the static data name matches the  file name */
+            /* Changed to ignore directory and only compare base name
+             LDH 1/2/08*/
+            char *p;
+            p = strrchr(cnvName, U_FILE_SEP_CHAR); /* Find last file separator */
+
+            if(p == nullptr)            /* OK, try alternate */
+            {
+                p = strrchr(cnvName, U_FILE_ALT_SEP_CHAR);
+                if(p == nullptr)
+                {
+                    p=cnvName; /* If no separators, no problem */
+                }
+            }
+            else
+            {
+                p++;   /* If found separator, don't include it in compare */
+            }
+            if(uprv_stricmp(p,data.staticData.name) && !QUIET)
+            {
+                fprintf(stderr, "Warning: %s%s claims to be '%s'\n",
+                    cnvName,  CONVERTER_FILE_EXTENSION,
+                    data.staticData.name);
+            }
+
+            uprv_strcpy((char*)data.staticData.name, cnvName);
+
+            if(!uprv_isInvariantString((char*)data.staticData.name, -1)) {
+                fprintf(stderr,
+                    "Error: A converter name must contain only invariant characters.\n"
+                    "%s is not a valid converter name.\n",
+                    data.staticData.name);
+                if(U_SUCCESS(err)) {
+                    err = U_INVALID_TABLE_FORMAT;
+                }
+            }
+
+            localError = U_ZERO_ERROR;
+            writeConverterData(&data, cnvName, destdir, &localError);
+
+            if(U_FAILURE(localError))
+            {
+                /* if an error is found, print out an error msg and keep going*/
+                fprintf(stderr, "Error writing \"%s\" file for \"%s\" (%s)\n", outFileName.data(), arg,
+                    u_errorName(localError));
+                if(U_SUCCESS(err)) {
+                    err = localError;
+                }
+            }
+            else if (printFilename)
+            {
+                puts(outFileName.data() + outBasenameStart);
+            }
+        }
+        fflush(stdout);
+        fflush(stderr);
+
+        cleanupConvData(&data);
+    }
+
+    return err;
+}
+
+static void
+getPlatformAndCCSIDFromName(const char *name, int8_t *pPlatform, int32_t *pCCSID) {
+    if( (name[0]=='i' || name[0]=='I') &&
+        (name[1]=='b' || name[1]=='B') &&
+        (name[2]=='m' || name[2]=='M')
+    ) {
+        name+=3;
+        if(*name=='-') {
+            ++name;
+        }
+        *pPlatform=UCNV_IBM;
+        *pCCSID=(int32_t)uprv_strtoul(name, nullptr, 10);
+    } else {
+        *pPlatform=UCNV_UNKNOWN;
+        *pCCSID=0;
+    }
+}
+
+static void
+readHeader(ConvData *data,
+           FileStream* convFile,
+           UErrorCode *pErrorCode) {
+    char line[1024];
+    char *s, *key, *value;
+    const UConverterStaticData *prototype;
+    UConverterStaticData *staticData;
+
+    if(U_FAILURE(*pErrorCode)) {
+        return;
+    }
+
+    staticData=&data->staticData;
+    staticData->platform=UCNV_IBM;
+    staticData->subCharLen=0;
+
+    while(T_FileStream_readLine(convFile, line, sizeof(line))) {
+        /* basic parsing and handling of state-related items */
+        if(ucm_parseHeaderLine(data->ucm, line, &key, &value)) {
+            continue;
+        }
+
+        /* stop at the beginning of the mapping section */
+        if(uprv_strcmp(line, "CHARMAP")==0) {
+            break;
+        }
+
+        /* collect the information from the header field, ignore unknown keys */
+        if(uprv_strcmp(key, "code_set_name")==0) {
+            if(*value!=0) {
+                uprv_strcpy((char *)staticData->name, value);
+                getPlatformAndCCSIDFromName(value, &staticData->platform, &staticData->codepage);
+            }
+        } else if(uprv_strcmp(key, "subchar")==0) {
+            uint8_t bytes[UCNV_EXT_MAX_BYTES];
+            int8_t length;
+
+            s=value;
+            length=ucm_parseBytes(bytes, line, (const char **)&s);
+            if(1<=length && length<=4 && *s==0) {
+                staticData->subCharLen=length;
+                uprv_memcpy(staticData->subChar, bytes, length);
+            } else {
+                fprintf(stderr, "error: illegal <subchar> %s\n", value);
+                *pErrorCode=U_INVALID_TABLE_FORMAT;
+                return;
+            }
+        } else if(uprv_strcmp(key, "subchar1")==0) {
+            uint8_t bytes[UCNV_EXT_MAX_BYTES];
+
+            s=value;
+            if(1==ucm_parseBytes(bytes, line, (const char **)&s) && *s==0) {
+                staticData->subChar1=bytes[0];
+            } else {
+                fprintf(stderr, "error: illegal <subchar1> %s\n", value);
+                *pErrorCode=U_INVALID_TABLE_FORMAT;
+                return;
+            }
+        }
+    }
+
+    /* copy values from the UCMFile to the static data */
+    staticData->maxBytesPerChar=(int8_t)data->ucm->states.maxCharLength;
+    staticData->minBytesPerChar=(int8_t)data->ucm->states.minCharLength;
+    staticData->conversionType=data->ucm->states.conversionType;
+
+    if(staticData->conversionType==UCNV_UNSUPPORTED_CONVERTER) {
+        fprintf(stderr, "ucm error: missing conversion type (<uconv_class>)\n");
+        *pErrorCode=U_INVALID_TABLE_FORMAT;
+        return;
+    }
+
+    /*
+     * Now that we know the type, copy any 'default' values from the table.
+     * We need not check the type any further because the parser only
+     * recognizes what we have prototypes for.
+     *
+     * For delta (extension-only) tables, copy values from the base file
+     * instead, see createConverter().
+     */
+    if(data->ucm->baseName[0]==0) {
+        prototype=ucnv_converterStaticData[staticData->conversionType];
+        if(prototype!=nullptr) {
+            if(staticData->name[0]==0) {
+                uprv_strcpy((char *)staticData->name, prototype->name);
+            }
+
+            if(staticData->codepage==0) {
+                staticData->codepage=prototype->codepage;
+            }
+
+            if(staticData->platform==0) {
+                staticData->platform=prototype->platform;
+            }
+
+            if(staticData->minBytesPerChar==0) {
+                staticData->minBytesPerChar=prototype->minBytesPerChar;
+            }
+
+            if(staticData->maxBytesPerChar==0) {
+                staticData->maxBytesPerChar=prototype->maxBytesPerChar;
+            }
+
+            if(staticData->subCharLen==0) {
+                staticData->subCharLen=prototype->subCharLen;
+                if(prototype->subCharLen>0) {
+                    uprv_memcpy(staticData->subChar, prototype->subChar, prototype->subCharLen);
+                }
+            }
+        }
+    }
+
+    if(data->ucm->states.outputType<0) {
+        data->ucm->states.outputType=(int8_t)data->ucm->states.maxCharLength-1;
+    }
+
+    if( staticData->subChar1!=0 &&
+            (staticData->minBytesPerChar>1 ||
+                (staticData->conversionType!=UCNV_MBCS &&
+                 staticData->conversionType!=UCNV_EBCDIC_STATEFUL))
+    ) {
+        fprintf(stderr, "error: <subchar1> defined for a type other than MBCS or EBCDIC_STATEFUL\n");
+        *pErrorCode=U_INVALID_TABLE_FORMAT;
+    }
+}
+
+/* return true if a base table was read, false for an extension table */
+static UBool
+readFile(ConvData *data, const char* converterName,
+         UErrorCode *pErrorCode) {
+    char line[1024];
+    char *end;
+    FileStream *convFile;
+
+    UCMStates *baseStates;
+    UBool dataIsBase;
+
+    if(U_FAILURE(*pErrorCode)) {
+        return false;
+    }
+
+    data->ucm=ucm_open();
+
+    convFile=T_FileStream_open(converterName, "r");
+    if(convFile==nullptr) {
+        *pErrorCode=U_FILE_ACCESS_ERROR;
+        return false;
+    }
+
+    readHeader(data, convFile, pErrorCode);
+    if(U_FAILURE(*pErrorCode)) {
+        return false;
+    }
+
+    if(data->ucm->baseName[0]==0) {
+        dataIsBase=true;
+        baseStates=&data->ucm->states;
+        ucm_processStates(baseStates, IGNORE_SISO_CHECK);
+    } else {
+        dataIsBase=false;
+        baseStates=nullptr;
+    }
+
+    /* read the base table */
+    ucm_readTable(data->ucm, convFile, dataIsBase, baseStates, pErrorCode);
+    if(U_FAILURE(*pErrorCode)) {
+        return false;
+    }
+
+    /* read an extension table if there is one */
+    while(T_FileStream_readLine(convFile, line, sizeof(line))) {
+        end=uprv_strchr(line, 0);
+        while(line<end &&
+              (*(end-1)=='\n' || *(end-1)=='\r' || *(end-1)==' ' || *(end-1)=='\t')) {
+            --end;
+        }
+        *end=0;
+
+        if(line[0]=='#' || u_skipWhitespace(line)==end) {
+            continue; /* ignore empty and comment lines */
+        }
+
+        if(0==uprv_strcmp(line, "CHARMAP")) {
+            /* read the extension table */
+            ucm_readTable(data->ucm, convFile, false, baseStates, pErrorCode);
+        } else {
+            fprintf(stderr, "unexpected text after the base mapping table\n");
+        }
+        break;
+    }
+
+    T_FileStream_close(convFile);
+
+    if(data->ucm->base->flagsType==UCM_FLAGS_MIXED || data->ucm->ext->flagsType==UCM_FLAGS_MIXED) {
+        fprintf(stderr, "error: some entries have the mapping precision (with '|'), some do not\n");
+        *pErrorCode=U_INVALID_TABLE_FORMAT;
+    }
+
+    return dataIsBase;
+}
+
+static void
+createConverter(ConvData *data, const char *converterName, UErrorCode *pErrorCode) {
+    ConvData baseData;
+    UBool dataIsBase;
+
+    UConverterStaticData *staticData;
+    UCMStates *states, *baseStates;
+
+    if(U_FAILURE(*pErrorCode)) {
+        return;
+    }
+
+    initConvData(data);
+
+    dataIsBase=readFile(data, converterName, pErrorCode);
+    if(U_FAILURE(*pErrorCode)) {
+        return;
+    }
+
+    staticData=&data->staticData;
+    states=&data->ucm->states;
+
+    if(dataIsBase) {
+        /*
+         * Build a normal .cnv file with a base table
+         * and an optional extension table.
+         */
+        data->cnvData=MBCSOpen(data->ucm);
+        if(data->cnvData==nullptr) {
+            *pErrorCode=U_MEMORY_ALLOCATION_ERROR;
+
+        } else if(!data->cnvData->isValid(data->cnvData,
+                            staticData->subChar, staticData->subCharLen)
+        ) {
+            fprintf(stderr, "       the substitution character byte sequence is illegal in this codepage structure!\n");
+            *pErrorCode=U_INVALID_TABLE_FORMAT;
+
+        } else if(staticData->subChar1!=0 &&
+                    !data->cnvData->isValid(data->cnvData, &staticData->subChar1, 1)
+        ) {
+            fprintf(stderr, "       the subchar1 byte is illegal in this codepage structure!\n");
+            *pErrorCode=U_INVALID_TABLE_FORMAT;
+
+        } else if(
+            data->ucm->ext->mappingsLength>0 &&
+            !ucm_checkBaseExt(states, data->ucm->base, data->ucm->ext, data->ucm->ext, false)
+        ) {
+            *pErrorCode=U_INVALID_TABLE_FORMAT;
+        } else if(data->ucm->base->flagsType&UCM_FLAGS_EXPLICIT) {
+            /* sort the table so that it can be turned into UTF-8-friendly data */
+            ucm_sortTable(data->ucm->base);
+        }
+
+        if(U_SUCCESS(*pErrorCode)) {
+            if(
+                /* add the base table after ucm_checkBaseExt()! */
+                !data->cnvData->addTable(data->cnvData, data->ucm->base, &data->staticData)
+            ) {
+                *pErrorCode=U_INVALID_TABLE_FORMAT;
+            } else {
+                /*
+                 * addTable() may have requested moving more mappings to the extension table
+                 * if they fit into the base toUnicode table but not into the
+                 * base fromUnicode table.
+                 * (Especially for UTF-8-friendly fromUnicode tables.)
+                 * Such mappings will have the MBCS_FROM_U_EXT_FLAG set, which causes them
+                 * to be excluded from the extension toUnicode data.
+                 * See MBCSOkForBaseFromUnicode() for which mappings do not fit into
+                 * the base fromUnicode table.
+                 */
+                ucm_moveMappings(data->ucm->base, data->ucm->ext);
+                ucm_sortTable(data->ucm->ext);
+                if(data->ucm->ext->mappingsLength>0) {
+                    /* prepare the extension table, if there is one */
+                    data->extData=CnvExtOpen(data->ucm);
+                    if(data->extData==nullptr) {
+                        *pErrorCode=U_MEMORY_ALLOCATION_ERROR;
+                    } else if(
+                        !data->extData->addTable(data->extData, data->ucm->ext, &data->staticData)
+                    ) {
+                        *pErrorCode=U_INVALID_TABLE_FORMAT;
+                    }
+                }
+            }
+        }
+    } else {
+        /* Build an extension-only .cnv file. */
+        char baseFilename[500];
+        char *basename;
+
+        initConvData(&baseData);
+
+        /* assemble a path/filename for data->ucm->baseName */
+        uprv_strcpy(baseFilename, converterName);
+        basename=(char *)findBasename(baseFilename);
+        uprv_strcpy(basename, data->ucm->baseName);
+        uprv_strcat(basename, ".ucm");
+
+        /* read the base table */
+        dataIsBase=readFile(&baseData, baseFilename, pErrorCode);
+        if(U_FAILURE(*pErrorCode)) {
+            return;
+        } else if(!dataIsBase) {
+            fprintf(stderr, "error: the <icu:base> file \"%s\" is not a base table file\n", baseFilename);
+            *pErrorCode=U_INVALID_TABLE_FORMAT;
+        } else {
+            /* prepare the extension table */
+            data->extData=CnvExtOpen(data->ucm);
+            if(data->extData==nullptr) {
+                *pErrorCode=U_MEMORY_ALLOCATION_ERROR;
+            } else {
+                /* fill in gaps in extension file header fields */
+                UCMapping *m, *mLimit;
+                uint8_t fallbackFlags;
+
+                baseStates=&baseData.ucm->states;
+                if(states->conversionType==UCNV_DBCS) {
+                    staticData->minBytesPerChar=(int8_t)(states->minCharLength=2);
+                } else if(states->minCharLength==0) {
+                    staticData->minBytesPerChar=(int8_t)(states->minCharLength=baseStates->minCharLength);
+                }
+                if(states->maxCharLength<states->minCharLength) {
+                    staticData->maxBytesPerChar=(int8_t)(states->maxCharLength=baseStates->maxCharLength);
+                }
+
+                if(staticData->subCharLen==0) {
+                    uprv_memcpy(staticData->subChar, baseData.staticData.subChar, 4);
+                    staticData->subCharLen=baseData.staticData.subCharLen;
+                }
+                /*
+                 * do not copy subChar1 -
+                 * only use what is explicitly specified
+                 * because it cannot be unset in the extension file header
+                 */
+
+                /* get the fallback flags */
+                fallbackFlags=0;
+                for(m=baseData.ucm->base->mappings, mLimit=m+baseData.ucm->base->mappingsLength;
+                    m<mLimit && fallbackFlags!=3;
+                    ++m
+                ) {
+                    if(m->f==1) {
+                        fallbackFlags|=1;
+                    } else if(m->f==3) {
+                        fallbackFlags|=2;
+                    }
+                }
+
+                if(fallbackFlags&1) {
+                    staticData->hasFromUnicodeFallback=true;
+                }
+                if(fallbackFlags&2) {
+                    staticData->hasToUnicodeFallback=true;
+                }
+
+                if(1!=ucm_countChars(baseStates, staticData->subChar, staticData->subCharLen)) {
+                    fprintf(stderr, "       the substitution character byte sequence is illegal in this codepage structure!\n");
+                    *pErrorCode=U_INVALID_TABLE_FORMAT;
+
+                } else if(staticData->subChar1!=0 && 1!=ucm_countChars(baseStates, &staticData->subChar1, 1)) {
+                    fprintf(stderr, "       the subchar1 byte is illegal in this codepage structure!\n");
+                    *pErrorCode=U_INVALID_TABLE_FORMAT;
+
+                } else if(
+                    !ucm_checkValidity(data->ucm->ext, baseStates) ||
+                    !ucm_checkBaseExt(baseStates, baseData.ucm->base, data->ucm->ext, data->ucm->ext, false)
+                ) {
+                    *pErrorCode=U_INVALID_TABLE_FORMAT;
+                } else {
+                    if(states->maxCharLength>1) {
+                        /*
+                         * When building a normal .cnv file with a base table
+                         * for an MBCS (not SBCS) table with explicit precision flags,
+                         * the MBCSAddTable() function marks some mappings for moving
+                         * to the extension table.
+                         * They fit into the base toUnicode table but not into the
+                         * base fromUnicode table.
+                         * (Note: We do have explicit precision flags because they are
+                         * required for extension table generation, and
+                         * ucm_checkBaseExt() verified it.)
+                         *
+                         * We do not call MBCSAddTable() here (we probably could)
+                         * so we need to do the analysis before building the extension table.
+                         * We assume that MBCSAddTable() will build a UTF-8-friendly table.
+                         * Redundant mappings in the extension table are ok except they cost some size.
+                         *
+                         * Do this after ucm_checkBaseExt().
+                         */
+                        const MBCSData *mbcsData=MBCSGetDummy();
+                        int32_t needsMove=0;
+                        for(m=baseData.ucm->base->mappings, mLimit=m+baseData.ucm->base->mappingsLength;
+                            m<mLimit;
+                            ++m
+                        ) {
+                            if(!MBCSOkForBaseFromUnicode(mbcsData, m->b.bytes, m->bLen, m->u, m->f)) {
+                                m->f|=MBCS_FROM_U_EXT_FLAG;
+                                m->moveFlag=UCM_MOVE_TO_EXT;
+                                ++needsMove;
+                            }
+                        }
+
+                        if(needsMove!=0) {
+                            ucm_moveMappings(baseData.ucm->base, data->ucm->ext);
+                            ucm_sortTable(data->ucm->ext);
+                        }
+                    }
+                    if(!data->extData->addTable(data->extData, data->ucm->ext, &data->staticData)) {
+                        *pErrorCode=U_INVALID_TABLE_FORMAT;
+                    }
+                }
+            }
+        }
+
+        cleanupConvData(&baseData);
+    }
+}
+
+/*
+ * Hey, Emacs, please set the following:
+ *
+ * Local Variables:
+ * indent-tabs-mode: nil
+ * End:
+ *
+ */