diff options
Diffstat (limited to '')
| -rw-r--r-- | intl/icu/source/tools/gendict/Makefile.in | 96 | ||||
| -rw-r--r-- | intl/icu/source/tools/gendict/gendict.1.in | 133 | ||||
| -rw-r--r-- | intl/icu/source/tools/gendict/gendict.cpp | 480 | ||||
| -rw-r--r-- | intl/icu/source/tools/gendict/gendict.vcxproj | 80 | ||||
| -rw-r--r-- | intl/icu/source/tools/gendict/gendict.vcxproj.filters | 22 | ||||
| -rw-r--r-- | intl/icu/source/tools/gendict/sources.txt | 1 |
6 files changed, 812 insertions, 0 deletions
diff --git a/intl/icu/source/tools/gendict/Makefile.in b/intl/icu/source/tools/gendict/Makefile.in new file mode 100644 index 0000000000..d2cdb1c3c3 --- /dev/null +++ b/intl/icu/source/tools/gendict/Makefile.in @@ -0,0 +1,96 @@ +## Makefile.in for ICU - tools/gendict +## Copyright (C) 2016 and later: Unicode, Inc. and others. +## License & terms of use: http://www.unicode.org/copyright.html +## Copyright (c) 2002-2012 International Business Machines Corporation and +## others. All Rights Reserved. + +## Source directory information +srcdir = @srcdir@ +top_srcdir = @top_srcdir@ + +top_builddir = ../.. + +include $(top_builddir)/icudefs.mk + +## Build directory information +subdir = tools/gendict + +TARGET_STUB_NAME = gendict + +SECTION = 1 + +MAN_FILES = $(TARGET_STUB_NAME).$(SECTION) + + +## Extra files to remove for 'make clean' +CLEANFILES = *~ $(DEPS) $(MAN_FILES) + +## Target information +TARGET = $(BINDIR)/$(TARGET_STUB_NAME)$(EXEEXT) + +CPPFLAGS += -I$(top_srcdir)/common -I$(srcdir)/../toolutil +LIBS = $(LIBICUTOOLUTIL) $(LIBICUI18N) $(LIBICUUC) $(DEFAULT_LIBS) $(LIB_M) + +SOURCES = $(shell cat $(srcdir)/sources.txt) +OBJECTS = $(SOURCES:.cpp=.o) + +DEPS = $(OBJECTS:.o=.d) + +## List of phony targets +.PHONY : all all-local install install-local clean clean-local \ +distclean distclean-local dist dist-local check check-local install-man + +## Clear suffix list +.SUFFIXES : + +## List of standard targets +all: all-local +install: install-local +clean: clean-local +distclean : distclean-local +dist: dist-local +check: all check-local + +all-local: $(TARGET) $(MAN_FILES) + +install-local: all-local install-man + $(MKINSTALLDIRS) $(DESTDIR)$(bindir) + $(INSTALL) $(TARGET) $(DESTDIR)$(bindir) + +install-man: $(MAN_FILES) + $(MKINSTALLDIRS) $(DESTDIR)$(mandir)/man$(SECTION) + $(INSTALL_DATA) $? $(DESTDIR)$(mandir)/man$(SECTION) + +dist-local: + +clean-local: + test -z "$(CLEANFILES)" || $(RMV) $(CLEANFILES) + $(RMV) $(TARGET) $(OBJECTS) + +distclean-local: clean-local + $(RMV) Makefile + +check-local: all-local + +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + cd $(top_builddir) \ + && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= $(SHELL) ./config.status + +$(TARGET) : $(OBJECTS) + $(LINK.cc) $(OUTOPT)$@ $^ $(LIBS) + $(POST_BUILD_STEP) + + +%.$(SECTION): $(srcdir)/%.$(SECTION).in + cd $(top_builddir) \ + && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= $(SHELL) ./config.status + + +ifeq (,$(MAKECMDGOALS)) +-include $(DEPS) +else +ifneq ($(patsubst %clean,,$(MAKECMDGOALS)),) +-include $(DEPS) +endif +endif + diff --git a/intl/icu/source/tools/gendict/gendict.1.in b/intl/icu/source/tools/gendict/gendict.1.in new file mode 100644 index 0000000000..f204f83e4e --- /dev/null +++ b/intl/icu/source/tools/gendict/gendict.1.in @@ -0,0 +1,133 @@ +.\" Hey, Emacs! This is -*-nroff-*- you know... +.\" +.\" gendict.1: manual page for the gendict utility +.\" +.\" Copyright (C) 2016 and later: Unicode, Inc. and others. +.\" License & terms of use: http://www.unicode.org/copyright.html +.\" Copyright (C) 2012 International Business Machines Corporation and others +.\" +.TH GENDICT 1 "1 June 2012" "ICU MANPAGE" "ICU @VERSION@ Manual" +.SH NAME +.B gendict +\- Compiles word list into ICU string trie dictionary +.SH SYNOPSIS +.B gendict +[ +.BR "\fB\-\-uchars" +| +.BR "\fB\-\-bytes" +.BI "\fB\-\-transform" " transform" +] +[ +.BR "\-h\fP, \fB\-?\fP, \fB\-\-help" +] +[ +.BR "\-V\fP, \fB\-\-version" +] +[ +.BR "\-c\fP, \fB\-\-copyright" +] +[ +.BR "\-v\fP, \fB\-\-verbose" +] +[ +.BI "\-i\fP, \fB\-\-icudatadir" " directory" +] +.IR " input-file" +.IR " output\-file" +.SH DESCRIPTION +.B gendict +reads the word list from +.I dictionary-file +and creates a string trie dictionary file. Normally this data file has the +.B .dict +extension. +.PP +Words begin at the beginning of a line and are terminated by the first whitespace. +Lines that begin with whitespace are ignored. +.SH OPTIONS +.TP +.BR "\-h\fP, \fB\-?\fP, \fB\-\-help" +Print help about usage and exit. +.TP +.BR "\-V\fP, \fB\-\-version" +Print the version of +.B gendict +and exit. +.TP +.BR "\-c\fP, \fB\-\-copyright" +Embeds the standard ICU copyright into the +.IR output-file . +.TP +.BR "\-v\fP, \fB\-\-verbose" +Display extra informative messages during execution. +.TP +.BI "\-i\fP, \fB\-\-icudatadir" " directory" +Look for any necessary ICU data files in +.IR directory . +For example, the file +.B pnames.icu +must be located when ICU's data is not built as a shared library. +The default ICU data directory is specified by the environment variable +.BR ICU_DATA . +Most configurations of ICU do not require this argument. +.TP +.BR "\fB\-\-uchars" +Set the output trie type to UChar. Mutually exclusive with +.BR --bytes. +.TP +.BR "\fB\-\-bytes" +Set the output trie type to Bytes. Mutually exclusive with +.BR --uchars. +.TP +.BR "\fB\-\-transform" +Set the transform type. Should only be specified with +.BR --bytes. +Currently supported transforms are: +.BR offset-<hex-number>, +which specifies an offset to subtract from all input characters. +It should be noted that the offset transform also maps U+200D +to 0xFF and U+200C to 0xFE, in order to offer compatibility to +languages that require these characters. +A transform must be specified for a bytes trie, and when applied +to the non-value characters in the +.IR input-file +must produce output between 0x00 and 0xFF. +.TP +.BI " input\-file" +The source file to read. +.TP +.BI " output\-file" +The file to write the output dictionary to. +.SH CAVEATS +The +.IR input-file +is assumed to be encoded in UTF-8. +The integers in the +.IR input-file +that are used as values must be made up of ASCII digits. They +may be specified either in hex, by using a 0x prefix, or in +decimal. +Either +.BI --bytes +or +.BI --uchars +must be specified. +.SH ENVIRONMENT +.TP 10 +.B ICU_DATA +Specifies the directory containing ICU data. Defaults to +.BR @thepkgicudatadir@/@PACKAGE@/@VERSION@/ . +Some tools in ICU depend on the presence of the trailing slash. It is thus +important to make sure that it is present if +.B ICU_DATA +is set. +.SH AUTHORS +Maxime Serrano +.SH VERSION +1.0 +.SH COPYRIGHT +Copyright (C) 2012 International Business Machines Corporation and others +.SH SEE ALSO +.BR http://www.icu-project.org/userguide/boundaryAnalysis.html + diff --git a/intl/icu/source/tools/gendict/gendict.cpp b/intl/icu/source/tools/gendict/gendict.cpp new file mode 100644 index 0000000000..995bd32724 --- /dev/null +++ b/intl/icu/source/tools/gendict/gendict.cpp @@ -0,0 +1,480 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (C) 2002-2016, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* +* File gendict.cpp +*/ + +#include "unicode/utypes.h" +#include "unicode/uchar.h" +#include "unicode/ucnv.h" +#include "unicode/uniset.h" +#include "unicode/unistr.h" +#include "unicode/uclean.h" +#include "unicode/udata.h" +#include "unicode/putil.h" +#include "unicode/ucharstriebuilder.h" +#include "unicode/bytestriebuilder.h" +#include "unicode/ucharstrie.h" +#include "unicode/bytestrie.h" +#include "unicode/ucnv.h" +#include "unicode/ustring.h" +#include "unicode/utf16.h" + +#include "charstr.h" +#include "dictionarydata.h" +#include "uoptions.h" +#include "unewdata.h" +#include "cmemory.h" +#include "uassert.h" +#include "ucbuf.h" +#include "toolutil.h" +#include "cstring.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "putilimp.h" +UDate startTime; + +static int elapsedTime() { + return (int)uprv_floor((uprv_getRawUTCtime()-startTime)/1000.0); +} + +U_NAMESPACE_USE + +static char *progName; +static UOption options[]={ + UOPTION_HELP_H, /* 0 */ + UOPTION_HELP_QUESTION_MARK, /* 1 */ + UOPTION_VERBOSE, /* 2 */ + UOPTION_ICUDATADIR, /* 4 */ + UOPTION_COPYRIGHT, /* 5 */ + { "uchars", nullptr, nullptr, nullptr, '\1', UOPT_NO_ARG, 0}, /* 6 */ + { "bytes", nullptr, nullptr, nullptr, '\1', UOPT_NO_ARG, 0}, /* 7 */ + { "transform", nullptr, nullptr, nullptr, '\1', UOPT_REQUIRES_ARG, 0}, /* 8 */ + UOPTION_QUIET, /* 9 */ +}; + +enum arguments { + ARG_HELP = 0, + ARG_QMARK, + ARG_VERBOSE, + ARG_ICUDATADIR, + ARG_COPYRIGHT, + ARG_UCHARS, + ARG_BYTES, + ARG_TRANSFORM, + ARG_QUIET +}; + +// prints out the standard usage method describing command line arguments, +// then bails out with the desired exit code +static void usageAndDie(UErrorCode retCode) { + fprintf((U_SUCCESS(retCode) ? stdout : stderr), "Usage: %s -trietype [-options] input-dictionary-file output-file\n", progName); + fprintf((U_SUCCESS(retCode) ? stdout : stderr), + "\tRead in a word list and write out a string trie dictionary\n" + "options:\n" + "\t-h or -? or --help this usage text\n" + "\t-V or --version show a version message\n" + "\t-c or --copyright include a copyright notice\n" + "\t-v or --verbose turn on verbose output\n" + "\t-q or --quiet do not display warnings and progress\n" + "\t-i or --icudatadir directory for locating any needed intermediate data files,\n" // TODO: figure out if we need this option + "\t followed by path, defaults to %s\n" + "\t--uchars output a UCharsTrie (mutually exclusive with -b!)\n" + "\t--bytes output a BytesTrie (mutually exclusive with -u!)\n" + "\t--transform the kind of transform to use (eg --transform offset-40A3,\n" + "\t which specifies an offset transform with constant 0x40A3)\n", + u_getDataDirectory()); + exit(retCode); +} + + +/* UDataInfo cf. udata.h */ +static UDataInfo dataInfo = { + sizeof(UDataInfo), + 0, + + U_IS_BIG_ENDIAN, + U_CHARSET_FAMILY, + U_SIZEOF_UCHAR, + 0, + + { 0x44, 0x69, 0x63, 0x74 }, /* "Dict" */ + { 1, 0, 0, 0 }, /* format version */ + { 0, 0, 0, 0 } /* data version */ +}; + +#if !UCONFIG_NO_BREAK_ITERATION + +// A wrapper for both BytesTrieBuilder and UCharsTrieBuilder. +// may want to put this somewhere in ICU, as it could be useful outside +// of this tool? +class DataDict { +private: + BytesTrieBuilder *bt; + UCharsTrieBuilder *ut; + UChar32 transformConstant; + int32_t transformType; +public: + // constructs a new data dictionary. if there is an error, + // it will be returned in status + // isBytesTrie != 0 will produce a BytesTrieBuilder, + // isBytesTrie == 0 will produce a UCharsTrieBuilder + DataDict(UBool isBytesTrie, UErrorCode &status) : bt(nullptr), ut(nullptr), + transformConstant(0), transformType(DictionaryData::TRANSFORM_NONE) { + if (isBytesTrie) { + bt = new BytesTrieBuilder(status); + } else { + ut = new UCharsTrieBuilder(status); + } + } + + ~DataDict() { + delete bt; + delete ut; + } + +private: + char transform(UChar32 c, UErrorCode &status) { + if (transformType == DictionaryData::TRANSFORM_TYPE_OFFSET) { + if (c == 0x200D) { return (char)0xFF; } + else if (c == 0x200C) { return (char)0xFE; } + int32_t delta = c - transformConstant; + if (delta < 0 || 0xFD < delta) { + fprintf(stderr, "Codepoint U+%04lx out of range for --transform offset-%04lx!\n", + (long)c, (long)transformConstant); + exit(U_ILLEGAL_ARGUMENT_ERROR); // TODO: should return and print the line number + } + return (char)delta; + } else { // no such transform type + status = U_INTERNAL_PROGRAM_ERROR; + return (char)c; // it should be noted this transform type will not generally work + } + } + + void transform(const UnicodeString &word, CharString &buf, UErrorCode &errorCode) { + UChar32 c = 0; + int32_t len = word.length(); + for (int32_t i = 0; i < len; i += U16_LENGTH(c)) { + c = word.char32At(i); + buf.append(transform(c, errorCode), errorCode); + } + } + +public: + // sets the desired transformation data. + // should be populated from a command line argument + // so far the only acceptable format is offset-<hex constant> + // eventually others (mask-<hex constant>?) may be enabled + // more complex functions may be more difficult + void setTransform(const char *t) { + if (strncmp(t, "offset-", 7) == 0) { + char *end; + unsigned long base = uprv_strtoul(t + 7, &end, 16); + if (end == (t + 7) || *end != 0 || base > 0x10FF80) { + fprintf(stderr, "Syntax for offset value in --transform offset-%s invalid!\n", t + 7); + usageAndDie(U_ILLEGAL_ARGUMENT_ERROR); + } + transformType = DictionaryData::TRANSFORM_TYPE_OFFSET; + transformConstant = (UChar32)base; + } + else { + fprintf(stderr, "Invalid transform specified: %s\n", t); + usageAndDie(U_ILLEGAL_ARGUMENT_ERROR); + } + } + + // add a word to the trie + void addWord(const UnicodeString &word, int32_t value, UErrorCode &status) { + if (bt) { + CharString buf; + transform(word, buf, status); + bt->add(buf.toStringPiece(), value, status); + } + if (ut) { ut->add(word, value, status); } + } + + // if we are a bytestrie, give back the StringPiece representing the serialized version of us + StringPiece serializeBytes(UErrorCode &status) { + return bt->buildStringPiece(USTRINGTRIE_BUILD_SMALL, status); + } + + // if we are a ucharstrie, produce the UnicodeString representing the serialized version of us + void serializeUChars(UnicodeString &s, UErrorCode &status) { + ut->buildUnicodeString(USTRINGTRIE_BUILD_SMALL, s, status); + } + + int32_t getTransform() { + return (int32_t)(transformType | transformConstant); + } +}; +#endif + +static const char16_t LINEFEED_CHARACTER = 0x000A; +static const char16_t CARRIAGE_RETURN_CHARACTER = 0x000D; + +static UBool readLine(UCHARBUF *f, UnicodeString &fileLine, IcuToolErrorCode &errorCode) { + int32_t lineLength; + const char16_t *line = ucbuf_readline(f, &lineLength, errorCode); + if(line == nullptr || errorCode.isFailure()) { return false; } + // Strip trailing CR/LF, comments, and spaces. + const char16_t *comment = u_memchr(line, 0x23, lineLength); // '#' + if(comment != nullptr) { + lineLength = (int32_t)(comment - line); + } else { + while(lineLength > 0 && (line[lineLength - 1] == CARRIAGE_RETURN_CHARACTER || line[lineLength - 1] == LINEFEED_CHARACTER)) { --lineLength; } + } + while(lineLength > 0 && u_isspace(line[lineLength - 1])) { --lineLength; } + fileLine.setTo(false, line, lineLength); + return true; +} + +//---------------------------------------------------------------------------- +// +// main for gendict +// +//---------------------------------------------------------------------------- +int main(int argc, char **argv) { + // + // Pick up and check the command line arguments, + // using the standard ICU tool utils option handling. + // + U_MAIN_INIT_ARGS(argc, argv); + progName = argv[0]; + argc=u_parseArgs(argc, argv, UPRV_LENGTHOF(options), options); + if(argc<0) { + // Unrecognized option + fprintf(stderr, "error in command line argument \"%s\"\n", argv[-argc]); + usageAndDie(U_ILLEGAL_ARGUMENT_ERROR); + } + + if(options[ARG_HELP].doesOccur || options[ARG_QMARK].doesOccur) { + // -? or -h for help. + usageAndDie(U_ZERO_ERROR); + } + + UBool verbose = options[ARG_VERBOSE].doesOccur; + UBool quiet = options[ARG_QUIET].doesOccur; + + if (argc < 3) { + fprintf(stderr, "input and output file must both be specified.\n"); + usageAndDie(U_ILLEGAL_ARGUMENT_ERROR); + } + const char *outFileName = argv[2]; + const char *wordFileName = argv[1]; + + startTime = uprv_getRawUTCtime(); // initialize start timer + + if (options[ARG_ICUDATADIR].doesOccur) { + u_setDataDirectory(options[ARG_ICUDATADIR].value); + } + + const char *copyright = nullptr; + if (options[ARG_COPYRIGHT].doesOccur) { + copyright = U_COPYRIGHT_STRING; + } + + if (options[ARG_UCHARS].doesOccur == options[ARG_BYTES].doesOccur) { + fprintf(stderr, "you must specify exactly one type of trie to output!\n"); + usageAndDie(U_ILLEGAL_ARGUMENT_ERROR); + } + UBool isBytesTrie = options[ARG_BYTES].doesOccur; + if (isBytesTrie != options[ARG_TRANSFORM].doesOccur) { + fprintf(stderr, "you must provide a transformation for a bytes trie, and must not provide one for a uchars trie!\n"); + usageAndDie(U_ILLEGAL_ARGUMENT_ERROR); + } + + IcuToolErrorCode status("gendict/main()"); + +#if UCONFIG_NO_BREAK_ITERATION || UCONFIG_NO_FILE_IO + const char* outDir=nullptr; + + UNewDataMemory *pData; + char msg[1024]; + UErrorCode tempstatus = U_ZERO_ERROR; + + /* write message with just the name */ // potential for a buffer overflow here... + snprintf(msg, sizeof(msg), "gendict writes dummy %s because of UCONFIG_NO_BREAK_ITERATION and/or UCONFIG_NO_FILE_IO, see uconfig.h", outFileName); + fprintf(stderr, "%s\n", msg); + + /* write the dummy data file */ + pData = udata_create(outDir, nullptr, outFileName, &dataInfo, nullptr, &tempstatus); + udata_writeBlock(pData, msg, strlen(msg)); + udata_finish(pData, &tempstatus); + return (int)tempstatus; + +#else + // Read in the dictionary source file + if (verbose) { printf("Opening file %s...\n", wordFileName); } + const char *codepage = "UTF-8"; + LocalUCHARBUFPointer f(ucbuf_open(wordFileName, &codepage, true, false, status)); + if (status.isFailure()) { + fprintf(stderr, "error opening input file: ICU Error \"%s\"\n", status.errorName()); + exit(status.reset()); + } + if (verbose) { printf("Initializing dictionary builder of type %s...\n", (isBytesTrie ? "BytesTrie" : "UCharsTrie")); } + DataDict dict(isBytesTrie, status); + if (status.isFailure()) { + fprintf(stderr, "new DataDict: ICU Error \"%s\"\n", status.errorName()); + exit(status.reset()); + } + if (options[ARG_TRANSFORM].doesOccur) { + dict.setTransform(options[ARG_TRANSFORM].value); + } + + UnicodeString fileLine; + if (verbose) { puts("Adding words to dictionary..."); } + UBool hasValues = false; + UBool hasValuelessContents = false; + int lineCount = 0; + int wordCount = 0; + int minlen = 255; + int maxlen = 0; + UBool isOk = true; + while (readLine(f.getAlias(), fileLine, status)) { + lineCount++; + if (fileLine.isEmpty()) continue; + + // Parse word [spaces value]. + int32_t keyLen; + for (keyLen = 0; keyLen < fileLine.length() && !u_isspace(fileLine[keyLen]); ++keyLen) {} + if (keyLen == 0) { + fprintf(stderr, "Error: no word on line %i!\n", lineCount); + isOk = false; + continue; + } + int32_t valueStart; + for (valueStart = keyLen; + valueStart < fileLine.length() && u_isspace(fileLine[valueStart]); + ++valueStart) {} + + if (keyLen < valueStart) { + int32_t valueLength = fileLine.length() - valueStart; + if (valueLength > 15) { + fprintf(stderr, "Error: value too long on line %i!\n", lineCount); + isOk = false; + continue; + } + char s[16]; + fileLine.extract(valueStart, valueLength, s, 16, US_INV); + char *end; + unsigned long value = uprv_strtoul(s, &end, 0); + if (end == s || *end != 0 || (int32_t)uprv_strlen(s) != valueLength || value > 0xffffffff) { + fprintf(stderr, "Error: value syntax error or value too large on line %i!\n", lineCount); + isOk = false; + continue; + } + dict.addWord(fileLine.tempSubString(0, keyLen), (int32_t)value, status); + hasValues = true; + wordCount++; + if (keyLen < minlen) minlen = keyLen; + if (keyLen > maxlen) maxlen = keyLen; + } else { + dict.addWord(fileLine.tempSubString(0, keyLen), 0, status); + hasValuelessContents = true; + wordCount++; + if (keyLen < minlen) minlen = keyLen; + if (keyLen > maxlen) maxlen = keyLen; + } + + if (status.isFailure()) { + fprintf(stderr, "ICU Error \"%s\": Failed to add word to trie at input line %d in input file\n", + status.errorName(), lineCount); + exit(status.reset()); + } + } + if (verbose) { printf("Processed %d lines, added %d words, minlen %d, maxlen %d\n", lineCount, wordCount, minlen, maxlen); } + + if (!isOk && status.isSuccess()) { + status.set(U_ILLEGAL_ARGUMENT_ERROR); + } + if (hasValues && hasValuelessContents) { + fprintf(stderr, "warning: file contained both valued and unvalued strings!\n"); + } + + if (verbose) { printf("Serializing data...isBytesTrie? %d\n", isBytesTrie); } + int32_t outDataSize; + const void *outData; + UnicodeString usp; + if (isBytesTrie) { + StringPiece sp = dict.serializeBytes(status); + outDataSize = sp.size(); + outData = sp.data(); + } else { + dict.serializeUChars(usp, status); + outDataSize = usp.length() * U_SIZEOF_UCHAR; + outData = usp.getBuffer(); + } + if (status.isFailure()) { + fprintf(stderr, "gendict: got failure of type %s while serializing, if U_ILLEGAL_ARGUMENT_ERROR possibly due to duplicate dictionary entries\n", status.errorName()); + exit(status.reset()); + } + if (verbose) { puts("Opening output file..."); } + UNewDataMemory *pData = udata_create(nullptr, nullptr, outFileName, &dataInfo, copyright, status); + if (status.isFailure()) { + fprintf(stderr, "gendict: could not open output file \"%s\", \"%s\"\n", outFileName, status.errorName()); + exit(status.reset()); + } + + if (verbose) { puts("Writing to output file..."); } + int32_t indexes[DictionaryData::IX_COUNT] = { + DictionaryData::IX_COUNT * sizeof(int32_t), 0, 0, 0, 0, 0, 0, 0 + }; + int32_t size = outDataSize + indexes[DictionaryData::IX_STRING_TRIE_OFFSET]; + indexes[DictionaryData::IX_RESERVED1_OFFSET] = size; + indexes[DictionaryData::IX_RESERVED2_OFFSET] = size; + indexes[DictionaryData::IX_TOTAL_SIZE] = size; + + indexes[DictionaryData::IX_TRIE_TYPE] = isBytesTrie ? DictionaryData::TRIE_TYPE_BYTES : DictionaryData::TRIE_TYPE_UCHARS; + if (hasValues) { + indexes[DictionaryData::IX_TRIE_TYPE] |= DictionaryData::TRIE_HAS_VALUES; + } + + indexes[DictionaryData::IX_TRANSFORM] = dict.getTransform(); + udata_writeBlock(pData, indexes, sizeof(indexes)); + udata_writeBlock(pData, outData, outDataSize); + size_t bytesWritten = udata_finish(pData, status); + if (status.isFailure()) { + fprintf(stderr, "gendict: error \"%s\" writing the output file\n", status.errorName()); + exit(status.reset()); + } + + if (bytesWritten != (size_t)size) { + fprintf(stderr, "Error writing to output file \"%s\"\n", outFileName); + exit(U_INTERNAL_PROGRAM_ERROR); + } + + if (!quiet) { printf("%s: done writing\t%s (%ds).\n", progName, outFileName, elapsedTime()); } + +#ifdef TEST_GENDICT + if (isBytesTrie) { + BytesTrie::Iterator it(outData, outDataSize, status); + while (it.hasNext()) { + it.next(status); + const StringPiece s = it.getString(); + int32_t val = it.getValue(); + printf("%s -> %i\n", s.data(), val); + } + } else { + UCharsTrie::Iterator it((const char16_t *)outData, outDataSize, status); + while (it.hasNext()) { + it.next(status); + const UnicodeString s = it.getString(); + int32_t val = it.getValue(); + char tmp[1024]; + s.extract(0, s.length(), tmp, 1024); + printf("%s -> %i\n", tmp, val); + } + } +#endif + + return 0; +#endif /* #if !UCONFIG_NO_BREAK_ITERATION */ +} diff --git a/intl/icu/source/tools/gendict/gendict.vcxproj b/intl/icu/source/tools/gendict/gendict.vcxproj new file mode 100644 index 0000000000..42c1ba7869 --- /dev/null +++ b/intl/icu/source/tools/gendict/gendict.vcxproj @@ -0,0 +1,80 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <PropertyGroup Label="Globals"> + <ProjectGuid>{9D4211F7-2C77-439C-82F0-30A4E43BA569}</ProjectGuid> + </PropertyGroup> + <PropertyGroup Label="Configuration"> + <ConfigurationType>Application</ConfigurationType> + <UseOfMfc>false</UseOfMfc> + <CharacterSet>MultiByte</CharacterSet> + </PropertyGroup> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" /> + <!-- The following import will include the 'default' configuration options for VS projects. --> + <Import Project="..\..\allinone\Build.Windows.ProjectConfiguration.props" /> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> + <ImportGroup Label="ExtensionSettings"> + </ImportGroup> + <PropertyGroup Label="UserMacros" /> + <PropertyGroup> + <_ProjectFileVersion>10.0.30319.1</_ProjectFileVersion> + <OutDir>.\$(Platform)\$(Configuration)\</OutDir> + <IntDir>.\$(Platform)\$(Configuration)\</IntDir> + <!-- The ICU projects use "Win32" to mean "x86", so we need to special case it. --> + <OutDir Condition="'$(Platform)'=='Win32'">.\x86\$(Configuration)\</OutDir> + <IntDir Condition="'$(Platform)'=='Win32'">.\x86\$(Configuration)\</IntDir> + <!-- Disable Incremental Linking for Release builds as it prevents Link-time Code Generation --> + <LinkIncremental Condition="'$(Configuration)'=='Debug'">true</LinkIncremental> + <LinkIncremental Condition="'$(Configuration)'=='Release'">false</LinkIncremental> + </PropertyGroup> + <!-- Options that are common to *all* configurations --> + <ItemDefinitionGroup> + <Midl> + <TypeLibraryName>$(OutDir)\gendict.tlb</TypeLibraryName> + </Midl> + <ClCompile> + <WarningLevel>Level3</WarningLevel> + <CompileAs>Default</CompileAs> + <DisableLanguageExtensions>true</DisableLanguageExtensions> + <AdditionalIncludeDirectories>..\..\common;..\toolutil;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <PrecompiledHeaderOutputFile>$(OutDir)\gendict.pch</PrecompiledHeaderOutputFile> + <AssemblerListingLocation>$(OutDir)/</AssemblerListingLocation> + <ObjectFileName>$(OutDir)/</ObjectFileName> + <ProgramDataBaseFileName>$(OutDir)\gendict.pdb</ProgramDataBaseFileName> + </ClCompile> + <Link> + <SubSystem>Console</SubSystem> + <OutputFile>$(OutDir)\gendict.exe</OutputFile> + <AdditionalLibraryDirectories>..\..\..\$(IcuLibOutputDir);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories> + </Link> + <CustomBuildStep> + <Command>copy "$(TargetPath)" ..\..\..\$(IcuBinOutputDir)</Command> + <Outputs>..\..\..\$(IcuBinOutputDir)\$(TargetFileName);%(Outputs)</Outputs> + </CustomBuildStep> + </ItemDefinitionGroup> + <!-- Options that are common to all 'Debug' project configurations --> + <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'"> + <ClCompile> + <BrowseInformation>true</BrowseInformation> + <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary> + </ClCompile> + <Link> + <AdditionalDependencies>icuucd.lib;icutud.lib;%(AdditionalDependencies)</AdditionalDependencies> + </Link> + </ItemDefinitionGroup> + <!-- Options that are common to all 'Release' project configurations --> + <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'"> + <ClCompile> + <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary> + <FunctionLevelLinking>true</FunctionLevelLinking> + </ClCompile> + <Link> + <AdditionalDependencies>icuuc.lib;icutu.lib;%(AdditionalDependencies)</AdditionalDependencies> + </Link> + </ItemDefinitionGroup> + <ItemGroup> + <ClCompile Include="gendict.cpp" /> + </ItemGroup> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> + <ImportGroup Label="ExtensionTargets"> + </ImportGroup> +</Project>
\ No newline at end of file diff --git a/intl/icu/source/tools/gendict/gendict.vcxproj.filters b/intl/icu/source/tools/gendict/gendict.vcxproj.filters new file mode 100644 index 0000000000..52823d6f9a --- /dev/null +++ b/intl/icu/source/tools/gendict/gendict.vcxproj.filters @@ -0,0 +1,22 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <ItemGroup> + <Filter Include="Source Files"> + <UniqueIdentifier>{570fb8ae-ac18-467d-8502-470a241a60d4}</UniqueIdentifier> + <Extensions>cpp;c;cxx;rc;def;r;odl;idl;hpj;bat</Extensions> + </Filter> + <Filter Include="Header Files"> + <UniqueIdentifier>{7b2185f2-4ff9-4419-b596-0a21e37414c9}</UniqueIdentifier> + <Extensions>h;hpp;hxx;hm;inl</Extensions> + </Filter> + <Filter Include="Resource Files"> + <UniqueIdentifier>{1dc5e7e3-4d1b-4031-a31f-c39b3a3e283a}</UniqueIdentifier> + <Extensions>ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe</Extensions> + </Filter> + </ItemGroup> + <ItemGroup> + <ClCompile Include="gendict.cpp"> + <Filter>Source Files</Filter> + </ClCompile> + </ItemGroup> +</Project> diff --git a/intl/icu/source/tools/gendict/sources.txt b/intl/icu/source/tools/gendict/sources.txt new file mode 100644 index 0000000000..6647c5c2c8 --- /dev/null +++ b/intl/icu/source/tools/gendict/sources.txt @@ -0,0 +1 @@ +gendict.cpp |