diff options
Diffstat (limited to 'intl/icu/source/tools/gennorm2')
| -rw-r--r-- | intl/icu/source/tools/gennorm2/BUILD.bazel | 39 | ||||
| -rw-r--r-- | intl/icu/source/tools/gennorm2/Makefile.in | 82 | ||||
| -rw-r--r-- | intl/icu/source/tools/gennorm2/extradata.cpp | 254 | ||||
| -rw-r--r-- | intl/icu/source/tools/gennorm2/extradata.h | 70 | ||||
| -rw-r--r-- | intl/icu/source/tools/gennorm2/gennorm2.cpp | 333 | ||||
| -rw-r--r-- | intl/icu/source/tools/gennorm2/gennorm2.vcxproj | 103 | ||||
| -rw-r--r-- | intl/icu/source/tools/gennorm2/n2builder.cpp | 1051 | ||||
| -rw-r--r-- | intl/icu/source/tools/gennorm2/n2builder.h | 122 | ||||
| -rw-r--r-- | intl/icu/source/tools/gennorm2/norms.cpp | 324 | ||||
| -rw-r--r-- | intl/icu/source/tools/gennorm2/norms.h | 215 | ||||
| -rw-r--r-- | intl/icu/source/tools/gennorm2/sources.txt | 4 |
11 files changed, 2597 insertions, 0 deletions
diff --git a/intl/icu/source/tools/gennorm2/BUILD.bazel b/intl/icu/source/tools/gennorm2/BUILD.bazel new file mode 100644 index 0000000000..c602897baf --- /dev/null +++ b/intl/icu/source/tools/gennorm2/BUILD.bazel @@ -0,0 +1,39 @@ +# © 2021 and later: Unicode, Inc. and others. +# License & terms of use: http://www.unicode.org/copyright.html + +# This Bazel build file defines a target for the gennorm2 binary that generates +# headers needed for bootstrapping the ICU4C build process in a way that +# integrates the normalization data. + +load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library") + +package( + default_visibility = ["//visibility:public"], +) + +cc_binary( + name = "gennorm2", + srcs = glob([ + "*.c", + "*.cpp", + "*.h", # cannot have hdrs section in cc_binary + ]), + deps = [ + "//icu4c/source/common:uhash", + "//icu4c/source/common:umutablecptrie", + "//icu4c/source/common:ucptrie", + "//icu4c/source/common:errorcode", + "//icu4c/source/common:uniset", + "//icu4c/source/common:uvector32", + + "//icu4c/source/common:platform", + "//icu4c/source/common:headers", + + "//icu4c/source/tools/toolutil:toolutil", + "//icu4c/source/tools/toolutil:unewdata", + "//icu4c/source/tools/toolutil:writesrc", + "//icu4c/source/tools/toolutil:uoptions", + "//icu4c/source/tools/toolutil:uparse", + ], + linkopts = ["-pthread"], +) diff --git a/intl/icu/source/tools/gennorm2/Makefile.in b/intl/icu/source/tools/gennorm2/Makefile.in new file mode 100644 index 0000000000..84f5830e67 --- /dev/null +++ b/intl/icu/source/tools/gennorm2/Makefile.in @@ -0,0 +1,82 @@ +## Makefile.in for ICU - tools/gennorm2 +## Copyright (C) 2016 and later: Unicode, Inc. and others. +## License & terms of use: http://www.unicode.org/copyright.html +## Copyright (c) 2009-2011, International Business Machines Corporation and +## others. All Rights Reserved. +## Steven R. Loomis/Markus W. Scherer + +## Source directory information +srcdir = @srcdir@ +top_srcdir = @top_srcdir@ + +top_builddir = ../.. + +include $(top_builddir)/icudefs.mk + +## Build directory information +subdir = tools/gennorm2 + +TARGET_STUB_NAME = gennorm2 + +## Extra files to remove for 'make clean' +CLEANFILES = *~ $(DEPS) + +## Target information +TARGET = $(BINDIR)/$(TARGET_STUB_NAME)$(EXEEXT) + +CPPFLAGS += -I$(srcdir) -I$(top_srcdir)/common -I$(srcdir)/../toolutil +LIBS = $(LIBICUTOOLUTIL) $(LIBICUI18N) $(LIBICUUC) $(DEFAULT_LIBS) $(LIB_M) + +SOURCES = $(shell cat $(srcdir)/sources.txt) +OBJECTS = $(SOURCES:.cpp=.o) + +DEPS = $(OBJECTS:.o=.d) + +## List of phony targets +.PHONY : all all-local install install-local clean clean-local \ +distclean distclean-local dist dist-local check check-local install-man + +## Clear suffix list +.SUFFIXES : + +## List of standard targets +all: all-local +install: install-local +clean: clean-local +distclean : distclean-local +dist: dist-local +check: all check-local + +all-local: $(TARGET) + +install-local: all-local + $(MKINSTALLDIRS) $(DESTDIR)$(sbindir) + $(INSTALL) $(TARGET) $(DESTDIR)$(sbindir) + +dist-local: + +clean-local: + test -z "$(CLEANFILES)" || $(RMV) $(CLEANFILES) + $(RMV) $(TARGET) $(OBJECTS) + +distclean-local: clean-local + $(RMV) Makefile + +check-local: all-local + +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + cd $(top_builddir) \ + && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= $(SHELL) ./config.status + +$(TARGET) : $(OBJECTS) + $(LINK.cc) $(OUTOPT)$@ $^ $(LIBS) + $(POST_BUILD_STEP) + + +ifeq (,$(MAKECMDGOALS)) +-include $(DEPS) +else +ifneq ($(patsubst %clean,,$(MAKECMDGOALS)),) +-include $(DEPS) +endif +endif diff --git a/intl/icu/source/tools/gennorm2/extradata.cpp b/intl/icu/source/tools/gennorm2/extradata.cpp new file mode 100644 index 0000000000..f31bc418ea --- /dev/null +++ b/intl/icu/source/tools/gennorm2/extradata.cpp @@ -0,0 +1,254 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +// extradata.cpp +// created: 2017jun04 Markus W. Scherer +// (pulled out of n2builder.cpp) + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_NORMALIZATION + +#include <stdio.h> +#include <stdlib.h> +#include "unicode/errorcode.h" +#include "unicode/unistr.h" +#include "unicode/utf16.h" +#include "extradata.h" +#include "normalizer2impl.h" +#include "norms.h" +#include "toolutil.h" +#include "utrie2.h" +#include "uvectr32.h" + +U_NAMESPACE_BEGIN + +ExtraData::ExtraData(Norms &n, UBool fast) : + Norms::Enumerator(n), + yesYesCompositions(1000, (UChar32)0xffff, 2), // 0=inert, 1=Jamo L, 2=start of compositions + yesNoMappingsAndCompositions(1000, (UChar32)0, 1), // 0=Hangul LV, 1=start of normal data + yesNoMappingsOnly(1000, (UChar32)0, 1), // 0=Hangul LVT, 1=start of normal data + optimizeFast(fast) { + // Hangul LV algorithmically decomposes to two Jamo. + // Some code may harmlessly read this firstUnit. + yesNoMappingsAndCompositions.setCharAt(0, 2); + // Hangul LVT algorithmically decomposes to three Jamo. + // Some code may harmlessly read this firstUnit. + yesNoMappingsOnly.setCharAt(0, 3); +} + +int32_t ExtraData::writeMapping(UChar32 c, const Norm &norm, UnicodeString &dataString) { + UnicodeString &m=*norm.mapping; + int32_t length=m.length(); + // Write the mapping & raw mapping extraData. + int32_t firstUnit=length|(norm.trailCC<<8); + int32_t preMappingLength=0; + if(norm.rawMapping!=nullptr) { + UnicodeString &rm=*norm.rawMapping; + int32_t rmLength=rm.length(); + if(rmLength>Normalizer2Impl::MAPPING_LENGTH_MASK) { + fprintf(stderr, + "gennorm2 error: " + "raw mapping for U+%04lX longer than maximum of %d\n", + (long)c, Normalizer2Impl::MAPPING_LENGTH_MASK); + exit(U_INVALID_FORMAT_ERROR); + } + char16_t rm0=rm.charAt(0); + if( rmLength==length-1 && + // 99: overlong substring lengths get pinned to remainder lengths anyway + 0==rm.compare(1, 99, m, 2, 99) && + rm0>Normalizer2Impl::MAPPING_LENGTH_MASK + ) { + // Compression: + // rawMapping=rm0+mapping.substring(2) -> store only rm0 + // + // The raw mapping is the same as the final mapping after replacing + // the final mapping's first two code units with the raw mapping's first one. + // In this case, we store only that first unit, rm0. + // This helps with a few hundred mappings. + dataString.append(rm0); + preMappingLength=1; + } else { + // Store the raw mapping with its length. + dataString.append(rm); + dataString.append((char16_t)rmLength); + preMappingLength=rmLength+1; + } + firstUnit|=Normalizer2Impl::MAPPING_HAS_RAW_MAPPING; + } + int32_t cccLccc=norm.cc|(norm.leadCC<<8); + if(cccLccc!=0) { + dataString.append((char16_t)cccLccc); + ++preMappingLength; + firstUnit|=Normalizer2Impl::MAPPING_HAS_CCC_LCCC_WORD; + } + dataString.append((char16_t)firstUnit); + dataString.append(m); + return preMappingLength; +} + +int32_t ExtraData::writeNoNoMapping(UChar32 c, const Norm &norm, + UnicodeString &dataString, + Hashtable &previousMappings) { + UnicodeString newMapping; + int32_t offset=writeMapping(c, norm, newMapping); + UBool found=false; + int32_t previousOffset=previousMappings.getiAndFound(newMapping, found); + if(found) { + // Duplicate, point to the identical mapping that has already been stored. + offset=previousOffset; + } else { + // Append this new mapping and + // enter it into the hashtable, avoiding value 0 which is "not found". + offset=dataString.length()+offset; + dataString.append(newMapping); + IcuToolErrorCode errorCode("gennorm2/writeExtraData()/Hashtable.putiAllowZero()"); + previousMappings.putiAllowZero(newMapping, offset, errorCode); + } + return offset; +} + +UBool ExtraData::setNoNoDelta(UChar32 c, Norm &norm) const { + // Try a compact, algorithmic encoding to a single compYesAndZeroCC code point. + // Do not map from ASCII to non-ASCII. + if(norm.mappingCP>=0 && + !(c<=0x7f && norm.mappingCP>0x7f) && + norms.getNormRef(norm.mappingCP).type<Norm::NO_NO_COMP_YES) { + int32_t delta=norm.mappingCP-c; + if(-Normalizer2Impl::MAX_DELTA<=delta && delta<=Normalizer2Impl::MAX_DELTA) { + norm.type=Norm::NO_NO_DELTA; + norm.offset=delta; + return true; + } + } + return false; +} + +void ExtraData::writeCompositions(UChar32 c, const Norm &norm, UnicodeString &dataString) { + if(norm.cc!=0) { + fprintf(stderr, + "gennorm2 error: " + "U+%04lX combines-forward and has ccc!=0, not possible in Unicode normalization\n", + (long)c); + exit(U_INVALID_FORMAT_ERROR); + } + int32_t length; + const CompositionPair *pairs=norm.getCompositionPairs(length); + for(int32_t i=0; i<length; ++i) { + const CompositionPair &pair=pairs[i]; + // 22 bits for the composite character and whether it combines forward. + UChar32 compositeAndFwd=pair.composite<<1; + if(norms.getNormRef(pair.composite).compositions!=nullptr) { + compositeAndFwd|=1; // The composite character also combines-forward. + } + // Encode most pairs in two units and some in three. + int32_t firstUnit, secondUnit, thirdUnit; + if(pair.trail<Normalizer2Impl::COMP_1_TRAIL_LIMIT) { + if(compositeAndFwd<=0xffff) { + firstUnit=pair.trail<<1; + secondUnit=compositeAndFwd; + thirdUnit=-1; + } else { + firstUnit=(pair.trail<<1)|Normalizer2Impl::COMP_1_TRIPLE; + secondUnit=compositeAndFwd>>16; + thirdUnit=compositeAndFwd; + } + } else { + firstUnit=(Normalizer2Impl::COMP_1_TRAIL_LIMIT+ + (pair.trail>>Normalizer2Impl::COMP_1_TRAIL_SHIFT))| + Normalizer2Impl::COMP_1_TRIPLE; + secondUnit=(pair.trail<<Normalizer2Impl::COMP_2_TRAIL_SHIFT)| + (compositeAndFwd>>16); + thirdUnit=compositeAndFwd; + } + // Set the high bit of the first unit if this is the last composition pair. + if(i==(length-1)) { + firstUnit|=Normalizer2Impl::COMP_1_LAST_TUPLE; + } + dataString.append((char16_t)firstUnit).append((char16_t)secondUnit); + if(thirdUnit>=0) { + dataString.append((char16_t)thirdUnit); + } + } +} + +void ExtraData::rangeHandler(UChar32 start, UChar32 end, Norm &norm) { + if(start!=end) { + fprintf(stderr, + "gennorm2 error: unexpected shared data for " + "multiple code points U+%04lX..U+%04lX\n", + (long)start, (long)end); + exit(U_INTERNAL_PROGRAM_ERROR); + } + if(norm.error!=nullptr) { + fprintf(stderr, "gennorm2 error: U+%04lX %s\n", (long)start, norm.error); + exit(U_INVALID_FORMAT_ERROR); + } + writeExtraData(start, norm); +} + +// Ticket #13342 - Disable optimizations on MSVC for this function as a workaround. +#if (defined(_MSC_VER) && (_MSC_VER >= 1900) && defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 190024210)) +#pragma optimize( "", off ) +#endif + +void ExtraData::writeExtraData(UChar32 c, Norm &norm) { + switch(norm.type) { + case Norm::INERT: + break; // no extra data + case Norm::YES_YES_COMBINES_FWD: + norm.offset=yesYesCompositions.length(); + writeCompositions(c, norm, yesYesCompositions); + break; + case Norm::YES_NO_COMBINES_FWD: + norm.offset=yesNoMappingsAndCompositions.length()+ + writeMapping(c, norm, yesNoMappingsAndCompositions); + writeCompositions(c, norm, yesNoMappingsAndCompositions); + break; + case Norm::YES_NO_MAPPING_ONLY: + norm.offset=yesNoMappingsOnly.length()+ + writeMapping(c, norm, yesNoMappingsOnly); + break; + case Norm::NO_NO_COMP_YES: + if(!optimizeFast && setNoNoDelta(c, norm)) { + break; + } + norm.offset=writeNoNoMapping(c, norm, noNoMappingsCompYes, previousNoNoMappingsCompYes); + break; + case Norm::NO_NO_COMP_BOUNDARY_BEFORE: + if(!optimizeFast && setNoNoDelta(c, norm)) { + break; + } + norm.offset=writeNoNoMapping( + c, norm, noNoMappingsCompBoundaryBefore, previousNoNoMappingsCompBoundaryBefore); + break; + case Norm::NO_NO_COMP_NO_MAYBE_CC: + norm.offset=writeNoNoMapping( + c, norm, noNoMappingsCompNoMaybeCC, previousNoNoMappingsCompNoMaybeCC); + break; + case Norm::NO_NO_EMPTY: + // There can be multiple extra data entries for mappings to the empty string + // if they have different raw mappings. + norm.offset=writeNoNoMapping(c, norm, noNoMappingsEmpty, previousNoNoMappingsEmpty); + break; + case Norm::MAYBE_YES_COMBINES_FWD: + norm.offset=maybeYesCompositions.length(); + writeCompositions(c, norm, maybeYesCompositions); + break; + case Norm::MAYBE_YES_SIMPLE: + break; // no extra data + case Norm::YES_YES_WITH_CC: + break; // no extra data + default: // Should not occur. + exit(U_INTERNAL_PROGRAM_ERROR); + } +} + +// Ticket #13342 - Turn optimization back on. +#if (defined(_MSC_VER) && (_MSC_VER >= 1900) && defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 190024210)) +#pragma optimize( "", on ) +#endif + +U_NAMESPACE_END + +#endif // #if !UCONFIG_NO_NORMALIZATION diff --git a/intl/icu/source/tools/gennorm2/extradata.h b/intl/icu/source/tools/gennorm2/extradata.h new file mode 100644 index 0000000000..0a8e73087d --- /dev/null +++ b/intl/icu/source/tools/gennorm2/extradata.h @@ -0,0 +1,70 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +// extradata.h +// created: 2017jun04 Markus W. Scherer +// (pulled out of n2builder.cpp) + +// Write mappings and compositions in compact form for Normalizer2 "extra data", +// the data that does not fit into the trie itself. + +#ifndef __EXTRADATA_H__ +#define __EXTRADATA_H__ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_NORMALIZATION + +#include "unicode/errorcode.h" +#include "unicode/unistr.h" +#include "unicode/utf16.h" +#include "hash.h" +#include "norms.h" +#include "toolutil.h" +#include "utrie2.h" +#include "uvectr32.h" + +U_NAMESPACE_BEGIN + +class ExtraData : public Norms::Enumerator { +public: + ExtraData(Norms &n, UBool fast); + + void rangeHandler(UChar32 start, UChar32 end, Norm &norm) override; + + UnicodeString maybeYesCompositions; + UnicodeString yesYesCompositions; + UnicodeString yesNoMappingsAndCompositions; + UnicodeString yesNoMappingsOnly; + UnicodeString noNoMappingsCompYes; + UnicodeString noNoMappingsCompBoundaryBefore; + UnicodeString noNoMappingsCompNoMaybeCC; + UnicodeString noNoMappingsEmpty; + +private: + /** + * Requires norm.hasMapping(). + * Returns the offset of the "first unit" from the beginning of the extraData for c. + * That is the same as the length of the optional data + * for the raw mapping and the ccc/lccc word. + */ + int32_t writeMapping(UChar32 c, const Norm &norm, UnicodeString &dataString); + int32_t writeNoNoMapping(UChar32 c, const Norm &norm, + UnicodeString &dataString, Hashtable &previousMappings); + UBool setNoNoDelta(UChar32 c, Norm &norm) const; + /** Requires norm.compositions!=nullptr. */ + void writeCompositions(UChar32 c, const Norm &norm, UnicodeString &dataString); + void writeExtraData(UChar32 c, Norm &norm); + + UBool optimizeFast; + Hashtable previousNoNoMappingsCompYes; // If constructed in runtime code, pass in UErrorCode. + Hashtable previousNoNoMappingsCompBoundaryBefore; + Hashtable previousNoNoMappingsCompNoMaybeCC; + Hashtable previousNoNoMappingsEmpty; +}; + +U_NAMESPACE_END + +#endif // #if !UCONFIG_NO_NORMALIZATION + +#endif // __EXTRADATA_H__ diff --git a/intl/icu/source/tools/gennorm2/gennorm2.cpp b/intl/icu/source/tools/gennorm2/gennorm2.cpp new file mode 100644 index 0000000000..2575bf7ed8 --- /dev/null +++ b/intl/icu/source/tools/gennorm2/gennorm2.cpp @@ -0,0 +1,333 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* +* Copyright (C) 2009-2014, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: gennorm2.cpp +* encoding: UTF-8 +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2009nov25 +* created by: Markus W. Scherer +* +* This program reads text files that define Unicode normalization, +* parses them, and builds a binary data file. +*/ + +#include "unicode/utypes.h" +#include "n2builder.h" + +#include <fstream> +#include <stdio.h> +#include <stdlib.h> +#include <string> +#include <string.h> +#include "unicode/errorcode.h" +#include "unicode/localpointer.h" +#include "unicode/putil.h" +#include "unicode/uchar.h" +#include "unicode/unistr.h" +#include "charstr.h" +#include "normalizer2impl.h" +#include "toolutil.h" +#include "uoptions.h" +#include "uparse.h" + +#if UCONFIG_NO_NORMALIZATION +#include "unewdata.h" +#endif + +U_NAMESPACE_BEGIN + +UBool beVerbose=false, haveCopyright=true; + +#if !UCONFIG_NO_NORMALIZATION +void parseFile(std::ifstream &f, Normalizer2DataBuilder &builder); +#endif + +/* -------------------------------------------------------------------------- */ + +enum { + HELP_H, + HELP_QUESTION_MARK, + VERBOSE, + COPYRIGHT, + SOURCEDIR, + OUTPUT_FILENAME, + UNICODE_VERSION, + WRITE_C_SOURCE, + WRITE_COMBINED_DATA, + OPT_FAST +}; + +static UOption options[]={ + UOPTION_HELP_H, + UOPTION_HELP_QUESTION_MARK, + UOPTION_VERBOSE, + UOPTION_COPYRIGHT, + UOPTION_SOURCEDIR, + UOPTION_DEF("output", 'o', UOPT_REQUIRES_ARG), + UOPTION_DEF("unicode", 'u', UOPT_REQUIRES_ARG), + UOPTION_DEF("csource", '\1', UOPT_NO_ARG), + UOPTION_DEF("combined", '\1', UOPT_NO_ARG), + UOPTION_DEF("fast", '\1', UOPT_NO_ARG) +}; + +extern "C" int +main(int argc, char* argv[]) { + U_MAIN_INIT_ARGS(argc, argv); + + /* preset then read command line options */ + options[SOURCEDIR].value=""; + argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[HELP_H]), options); + + /* error handling, printing usage message */ + if(argc<0) { + fprintf(stderr, + "error in command line argument \"%s\"\n", + argv[-argc]); + } + if(!options[OUTPUT_FILENAME].doesOccur) { + argc=-1; + } + if( argc<2 || + options[HELP_H].doesOccur || options[HELP_QUESTION_MARK].doesOccur + ) { + fprintf(stderr, + "Usage: %s [-options] infiles+ -o outputfilename\n" + "\n" + "Reads the infiles with normalization data and\n" + "creates a binary file, or a C source file (--csource), with the data,\n" + "or writes a data file with the combined data (--combined).\n" + "See https://unicode-org.github.io/icu/userguide/transforms/normalization#data-file-syntax\n" + "\n" + "Alternate usage: %s [-options] a.txt b.txt minus p.txt q.txt -o outputfilename\n" + "\n" + "Computes the difference of (a, b) minus (p, q) and writes the diff data\n" + "in input-file syntax to the outputfilename.\n" + "It is then possible to build (p, q, diff) to get the same data as (a, b).\n" + "(Useful for computing minimal incremental mapping data files.)\n" + "\n", + argv[0], argv[0]); + fprintf(stderr, + "Options:\n" + "\t-h or -? or --help this usage text\n" + "\t-v or --verbose verbose output\n" + "\t-c or --copyright include a copyright notice\n" + "\t-u or --unicode Unicode version, followed by the version like 5.2.0\n"); + fprintf(stderr, + "\t-s or --sourcedir source directory, followed by the path\n" + "\t-o or --output output filename\n" + "\t --csource writes a C source file with initializers\n" + "\t --combined writes a .txt file (input-file syntax) with the\n" + "\t combined data from all of the input files\n"); + fprintf(stderr, + "\t --fast optimize the data for fast normalization,\n" + "\t which might increase its size (Writes fully decomposed\n" + "\t regular mappings instead of delta mappings.\n" + "\t You should measure the runtime speed to make sure that\n" + "\t this is a good trade-off.)\n"); + return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR; + } + + beVerbose=options[VERBOSE].doesOccur; + haveCopyright=options[COPYRIGHT].doesOccur; + + IcuToolErrorCode errorCode("gennorm2/main()"); + +#if UCONFIG_NO_NORMALIZATION + + fprintf(stderr, + "gennorm2 writes a dummy binary data file " + "because UCONFIG_NO_NORMALIZATION is set, \n" + "see icu/source/common/unicode/uconfig.h\n"); + udata_createDummy(nullptr, nullptr, options[OUTPUT_FILENAME].value, errorCode); + // Should not return an error since this is the expected behaviour if UCONFIG_NO_NORMALIZATION is on. + // return U_UNSUPPORTED_ERROR; + return 0; + +#else + + LocalPointer<Normalizer2DataBuilder> b1(new Normalizer2DataBuilder(errorCode), errorCode); + LocalPointer<Normalizer2DataBuilder> b2; + LocalPointer<Normalizer2DataBuilder> diff; + Normalizer2DataBuilder *builder = b1.getAlias(); + errorCode.assertSuccess(); + + if(options[UNICODE_VERSION].doesOccur) { + builder->setUnicodeVersion(options[UNICODE_VERSION].value); + } + + if(options[OPT_FAST].doesOccur) { + builder->setOptimization(Normalizer2DataBuilder::OPTIMIZE_FAST); + } + + // prepare the filename beginning with the source dir + CharString filename(options[SOURCEDIR].value, errorCode); + int32_t pathLength=filename.length(); + if( pathLength>0 && + filename[pathLength-1]!=U_FILE_SEP_CHAR && + filename[pathLength-1]!=U_FILE_ALT_SEP_CHAR + ) { + filename.append(U_FILE_SEP_CHAR, errorCode); + pathLength=filename.length(); + } + + bool doMinus = false; + for(int i=1; i<argc; ++i) { + printf("gennorm2: processing %s\n", argv[i]); + if(strcmp(argv[i], "minus") == 0) { + if(doMinus) { + fprintf(stderr, "gennorm2 error: only one 'minus' can be specified\n"); + exit(U_ILLEGAL_ARGUMENT_ERROR); + } + // Data from previous input files has been collected in b1. + // Collect data from further input files in b2. + b2.adoptInsteadAndCheckErrorCode(new Normalizer2DataBuilder(errorCode), errorCode); + diff.adoptInsteadAndCheckErrorCode(new Normalizer2DataBuilder(errorCode), errorCode); + errorCode.assertSuccess(); + builder = b2.getAlias(); + if(options[UNICODE_VERSION].doesOccur) { + builder->setUnicodeVersion(options[UNICODE_VERSION].value); + } + if(options[OPT_FAST].doesOccur) { + builder->setOptimization(Normalizer2DataBuilder::OPTIMIZE_FAST); + } + doMinus = true; + continue; + } + filename.append(argv[i], errorCode); + std::ifstream f(filename.data()); + if(f.fail()) { + fprintf(stderr, "gennorm2 error: unable to open %s\n", filename.data()); + exit(U_FILE_ACCESS_ERROR); + } + builder->setOverrideHandling(Normalizer2DataBuilder::OVERRIDE_PREVIOUS); + parseFile(f, *builder); + filename.truncate(pathLength); + } + + if(doMinus) { + Normalizer2DataBuilder::computeDiff(*b1, *b2, *diff); + diff->writeDataFile(options[OUTPUT_FILENAME].value, /* writeRemoved= */ true); + } else if(options[WRITE_COMBINED_DATA].doesOccur) { + builder->writeDataFile(options[OUTPUT_FILENAME].value, /* writeRemoved= */ false); + } else if(options[WRITE_C_SOURCE].doesOccur) { + builder->writeCSourceFile(options[OUTPUT_FILENAME].value); + } else { + builder->writeBinaryFile(options[OUTPUT_FILENAME].value); + } + + return errorCode.get(); + +#endif +} + +#if !UCONFIG_NO_NORMALIZATION + +void parseFile(std::ifstream &f, Normalizer2DataBuilder &builder) { + IcuToolErrorCode errorCode("gennorm2/parseFile()"); + std::string lineString; + uint32_t startCP, endCP; + while(std::getline(f, lineString)) { + if (lineString.empty()) { + continue; // skip empty lines. + } + char *line = &lineString.front(); + char *comment=(char *)strchr(line, '#'); + if(comment!=nullptr) { + *comment=0; + } + u_rtrim(line); + if(line[0]==0) { + continue; // skip empty and comment-only lines + } + if(line[0]=='*') { + const char *s=u_skipWhitespace(line+1); + if(0==strncmp(s, "Unicode", 7)) { + s=u_skipWhitespace(s+7); + builder.setUnicodeVersion(s); + } + continue; // reserved syntax + } + const char *delimiter; + int32_t rangeLength= + u_parseCodePointRangeAnyTerminator(line, &startCP, &endCP, &delimiter, errorCode); + if(errorCode.isFailure()) { + fprintf(stderr, "gennorm2 error: parsing code point range from %s\n", line); + exit(errorCode.reset()); + } + if (endCP >= 0xd800 && startCP <= 0xdfff) { + fprintf(stderr, "gennorm2 error: value or mapping for surrogate code points: %s\n", + line); + exit(U_ILLEGAL_ARGUMENT_ERROR); + } + delimiter=u_skipWhitespace(delimiter); + if(*delimiter==':') { + const char *s=u_skipWhitespace(delimiter+1); + char *end; + unsigned long value=strtoul(s, &end, 10); + if(end<=s || *u_skipWhitespace(end)!=0 || value>=0xff) { + fprintf(stderr, "gennorm2 error: parsing ccc from %s\n", line); + exit(U_PARSE_ERROR); + } + for(UChar32 c=(UChar32)startCP; c<=(UChar32)endCP; ++c) { + builder.setCC(c, (uint8_t)value); + } + continue; + } + if(*delimiter=='-') { + if(*u_skipWhitespace(delimiter+1)!=0) { + fprintf(stderr, "gennorm2 error: parsing remove-mapping %s\n", line); + exit(U_PARSE_ERROR); + } + for(UChar32 c=(UChar32)startCP; c<=(UChar32)endCP; ++c) { + builder.removeMapping(c); + } + continue; + } + if(*delimiter=='=' || *delimiter=='>') { + char16_t uchars[Normalizer2Impl::MAPPING_LENGTH_MASK]; + int32_t length=u_parseString(delimiter+1, uchars, UPRV_LENGTHOF(uchars), nullptr, errorCode); + if(errorCode.isFailure()) { + fprintf(stderr, "gennorm2 error: parsing mapping string from %s\n", line); + exit(errorCode.reset()); + } + UnicodeString mapping(false, uchars, length); + if(*delimiter=='=') { + if(rangeLength!=1) { + fprintf(stderr, + "gennorm2 error: round-trip mapping for more than 1 code point on %s\n", + line); + exit(U_PARSE_ERROR); + } + builder.setRoundTripMapping((UChar32)startCP, mapping); + } else { + for(UChar32 c=(UChar32)startCP; c<=(UChar32)endCP; ++c) { + builder.setOneWayMapping(c, mapping); + } + } + continue; + } + fprintf(stderr, "gennorm2 error: unrecognized data line %s\n", line); + exit(U_PARSE_ERROR); + } +} + +#endif // !UCONFIG_NO_NORMALIZATION + +U_NAMESPACE_END + +/* + * Hey, Emacs, please set the following: + * + * Local Variables: + * indent-tabs-mode: nil + * End: + * + */ diff --git a/intl/icu/source/tools/gennorm2/gennorm2.vcxproj b/intl/icu/source/tools/gennorm2/gennorm2.vcxproj new file mode 100644 index 0000000000..2f41299098 --- /dev/null +++ b/intl/icu/source/tools/gennorm2/gennorm2.vcxproj @@ -0,0 +1,103 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <PropertyGroup Label="Globals"> + <ProjectGuid>{C7891A65-80AB-4245-912E-5F1E17B0E6C4}</ProjectGuid> + <RootNamespace>gennorm2</RootNamespace> + </PropertyGroup> + <PropertyGroup Label="Configuration"> + <ConfigurationType>Application</ConfigurationType> + <CharacterSet>Unicode</CharacterSet> + </PropertyGroup> + <PropertyGroup Condition="'$(Configuration)'=='Release'" Label="Configuration"> + <WholeProgramOptimization>true</WholeProgramOptimization> + </PropertyGroup> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" /> + <!-- The following import will include the 'default' configuration options for VS projects. --> + <Import Project="..\..\allinone\Build.Windows.ProjectConfiguration.props" /> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> + <ImportGroup Label="ExtensionSettings"> + </ImportGroup> + <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="PropertySheets"> + <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" /> + </ImportGroup> + <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="PropertySheets"> + <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" /> + </ImportGroup> + <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets"> + <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" /> + </ImportGroup> + <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets"> + <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" /> + </ImportGroup> + <PropertyGroup Label="UserMacros" /> + <PropertyGroup> + <_ProjectFileVersion>10.0.30319.1</_ProjectFileVersion> + <OutDir>.\$(Platform)\$(Configuration)\</OutDir> + <IntDir>.\$(Platform)\$(Configuration)\</IntDir> + <!-- The ICU projects use "Win32" to mean "x86", so we need to special case it. --> + <OutDir Condition="'$(Platform)'=='Win32'">.\x86\$(Configuration)\</OutDir> + <IntDir Condition="'$(Platform)'=='Win32'">.\x86\$(Configuration)\</IntDir> + <!-- Disable Incremental Linking for Release builds as it prevents Link-time Code Generation --> + <LinkIncremental Condition="'$(Configuration)'=='Debug'">true</LinkIncremental> + <LinkIncremental Condition="'$(Configuration)'=='Release'">false</LinkIncremental> + </PropertyGroup> + <!-- Options that are common to *all* configurations --> + <ItemDefinitionGroup> + <Midl> + <TypeLibraryName>$(OutDir)\gennorm2.tlb</TypeLibraryName> + </Midl> + <ClCompile> + <WarningLevel>Level3</WarningLevel> + <CompileAs>Default</CompileAs> + <DisableLanguageExtensions>false</DisableLanguageExtensions> + <AdditionalIncludeDirectories>..\..\common;..\toolutil;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <PrecompiledHeaderOutputFile>$(OutDir)\gennorm2.pch</PrecompiledHeaderOutputFile> + <AssemblerListingLocation>$(OutDir)/</AssemblerListingLocation> + <ObjectFileName>$(OutDir)/</ObjectFileName> + <ProgramDataBaseFileName>$(OutDir)\gennorm2.pdb</ProgramDataBaseFileName> + </ClCompile> + <Link> + <SubSystem>Console</SubSystem> + <OutputFile>$(OutDir)\gennorm2.exe</OutputFile> + <AdditionalLibraryDirectories>..\..\..\$(IcuLibOutputDir);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories> + </Link> + <CustomBuildStep> + <Command>copy "$(TargetPath)" ..\..\..\$(IcuBinOutputDir)</Command> + <Outputs>..\..\..\$(IcuBinOutputDir)\$(TargetFileName);%(Outputs)</Outputs> + </CustomBuildStep> + </ItemDefinitionGroup> + <!-- Options that are common to all 'Debug' project configurations --> + <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'"> + <ClCompile> + <BrowseInformation>true</BrowseInformation> + <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary> + </ClCompile> + <Link> + <AdditionalDependencies>icuucd.lib;icutud.lib;%(AdditionalDependencies)</AdditionalDependencies> + </Link> + </ItemDefinitionGroup> + <!-- Options that are common to all 'Release' project configurations --> + <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'"> + <ClCompile> + <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary> + <FunctionLevelLinking>true</FunctionLevelLinking> + </ClCompile> + <Link> + <AdditionalDependencies>icuuc.lib;icutu.lib;%(AdditionalDependencies)</AdditionalDependencies> + </Link> + </ItemDefinitionGroup> + <ItemGroup> + <ClCompile Include="extradata.cpp" /> + <ClCompile Include="gennorm2.cpp" /> + <ClCompile Include="n2builder.cpp" /> + <ClCompile Include="norms.cpp" /> + </ItemGroup> + <ItemGroup> + <ClInclude Include="extradata.h" /> + <ClInclude Include="n2builder.h" /> + <ClInclude Include="norms.h" /> + </ItemGroup> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> + <ImportGroup Label="ExtensionTargets"> + </ImportGroup> +</Project>
\ No newline at end of file diff --git a/intl/icu/source/tools/gennorm2/n2builder.cpp b/intl/icu/source/tools/gennorm2/n2builder.cpp new file mode 100644 index 0000000000..a07327145d --- /dev/null +++ b/intl/icu/source/tools/gennorm2/n2builder.cpp @@ -0,0 +1,1051 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* +* Copyright (C) 2009-2016, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: n2builder.cpp +* encoding: UTF-8 +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2009nov25 +* created by: Markus W. Scherer +* +* Builds Normalizer2 data and writes a binary .nrm file. +* For the file format see source/common/normalizer2impl.h. +*/ + +#include "unicode/utypes.h" +#include "n2builder.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <vector> +#include "unicode/errorcode.h" +#include "unicode/localpointer.h" +#include "unicode/putil.h" +#include "unicode/ucptrie.h" +#include "unicode/udata.h" +#include "unicode/umutablecptrie.h" +#include "unicode/uniset.h" +#include "unicode/unistr.h" +#include "unicode/usetiter.h" +#include "unicode/ustring.h" +#include "charstr.h" +#include "extradata.h" +#include "hash.h" +#include "normalizer2impl.h" +#include "norms.h" +#include "toolutil.h" +#include "unewdata.h" +#include "uvectr32.h" +#include "writesrc.h" + +#if !UCONFIG_NO_NORMALIZATION + +/* UDataInfo cf. udata.h */ +static UDataInfo dataInfo={ + sizeof(UDataInfo), + 0, + + U_IS_BIG_ENDIAN, + U_CHARSET_FAMILY, + U_SIZEOF_UCHAR, + 0, + + { 0x4e, 0x72, 0x6d, 0x32 }, /* dataFormat="Nrm2" */ + { 4, 0, 0, 0 }, /* formatVersion */ + { 11, 0, 0, 0 } /* dataVersion (Unicode version) */ +}; + +U_NAMESPACE_BEGIN + +class HangulIterator { +public: + struct Range { + UChar32 start, end; + }; + + HangulIterator() : rangeIndex(0) {} + const Range *nextRange() { + if(rangeIndex<UPRV_LENGTHOF(ranges)) { + return ranges+rangeIndex++; + } else { + return nullptr; + } + } +private: + static const Range ranges[4]; + int32_t rangeIndex; +}; + +const HangulIterator::Range HangulIterator::ranges[4]={ + { Hangul::JAMO_L_BASE, Hangul::JAMO_L_END }, + { Hangul::JAMO_V_BASE, Hangul::JAMO_V_END }, + // JAMO_T_BASE+1: not U+11A7 + { Hangul::JAMO_T_BASE+1, Hangul::JAMO_T_END }, + { Hangul::HANGUL_BASE, Hangul::HANGUL_END }, +}; + +Normalizer2DataBuilder::Normalizer2DataBuilder(UErrorCode &errorCode) : + norms(errorCode), + phase(0), overrideHandling(OVERRIDE_PREVIOUS), optimization(OPTIMIZE_NORMAL), + norm16TrieBytes(nullptr), norm16TrieLength(0) { + memset(unicodeVersion, 0, sizeof(unicodeVersion)); + memset(indexes, 0, sizeof(indexes)); + memset(smallFCD, 0, sizeof(smallFCD)); +} + +Normalizer2DataBuilder::~Normalizer2DataBuilder() { + delete[] norm16TrieBytes; +} + +void +Normalizer2DataBuilder::setUnicodeVersion(const char *v) { + UVersionInfo nullVersion={ 0, 0, 0, 0 }; + UVersionInfo version; + u_versionFromString(version, v); + if( 0!=memcmp(version, unicodeVersion, U_MAX_VERSION_LENGTH) && + 0!=memcmp(nullVersion, unicodeVersion, U_MAX_VERSION_LENGTH) + ) { + char buffer[U_MAX_VERSION_STRING_LENGTH]; + u_versionToString(unicodeVersion, buffer); + fprintf(stderr, "gennorm2 error: multiple inconsistent Unicode version numbers %s vs. %s\n", + buffer, v); + exit(U_ILLEGAL_ARGUMENT_ERROR); + } + memcpy(unicodeVersion, version, U_MAX_VERSION_LENGTH); +} + +Norm *Normalizer2DataBuilder::checkNormForMapping(Norm *p, UChar32 c) { + if(p!=nullptr) { + if(p->mappingType!=Norm::NONE) { + if( overrideHandling==OVERRIDE_NONE || + (overrideHandling==OVERRIDE_PREVIOUS && p->mappingPhase==phase) + ) { + fprintf(stderr, + "error in gennorm2 phase %d: " + "not permitted to override mapping for U+%04lX from phase %d\n", + (int)phase, (long)c, (int)p->mappingPhase); + exit(U_INVALID_FORMAT_ERROR); + } + delete p->mapping; + p->mapping=nullptr; + } + p->mappingPhase=phase; + } + return p; +} + +void Normalizer2DataBuilder::setOverrideHandling(OverrideHandling oh) { + overrideHandling=oh; + ++phase; +} + +void Normalizer2DataBuilder::setCC(UChar32 c, uint8_t cc) { + norms.createNorm(c)->cc=cc; + norms.ccSet.add(c); +} + +static UBool isWellFormed(const UnicodeString &s) { + UErrorCode errorCode=U_ZERO_ERROR; + u_strToUTF8(nullptr, 0, nullptr, toUCharPtr(s.getBuffer()), s.length(), &errorCode); + return U_SUCCESS(errorCode) || errorCode==U_BUFFER_OVERFLOW_ERROR; +} + +void Normalizer2DataBuilder::setOneWayMapping(UChar32 c, const UnicodeString &m) { + if(!isWellFormed(m)) { + fprintf(stderr, + "error in gennorm2 phase %d: " + "illegal one-way mapping from U+%04lX to malformed string\n", + (int)phase, (long)c); + exit(U_INVALID_FORMAT_ERROR); + } + Norm *p=checkNormForMapping(norms.createNorm(c), c); + p->mapping=new UnicodeString(m); + p->mappingType=Norm::ONE_WAY; + p->setMappingCP(); + norms.mappingSet.add(c); +} + +void Normalizer2DataBuilder::setRoundTripMapping(UChar32 c, const UnicodeString &m) { + if(U_IS_SURROGATE(c)) { + fprintf(stderr, + "error in gennorm2 phase %d: " + "illegal round-trip mapping from surrogate code point U+%04lX\n", + (int)phase, (long)c); + exit(U_INVALID_FORMAT_ERROR); + } + if(!isWellFormed(m)) { + fprintf(stderr, + "error in gennorm2 phase %d: " + "illegal round-trip mapping from U+%04lX to malformed string\n", + (int)phase, (long)c); + exit(U_INVALID_FORMAT_ERROR); + } + int32_t numCP=u_countChar32(toUCharPtr(m.getBuffer()), m.length()); + if(numCP!=2) { + fprintf(stderr, + "error in gennorm2 phase %d: " + "illegal round-trip mapping from U+%04lX to %d!=2 code points\n", + (int)phase, (long)c, (int)numCP); + exit(U_INVALID_FORMAT_ERROR); + } + Norm *p=checkNormForMapping(norms.createNorm(c), c); + p->mapping=new UnicodeString(m); + p->mappingType=Norm::ROUND_TRIP; + p->mappingCP=U_SENTINEL; + norms.mappingSet.add(c); +} + +void Normalizer2DataBuilder::removeMapping(UChar32 c) { + // createNorm(c), not getNorm(c), to record a non-mapping and detect conflicting data. + Norm *p=checkNormForMapping(norms.createNorm(c), c); + p->mappingType=Norm::REMOVED; + norms.mappingSet.add(c); +} + +UBool Normalizer2DataBuilder::mappingHasCompBoundaryAfter(const BuilderReorderingBuffer &buffer, + Norm::MappingType mappingType) const { + if(buffer.isEmpty()) { + return false; // Maps-to-empty-string is no boundary of any kind. + } + int32_t lastStarterIndex=buffer.lastStarterIndex(); + if(lastStarterIndex<0) { + return false; // no starter + } + const int32_t lastIndex=buffer.length()-1; + if(mappingType==Norm::ONE_WAY && lastStarterIndex<lastIndex && buffer.ccAt(lastIndex)>1) { + // One-way mapping where after the last starter is at least one combining mark + // with a combining class greater than 1, + // which means that another combining mark can reorder before it. + // By contrast, in a round-trip mapping this does not prevent a boundary as long as + // the starter or composite does not combine-forward with a following combining mark. + return false; + } + UChar32 starter=buffer.charAt(lastStarterIndex); + if(lastStarterIndex==0 && norms.combinesBack(starter)) { + // The last starter is at the beginning of the mapping and combines backward. + return false; + } + if(Hangul::isJamoL(starter) || + (Hangul::isJamoV(starter) && + 0<lastStarterIndex && Hangul::isJamoL(buffer.charAt(lastStarterIndex-1)))) { + // A Jamo leading consonant or an LV pair combines-forward if it is at the end, + // otherwise it is blocked. + return lastStarterIndex!=lastIndex; + } + // Note: There can be no Hangul syllable in the fully decomposed mapping. + + // Multiple starters can combine into one. + // Look for the first of the last sequence of starters, excluding Jamos. + int32_t i=lastStarterIndex; + UChar32 c; + while(0<i && buffer.ccAt(i-1)==0 && !Hangul::isJamo(c=buffer.charAt(i-1))) { + starter=c; + --i; + } + // Compose as far as possible, and see if further compositions with + // characters following this mapping are possible. + const Norm *starterNorm=norms.getNorm(starter); + if(i==lastStarterIndex && + (starterNorm==nullptr || starterNorm->compositions==nullptr)) { + return true; // The last starter does not combine forward. + } + uint8_t prevCC=0; + while(++i<buffer.length()) { + uint8_t cc=buffer.ccAt(i); // !=0 if after last starter + if(i>lastStarterIndex && norms.combinesWithCCBetween(*starterNorm, prevCC, cc)) { + // The starter combines with a mark that reorders before the current one. + return false; + } + UChar32 c=buffer.charAt(i); + if(starterNorm!=nullptr && (prevCC<cc || prevCC==0) && + norms.getNormRef(c).combinesBack && (starter=starterNorm->combine(c))>=0) { + // The starter combines with c into a composite replacement starter. + starterNorm=norms.getNorm(starter); + if(i>=lastStarterIndex && + (starterNorm==nullptr || starterNorm->compositions==nullptr)) { + return true; // The composite does not combine further. + } + // Keep prevCC because we "removed" the combining mark. + } else if(cc==0) { + starterNorm=norms.getNorm(c); + if(i==lastStarterIndex && + (starterNorm==nullptr || starterNorm->compositions==nullptr)) { + return true; // The new starter does not combine forward. + } + prevCC=0; + } else { + prevCC=cc; + } + } + if(prevCC==0) { + return false; // forward-combining starter at the very end + } + if(norms.combinesWithCCBetween(*starterNorm, prevCC, 256)) { + // The starter combines with another mark. + return false; + } + return true; +} + +UBool Normalizer2DataBuilder::mappingRecomposes(const BuilderReorderingBuffer &buffer) const { + if(buffer.lastStarterIndex()<0) { + return false; // no starter + } + const Norm *starterNorm=nullptr; + uint8_t prevCC=0; + for(int32_t i=0; i<buffer.length(); ++i) { + UChar32 c=buffer.charAt(i); + uint8_t cc=buffer.ccAt(i); + if(starterNorm!=nullptr && (prevCC<cc || prevCC==0) && + norms.getNormRef(c).combinesBack && starterNorm->combine(c)>=0) { + return true; // normal composite + } else if(cc==0) { + if(Hangul::isJamoL(c)) { + if((i+1)<buffer.length() && Hangul::isJamoV(buffer.charAt(i+1))) { + return true; // Hangul syllable + } + starterNorm=nullptr; + } else { + starterNorm=norms.getNorm(c); + } + } + prevCC=cc; + } + return false; +} + +void Normalizer2DataBuilder::postProcess(Norm &norm) { + // Prerequisites: Compositions are built, mappings are recursively decomposed. + // Mappings are not yet in canonical order. + // + // This function works on a Norm struct. We do not know which code point(s) map(s) to it. + // Therefore, we cannot compute algorithmic mapping deltas here. + // Error conditions are checked, but printed later when we do know the offending code point. + if(norm.hasMapping()) { + if(norm.mapping->length()>Normalizer2Impl::MAPPING_LENGTH_MASK) { + norm.error="mapping longer than maximum of 31"; + return; + } + // Ensure canonical order. + BuilderReorderingBuffer buffer; + if(norm.rawMapping!=nullptr) { + norms.reorder(*norm.rawMapping, buffer); + buffer.reset(); + } + norms.reorder(*norm.mapping, buffer); + if(buffer.isEmpty()) { + // A character that is deleted (maps to an empty string) must + // get the worst-case lccc and tccc values because arbitrary + // characters on both sides will become adjacent. + norm.leadCC=1; + norm.trailCC=0xff; + } else { + norm.leadCC=buffer.ccAt(0); + norm.trailCC=buffer.ccAt(buffer.length()-1); + } + + norm.hasCompBoundaryBefore= + !buffer.isEmpty() && norm.leadCC==0 && !norms.combinesBack(buffer.charAt(0)); + norm.hasCompBoundaryAfter= + norm.compositions==nullptr && mappingHasCompBoundaryAfter(buffer, norm.mappingType); + + if(norm.combinesBack) { + norm.error="combines-back and decomposes, not possible in Unicode normalization"; + } else if(norm.mappingType==Norm::ROUND_TRIP) { + if(norm.compositions!=nullptr) { + norm.type=Norm::YES_NO_COMBINES_FWD; + } else { + norm.type=Norm::YES_NO_MAPPING_ONLY; + } + } else { // one-way mapping + if(norm.compositions!=nullptr) { + norm.error="combines-forward and has a one-way mapping, " + "not possible in Unicode normalization"; + } else if(buffer.isEmpty()) { + norm.type=Norm::NO_NO_EMPTY; + } else if(!norm.hasCompBoundaryBefore) { + norm.type=Norm::NO_NO_COMP_NO_MAYBE_CC; + } else if(mappingRecomposes(buffer)) { + norm.type=Norm::NO_NO_COMP_BOUNDARY_BEFORE; + } else { + // The mapping is comp-normalized. + norm.type=Norm::NO_NO_COMP_YES; + } + } + } else { // no mapping + norm.leadCC=norm.trailCC=norm.cc; + + norm.hasCompBoundaryBefore= + norm.cc==0 && !norm.combinesBack; + norm.hasCompBoundaryAfter= + norm.cc==0 && !norm.combinesBack && norm.compositions==nullptr; + + if(norm.combinesBack) { + if(norm.compositions!=nullptr) { + // Earlier code checked ccc=0. + norm.type=Norm::MAYBE_YES_COMBINES_FWD; + } else { + norm.type=Norm::MAYBE_YES_SIMPLE; // any ccc + } + } else if(norm.compositions!=nullptr) { + // Earlier code checked ccc=0. + norm.type=Norm::YES_YES_COMBINES_FWD; + } else if(norm.cc!=0) { + norm.type=Norm::YES_YES_WITH_CC; + } else { + norm.type=Norm::INERT; + } + } +} + +class Norm16Writer : public Norms::Enumerator { +public: + Norm16Writer(UMutableCPTrie *trie, Norms &n, Normalizer2DataBuilder &b) : + Norms::Enumerator(n), builder(b), norm16Trie(trie) {} + void rangeHandler(UChar32 start, UChar32 end, Norm &norm) override { + builder.writeNorm16(norm16Trie, start, end, norm); + } + Normalizer2DataBuilder &builder; + UMutableCPTrie *norm16Trie; +}; + +void Normalizer2DataBuilder::setSmallFCD(UChar32 c) { + UChar32 lead= c<=0xffff ? c : U16_LEAD(c); + smallFCD[lead>>8]|=(uint8_t)1<<((lead>>5)&7); +} + +void Normalizer2DataBuilder::writeNorm16(UMutableCPTrie *norm16Trie, UChar32 start, UChar32 end, Norm &norm) { + if((norm.leadCC|norm.trailCC)!=0) { + for(UChar32 c=start; c<=end; ++c) { + setSmallFCD(c); + } + } + + int32_t norm16; + switch(norm.type) { + case Norm::INERT: + norm16=Normalizer2Impl::INERT; + break; + case Norm::YES_YES_COMBINES_FWD: + norm16=norm.offset*2; + break; + case Norm::YES_NO_COMBINES_FWD: + norm16=indexes[Normalizer2Impl::IX_MIN_YES_NO]+norm.offset*2; + break; + case Norm::YES_NO_MAPPING_ONLY: + norm16=indexes[Normalizer2Impl::IX_MIN_YES_NO_MAPPINGS_ONLY]+norm.offset*2; + break; + case Norm::NO_NO_COMP_YES: + norm16=indexes[Normalizer2Impl::IX_MIN_NO_NO]+norm.offset*2; + break; + case Norm::NO_NO_COMP_BOUNDARY_BEFORE: + norm16=indexes[Normalizer2Impl::IX_MIN_NO_NO_COMP_BOUNDARY_BEFORE]+norm.offset*2; + break; + case Norm::NO_NO_COMP_NO_MAYBE_CC: + norm16=indexes[Normalizer2Impl::IX_MIN_NO_NO_COMP_NO_MAYBE_CC]+norm.offset*2; + break; + case Norm::NO_NO_EMPTY: + norm16=indexes[Normalizer2Impl::IX_MIN_NO_NO_EMPTY]+norm.offset*2; + break; + case Norm::NO_NO_DELTA: + { + // Positive offset from minNoNoDelta, shifted left for additional bits. + int32_t offset=(norm.offset+Normalizer2Impl::MAX_DELTA)<<Normalizer2Impl::DELTA_SHIFT; + if(norm.trailCC==0) { + // DELTA_TCCC_0==0 + } else if(norm.trailCC==1) { + offset|=Normalizer2Impl::DELTA_TCCC_1; + } else { + offset|=Normalizer2Impl::DELTA_TCCC_GT_1; + } + norm16=getMinNoNoDelta()+offset; + break; + } + case Norm::MAYBE_YES_COMBINES_FWD: + norm16=indexes[Normalizer2Impl::IX_MIN_MAYBE_YES]+norm.offset*2; + break; + case Norm::MAYBE_YES_SIMPLE: + norm16=Normalizer2Impl::MIN_NORMAL_MAYBE_YES+norm.cc*2; // ccc=0..255 + break; + case Norm::YES_YES_WITH_CC: + U_ASSERT(norm.cc!=0); + norm16=Normalizer2Impl::MIN_YES_YES_WITH_CC-2+norm.cc*2; // ccc=1..255 + break; + default: // Should not occur. + exit(U_INTERNAL_PROGRAM_ERROR); + } + U_ASSERT((norm16&1)==0); + if(norm.hasCompBoundaryAfter) { + norm16|=Normalizer2Impl::HAS_COMP_BOUNDARY_AFTER; + } + IcuToolErrorCode errorCode("gennorm2/writeNorm16()"); + umutablecptrie_setRange(norm16Trie, start, end, (uint32_t)norm16, errorCode); + + // Set the minimum code points for real data lookups in the quick check loops. + UBool isDecompNo= + (Norm::YES_NO_COMBINES_FWD<=norm.type && norm.type<=Norm::NO_NO_DELTA) || + norm.cc!=0; + if(isDecompNo && start<indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]) { + indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]=start; + } + UBool isCompNoMaybe= norm.type>=Norm::NO_NO_COMP_YES; + if(isCompNoMaybe && start<indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]) { + indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]=start; + } + if(norm.leadCC!=0 && start<indexes[Normalizer2Impl::IX_MIN_LCCC_CP]) { + indexes[Normalizer2Impl::IX_MIN_LCCC_CP]=start; + } +} + +void Normalizer2DataBuilder::setHangulData(UMutableCPTrie *norm16Trie) { + HangulIterator hi; + const HangulIterator::Range *range; + // Check that none of the Hangul/Jamo code points have data. + while((range=hi.nextRange())!=nullptr) { + for(UChar32 c=range->start; c<=range->end; ++c) { + if(umutablecptrie_get(norm16Trie, c)>Normalizer2Impl::INERT) { + fprintf(stderr, + "gennorm2 error: " + "illegal mapping/composition/ccc data for Hangul or Jamo U+%04lX\n", + (long)c); + exit(U_INVALID_FORMAT_ERROR); + } + } + } + // Set data for algorithmic runtime handling. + IcuToolErrorCode errorCode("gennorm2/setHangulData()"); + + // Jamo V/T are maybeYes + if(Hangul::JAMO_V_BASE<indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]) { + indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]=Hangul::JAMO_V_BASE; + } + umutablecptrie_setRange(norm16Trie, Hangul::JAMO_L_BASE, Hangul::JAMO_L_END, + Normalizer2Impl::JAMO_L, errorCode); + umutablecptrie_setRange(norm16Trie, Hangul::JAMO_V_BASE, Hangul::JAMO_V_END, + Normalizer2Impl::JAMO_VT, errorCode); + // JAMO_T_BASE+1: not U+11A7 + umutablecptrie_setRange(norm16Trie, Hangul::JAMO_T_BASE+1, Hangul::JAMO_T_END, + Normalizer2Impl::JAMO_VT, errorCode); + + // Hangul LV encoded as minYesNo + uint32_t lv=indexes[Normalizer2Impl::IX_MIN_YES_NO]; + // Hangul LVT encoded as minYesNoMappingsOnly|HAS_COMP_BOUNDARY_AFTER + uint32_t lvt=indexes[Normalizer2Impl::IX_MIN_YES_NO_MAPPINGS_ONLY]| + Normalizer2Impl::HAS_COMP_BOUNDARY_AFTER; + if(Hangul::HANGUL_BASE<indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]) { + indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]=Hangul::HANGUL_BASE; + } + // Set the first LV, then write all other Hangul syllables as LVT, + // then overwrite the remaining LV. + umutablecptrie_set(norm16Trie, Hangul::HANGUL_BASE, lv, errorCode); + umutablecptrie_setRange(norm16Trie, Hangul::HANGUL_BASE+1, Hangul::HANGUL_END, lvt, errorCode); + UChar32 c=Hangul::HANGUL_BASE; + while((c+=Hangul::JAMO_T_COUNT)<=Hangul::HANGUL_END) { + umutablecptrie_set(norm16Trie, c, lv, errorCode); + } + errorCode.assertSuccess(); +} + +LocalUCPTriePointer Normalizer2DataBuilder::processData() { + // Build composition lists before recursive decomposition, + // so that we still have the raw, pair-wise mappings. + CompositionBuilder compBuilder(norms); + norms.enumRanges(compBuilder); + + // Recursively decompose all mappings. + Decomposer decomposer(norms); + do { + decomposer.didDecompose=false; + norms.enumRanges(decomposer); + } while(decomposer.didDecompose); + + // Set the Norm::Type and other properties. + int32_t normsLength=norms.length(); + for(int32_t i=1; i<normsLength; ++i) { + postProcess(norms.getNormRefByIndex(i)); + } + + // Write the properties, mappings and composition lists to + // appropriate parts of the "extra data" array. + ExtraData extra(norms, optimization==OPTIMIZE_FAST); + norms.enumRanges(extra); + + extraData=extra.yesYesCompositions; + indexes[Normalizer2Impl::IX_MIN_YES_NO]=extraData.length()*2; + extraData.append(extra.yesNoMappingsAndCompositions); + indexes[Normalizer2Impl::IX_MIN_YES_NO_MAPPINGS_ONLY]=extraData.length()*2; + extraData.append(extra.yesNoMappingsOnly); + indexes[Normalizer2Impl::IX_MIN_NO_NO]=extraData.length()*2; + extraData.append(extra.noNoMappingsCompYes); + indexes[Normalizer2Impl::IX_MIN_NO_NO_COMP_BOUNDARY_BEFORE]=extraData.length()*2; + extraData.append(extra.noNoMappingsCompBoundaryBefore); + indexes[Normalizer2Impl::IX_MIN_NO_NO_COMP_NO_MAYBE_CC]=extraData.length()*2; + extraData.append(extra.noNoMappingsCompNoMaybeCC); + indexes[Normalizer2Impl::IX_MIN_NO_NO_EMPTY]=extraData.length()*2; + extraData.append(extra.noNoMappingsEmpty); + indexes[Normalizer2Impl::IX_LIMIT_NO_NO]=extraData.length()*2; + + // Pad the maybeYesCompositions length to a multiple of 4, + // so that NO_NO_DELTA bits 2..1 can be used without subtracting the center. + while(extra.maybeYesCompositions.length()&3) { + extra.maybeYesCompositions.append((char16_t)0); + } + extraData.insert(0, extra.maybeYesCompositions); + indexes[Normalizer2Impl::IX_MIN_MAYBE_YES]= + Normalizer2Impl::MIN_NORMAL_MAYBE_YES- + extra.maybeYesCompositions.length()*2; + + // Pad to even length for 4-byte alignment of following data. + if(extraData.length()&1) { + extraData.append((char16_t)0); + } + + int32_t minNoNoDelta=getMinNoNoDelta(); + U_ASSERT((minNoNoDelta&7)==0); + if(indexes[Normalizer2Impl::IX_LIMIT_NO_NO]>minNoNoDelta) { + fprintf(stderr, + "gennorm2 error: " + "data structure overflow, too much mapping composition data\n"); + exit(U_BUFFER_OVERFLOW_ERROR); + } + + // writeNorm16() and setHangulData() reduce these as needed. + indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]=0x110000; + indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]=0x110000; + indexes[Normalizer2Impl::IX_MIN_LCCC_CP]=0x110000; + + IcuToolErrorCode errorCode("gennorm2/processData()"); + UMutableCPTrie *norm16Trie = umutablecptrie_open( + Normalizer2Impl::INERT, Normalizer2Impl::INERT, errorCode); + errorCode.assertSuccess(); + + // Map each code point to its norm16 value, + // including the properties that fit directly, + // and the offset to the "extra data" if necessary. + Norm16Writer norm16Writer(norm16Trie, norms, *this); + norms.enumRanges(norm16Writer); + // TODO: iterate via getRange() instead of callback? + + setHangulData(norm16Trie); + + // Look for the "worst" norm16 value of any supplementary code point + // corresponding to a lead surrogate, and set it as that surrogate's value. + // Enables UTF-16 quick check inner loops to look at only code units. + // + // We could be more sophisticated: + // We could collect a bit set for whether there are values in the different + // norm16 ranges (yesNo, maybeYes, yesYesWithCC etc.) + // and select the best value that only breaks the composition and/or decomposition + // inner loops if necessary. + // However, that seems like overkill for an optimization for supplementary characters. + // + // First check that surrogate code *points* are inert. + // The parser should have rejected values/mappings for them. + uint32_t value; + UChar32 end = umutablecptrie_getRange(norm16Trie, 0xd800, UCPMAP_RANGE_NORMAL, 0, + nullptr, nullptr, &value); + if (value != Normalizer2Impl::INERT || end < 0xdfff) { + fprintf(stderr, + "gennorm2 error: not all surrogate code points are inert: U+d800..U+%04x=%lx\n", + (int)end, (long)value); + exit(U_INTERNAL_PROGRAM_ERROR); + } + uint32_t maxNorm16 = 0; + // ANDing values yields 0 bits where any value has a 0. + // Used for worst-case HAS_COMP_BOUNDARY_AFTER. + uint32_t andedNorm16 = 0; + end = 0; + for (UChar32 start = 0x10000;;) { + if (start > end) { + end = umutablecptrie_getRange(norm16Trie, start, UCPMAP_RANGE_NORMAL, 0, + nullptr, nullptr, &value); + if (end < 0) { break; } + } + if ((start & 0x3ff) == 0) { + // Data for a new lead surrogate. + maxNorm16 = andedNorm16 = value; + } else { + if (value > maxNorm16) { + maxNorm16 = value; + } + andedNorm16 &= value; + } + // Intersect each range with the code points for one lead surrogate. + UChar32 leadEnd = start | 0x3ff; + if (leadEnd <= end) { + // End of the supplementary block for a lead surrogate. + if (maxNorm16 >= (uint32_t)indexes[Normalizer2Impl::IX_LIMIT_NO_NO]) { + // Set noNo ("worst" value) if it got into "less-bad" maybeYes or ccc!=0. + // Otherwise it might end up at something like JAMO_VT which stays in + // the inner decomposition quick check loop. + maxNorm16 = (uint32_t)indexes[Normalizer2Impl::IX_LIMIT_NO_NO]; + } + maxNorm16 = + (maxNorm16 & ~Normalizer2Impl::HAS_COMP_BOUNDARY_AFTER)| + (andedNorm16 & Normalizer2Impl::HAS_COMP_BOUNDARY_AFTER); + if (maxNorm16 != Normalizer2Impl::INERT) { + umutablecptrie_set(norm16Trie, U16_LEAD(start), maxNorm16, errorCode); + } + if (value == Normalizer2Impl::INERT) { + // Potentially skip inert supplementary blocks for several lead surrogates. + start = (end + 1) & ~0x3ff; + } else { + start = leadEnd + 1; + } + } else { + start = end + 1; + } + } + + // Adjust supplementary minimum code points to break quick check loops at their lead surrogates. + // For an empty data file, minCP=0x110000 turns into 0xdc00 (first trail surrogate) + // which is harmless. + // As a result, the minimum code points are always BMP code points. + int32_t minCP=indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]; + if(minCP>=0x10000) { + indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]=U16_LEAD(minCP); + } + minCP=indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]; + if(minCP>=0x10000) { + indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]=U16_LEAD(minCP); + } + minCP=indexes[Normalizer2Impl::IX_MIN_LCCC_CP]; + if(minCP>=0x10000) { + indexes[Normalizer2Impl::IX_MIN_LCCC_CP]=U16_LEAD(minCP); + } + + LocalUCPTriePointer builtTrie( + umutablecptrie_buildImmutable(norm16Trie, UCPTRIE_TYPE_FAST, UCPTRIE_VALUE_BITS_16, errorCode)); + norm16TrieLength=ucptrie_toBinary(builtTrie.getAlias(), nullptr, 0, errorCode); + if(errorCode.get()!=U_BUFFER_OVERFLOW_ERROR) { + fprintf(stderr, "gennorm2 error: unable to build/serialize the normalization trie - %s\n", + errorCode.errorName()); + exit(errorCode.reset()); + } + umutablecptrie_close(norm16Trie); + errorCode.reset(); + norm16TrieBytes=new uint8_t[norm16TrieLength]; + ucptrie_toBinary(builtTrie.getAlias(), norm16TrieBytes, norm16TrieLength, errorCode); + errorCode.assertSuccess(); + + int32_t offset=(int32_t)sizeof(indexes); + indexes[Normalizer2Impl::IX_NORM_TRIE_OFFSET]=offset; + offset+=norm16TrieLength; + indexes[Normalizer2Impl::IX_EXTRA_DATA_OFFSET]=offset; + offset+=extraData.length()*2; + indexes[Normalizer2Impl::IX_SMALL_FCD_OFFSET]=offset; + offset+=sizeof(smallFCD); + int32_t totalSize=offset; + for(int32_t i=Normalizer2Impl::IX_RESERVED3_OFFSET; i<=Normalizer2Impl::IX_TOTAL_SIZE; ++i) { + indexes[i]=totalSize; + } + + if(beVerbose) { + printf("size of normalization trie: %5ld bytes\n", (long)norm16TrieLength); + printf("size of 16-bit extra data: %5ld uint16_t\n", (long)extraData.length()); + printf("size of small-FCD data: %5ld bytes\n", (long)sizeof(smallFCD)); + printf("size of binary data file contents: %5ld bytes\n", (long)totalSize); + printf("minDecompNoCodePoint: U+%04lX\n", (long)indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]); + printf("minCompNoMaybeCodePoint: U+%04lX\n", (long)indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]); + printf("minLcccCodePoint: U+%04lX\n", (long)indexes[Normalizer2Impl::IX_MIN_LCCC_CP]); + printf("minYesNo: (with compositions) 0x%04x\n", (int)indexes[Normalizer2Impl::IX_MIN_YES_NO]); + printf("minYesNoMappingsOnly: 0x%04x\n", (int)indexes[Normalizer2Impl::IX_MIN_YES_NO_MAPPINGS_ONLY]); + printf("minNoNo: (comp-normalized) 0x%04x\n", (int)indexes[Normalizer2Impl::IX_MIN_NO_NO]); + printf("minNoNoCompBoundaryBefore: 0x%04x\n", (int)indexes[Normalizer2Impl::IX_MIN_NO_NO_COMP_BOUNDARY_BEFORE]); + printf("minNoNoCompNoMaybeCC: 0x%04x\n", (int)indexes[Normalizer2Impl::IX_MIN_NO_NO_COMP_NO_MAYBE_CC]); + printf("minNoNoEmpty: 0x%04x\n", (int)indexes[Normalizer2Impl::IX_MIN_NO_NO_EMPTY]); + printf("limitNoNo: 0x%04x\n", (int)indexes[Normalizer2Impl::IX_LIMIT_NO_NO]); + printf("minNoNoDelta: 0x%04x\n", (int)minNoNoDelta); + printf("minMaybeYes: 0x%04x\n", (int)indexes[Normalizer2Impl::IX_MIN_MAYBE_YES]); + } + + UVersionInfo nullVersion={ 0, 0, 0, 0 }; + if(0==memcmp(nullVersion, unicodeVersion, 4)) { + u_versionFromString(unicodeVersion, U_UNICODE_VERSION); + } + memcpy(dataInfo.dataVersion, unicodeVersion, 4); + return builtTrie; +} + +void Normalizer2DataBuilder::writeBinaryFile(const char *filename) { + processData(); + + IcuToolErrorCode errorCode("gennorm2/writeBinaryFile()"); + UNewDataMemory *pData= + udata_create(nullptr, nullptr, filename, &dataInfo, + haveCopyright ? U_COPYRIGHT_STRING : nullptr, errorCode); + if(errorCode.isFailure()) { + fprintf(stderr, "gennorm2 error: unable to create the output file %s - %s\n", + filename, errorCode.errorName()); + exit(errorCode.reset()); + } + udata_writeBlock(pData, indexes, sizeof(indexes)); + udata_writeBlock(pData, norm16TrieBytes, norm16TrieLength); + udata_writeUString(pData, toUCharPtr(extraData.getBuffer()), extraData.length()); + udata_writeBlock(pData, smallFCD, sizeof(smallFCD)); + int32_t writtenSize=udata_finish(pData, errorCode); + if(errorCode.isFailure()) { + fprintf(stderr, "gennorm2: error %s writing the output file\n", errorCode.errorName()); + exit(errorCode.reset()); + } + int32_t totalSize=indexes[Normalizer2Impl::IX_TOTAL_SIZE]; + if(writtenSize!=totalSize) { + fprintf(stderr, "gennorm2 error: written size %ld != calculated size %ld\n", + (long)writtenSize, (long)totalSize); + exit(U_INTERNAL_PROGRAM_ERROR); + } +} + +void +Normalizer2DataBuilder::writeCSourceFile(const char *filename) { + LocalUCPTriePointer norm16Trie = processData(); + + IcuToolErrorCode errorCode("gennorm2/writeCSourceFile()"); + const char *basename=findBasename(filename); + CharString path(filename, (int32_t)(basename-filename), errorCode); + CharString dataName(basename, errorCode); + const char *extension=strrchr(basename, '.'); + if(extension!=nullptr) { + dataName.truncate((int32_t)(extension-basename)); + } + const char *name=dataName.data(); + errorCode.assertSuccess(); + + FILE *f=usrc_create(path.data(), basename, 2016, "icu/source/tools/gennorm2/n2builder.cpp"); + if(f==nullptr) { + fprintf(stderr, "gennorm2/writeCSourceFile() error: unable to create the output file %s\n", + filename); + exit(U_FILE_ACCESS_ERROR); + } + fputs("#ifdef INCLUDED_FROM_NORMALIZER2_CPP\n\n", f); + + char line[100]; + snprintf(line, sizeof(line), "static const UVersionInfo %s_formatVersion={", name); + usrc_writeArray(f, line, dataInfo.formatVersion, 8, 4, "", "};\n"); + snprintf(line, sizeof(line), "static const UVersionInfo %s_dataVersion={", name); + usrc_writeArray(f, line, dataInfo.dataVersion, 8, 4, "", "};\n\n"); + snprintf(line, sizeof(line), "static const int32_t %s_indexes[Normalizer2Impl::IX_COUNT]={\n", name); + usrc_writeArray(f, line, indexes, 32, Normalizer2Impl::IX_COUNT, "", "\n};\n\n"); + + usrc_writeUCPTrie(f, name, norm16Trie.getAlias(), UPRV_TARGET_SYNTAX_CCODE); + + snprintf(line, sizeof(line), "static const uint16_t %s_extraData[%%ld]={\n", name); + usrc_writeArray(f, line, extraData.getBuffer(), 16, extraData.length(), "", "\n};\n\n"); + snprintf(line, sizeof(line), "static const uint8_t %s_smallFCD[%%ld]={\n", name); + usrc_writeArray(f, line, smallFCD, 8, sizeof(smallFCD), "", "\n};\n\n"); + + fputs("#endif // INCLUDED_FROM_NORMALIZER2_CPP\n", f); + fclose(f); +} + +namespace { + +bool equalStrings(const UnicodeString *s1, const UnicodeString *s2) { + if(s1 == nullptr) { + return s2 == nullptr; + } else if(s2 == nullptr) { + return false; + } else { + return *s1 == *s2; + } +} + +const char *typeChars = "?-=>"; + +void writeMapping(FILE *f, const UnicodeString *m) { + if(m != nullptr && !m->isEmpty()) { + int32_t i = 0; + UChar32 c = m->char32At(i); + fprintf(f, "%04lX", (long)c); + while((i += U16_LENGTH(c)) < m->length()) { + c = m->char32At(i); + fprintf(f, " %04lX", (long)c); + } + } + fputs("\n", f); +} + +} // namespace + +void +Normalizer2DataBuilder::writeDataFile(const char *filename, bool writeRemoved) const { + // Do not processData() before writing the input-syntax data file. + FILE *f = fopen(filename, "w"); + if(f == nullptr) { + fprintf(stderr, "gennorm2/writeDataFile() error: unable to create the output file %s\n", + filename); + exit(U_FILE_ACCESS_ERROR); + return; + } + + if(unicodeVersion[0] != 0 || unicodeVersion[1] != 0 || + unicodeVersion[2] != 0 || unicodeVersion[3] != 0) { + char uv[U_MAX_VERSION_STRING_LENGTH]; + u_versionToString(unicodeVersion, uv); + fprintf(f, "* Unicode %s\n\n", uv); + } + + UnicodeSetIterator ccIter(norms.ccSet); + UChar32 start = U_SENTINEL; + UChar32 end = U_SENTINEL; + uint8_t prevCC = 0; + bool done = false; + bool didWrite = false; + do { + UChar32 c; + uint8_t cc; + if(ccIter.next() && !ccIter.isString()) { + c = ccIter.getCodepoint(); + cc = norms.getCC(c); + } else { + c = 0x110000; + cc = 0; + done = true; + } + if(cc == prevCC && c == (end + 1)) { + end = c; + } else { + if(prevCC != 0) { + if(start == end) { + fprintf(f, "%04lX:%d\n", (long)start, (int)prevCC); + } else { + fprintf(f, "%04lX..%04lX:%d\n", (long)start, (long)end, (int)prevCC); + } + didWrite = true; + } + start = end = c; + prevCC = cc; + } + } while(!done); + if(didWrite) { + fputs("\n", f); + } + + UnicodeSetIterator mIter(norms.mappingSet); + start = U_SENTINEL; + end = U_SENTINEL; + const UnicodeString *prevMapping = nullptr; + Norm::MappingType prevType = Norm::NONE; + done = false; + do { + UChar32 c; + const Norm *norm; + if(mIter.next() && !mIter.isString()) { + c = mIter.getCodepoint(); + norm = norms.getNorm(c); + } else { + c = 0x110000; + norm = nullptr; + done = true; + } + const UnicodeString *mapping; + Norm::MappingType type; + if(norm == nullptr) { + mapping = nullptr; + type = Norm::NONE; + } else { + type = norm->mappingType; + if(type == Norm::NONE) { + mapping = nullptr; + } else { + mapping = norm->mapping; + } + } + if(type == prevType && equalStrings(mapping, prevMapping) && c == (end + 1)) { + end = c; + } else { + if(writeRemoved ? prevType != Norm::NONE : prevType > Norm::REMOVED) { + if(start == end) { + fprintf(f, "%04lX%c", (long)start, typeChars[prevType]); + } else { + fprintf(f, "%04lX..%04lX%c", (long)start, (long)end, typeChars[prevType]); + } + writeMapping(f, prevMapping); + } + start = end = c; + prevMapping = mapping; + prevType = type; + } + } while(!done); + + fclose(f); +} + +void +Normalizer2DataBuilder::computeDiff(const Normalizer2DataBuilder &b1, + const Normalizer2DataBuilder &b2, + Normalizer2DataBuilder &diff) { + // Compute diff = b1 - b2 + // so that we should be able to get b1 = b2 + diff. + if(0 != memcmp(b1.unicodeVersion, b2.unicodeVersion, U_MAX_VERSION_LENGTH)) { + memcpy(diff.unicodeVersion, b1.unicodeVersion, U_MAX_VERSION_LENGTH); + } + + UnicodeSet ccSet(b1.norms.ccSet); + ccSet.addAll(b2.norms.ccSet); + UnicodeSetIterator ccIter(ccSet); + while(ccIter.next() && !ccIter.isString()) { + UChar32 c = ccIter.getCodepoint(); + uint8_t cc1 = b1.norms.getCC(c); + uint8_t cc2 = b2.norms.getCC(c); + if(cc1 != cc2) { + diff.setCC(c, cc1); + } + } + + UnicodeSet mSet(b1.norms.mappingSet); + mSet.addAll(b2.norms.mappingSet); + UnicodeSetIterator mIter(mSet); + while(mIter.next() && !mIter.isString()) { + UChar32 c = mIter.getCodepoint(); + const Norm *norm1 = b1.norms.getNorm(c); + const Norm *norm2 = b2.norms.getNorm(c); + const UnicodeString *mapping1; + Norm::MappingType type1; + if(norm1 == nullptr || !norm1->hasMapping()) { + mapping1 = nullptr; + type1 = Norm::NONE; + } else { + mapping1 = norm1->mapping; + type1 = norm1->mappingType; + } + const UnicodeString *mapping2; + Norm::MappingType type2; + if(norm2 == nullptr || !norm2->hasMapping()) { + mapping2 = nullptr; + type2 = Norm::NONE; + } else { + mapping2 = norm2->mapping; + type2 = norm2->mappingType; + } + if(type1 == type2 && equalStrings(mapping1, mapping2)) { + // Nothing to do. + } else if(type1 == Norm::NONE) { + diff.removeMapping(c); + } else if(type1 == Norm::ROUND_TRIP) { + diff.setRoundTripMapping(c, *mapping1); + } else if(type1 == Norm::ONE_WAY) { + diff.setOneWayMapping(c, *mapping1); + } + } +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_NORMALIZATION */ + +/* + * Hey, Emacs, please set the following: + * + * Local Variables: + * indent-tabs-mode: nil + * End: + */ diff --git a/intl/icu/source/tools/gennorm2/n2builder.h b/intl/icu/source/tools/gennorm2/n2builder.h new file mode 100644 index 0000000000..b3698253be --- /dev/null +++ b/intl/icu/source/tools/gennorm2/n2builder.h @@ -0,0 +1,122 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* +* Copyright (C) 2009-2014, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: n2builder.h +* encoding: UTF-8 +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2009nov25 +* created by: Markus W. Scherer +*/ + +#ifndef __N2BUILDER_H__ +#define __N2BUILDER_H__ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_NORMALIZATION + +#include "unicode/errorcode.h" +#include "unicode/umutablecptrie.h" +#include "unicode/unistr.h" +#include "normalizer2impl.h" // for IX_COUNT +#include "toolutil.h" +#include "norms.h" + +U_NAMESPACE_BEGIN + +extern UBool beVerbose, haveCopyright; + +class Normalizer2DataBuilder { +public: + Normalizer2DataBuilder(UErrorCode &errorCode); + ~Normalizer2DataBuilder(); + + enum OverrideHandling { + OVERRIDE_NONE, + OVERRIDE_ANY, + OVERRIDE_PREVIOUS + }; + + void setOverrideHandling(OverrideHandling oh); + + enum Optimization { + OPTIMIZE_NORMAL, + OPTIMIZE_FAST + }; + + void setOptimization(Optimization opt) { optimization=opt; } + + void setCC(UChar32 c, uint8_t cc); + void setOneWayMapping(UChar32 c, const UnicodeString &m); + void setRoundTripMapping(UChar32 c, const UnicodeString &m); + void removeMapping(UChar32 c); + + void setUnicodeVersion(const char *v); + + void writeBinaryFile(const char *filename); + void writeCSourceFile(const char *filename); + void writeDataFile(const char *filename, bool writeRemoved) const; + + static void computeDiff(const Normalizer2DataBuilder &b1, + const Normalizer2DataBuilder &b2, + Normalizer2DataBuilder &diff); + +private: + friend class Norm16Writer; + + Normalizer2DataBuilder(const Normalizer2DataBuilder &other) = delete; + Normalizer2DataBuilder &operator=(const Normalizer2DataBuilder &other) = delete; + + Norm *checkNormForMapping(Norm *p, UChar32 c); // check for permitted overrides + + /** + * A starter character with a mapping does not have a composition boundary after it + * if the character itself combines-forward (which is tested by the caller of this function), + * or it is deleted (mapped to the empty string), + * or its mapping contains no starter, + * or the last starter combines-forward. + */ + UBool mappingHasCompBoundaryAfter(const BuilderReorderingBuffer &buffer, + Norm::MappingType mappingType) const; + /** Returns true if the mapping by itself recomposes, that is, it is not comp-normalized. */ + UBool mappingRecomposes(const BuilderReorderingBuffer &buffer) const; + void postProcess(Norm &norm); + + void setSmallFCD(UChar32 c); + int32_t getMinNoNoDelta() const { + return indexes[Normalizer2Impl::IX_MIN_MAYBE_YES]- + ((2*Normalizer2Impl::MAX_DELTA+1)<<Normalizer2Impl::DELTA_SHIFT); + } + void writeNorm16(UMutableCPTrie *norm16Trie, UChar32 start, UChar32 end, Norm &norm); + void setHangulData(UMutableCPTrie *norm16Trie); + LocalUCPTriePointer processData(); + + Norms norms; + + int32_t phase; + OverrideHandling overrideHandling; + + Optimization optimization; + + int32_t indexes[Normalizer2Impl::IX_COUNT]; + uint8_t *norm16TrieBytes; + int32_t norm16TrieLength; + UnicodeString extraData; + uint8_t smallFCD[0x100]; + + UVersionInfo unicodeVersion; +}; + +U_NAMESPACE_END + +#endif // #if !UCONFIG_NO_NORMALIZATION + +#endif // __N2BUILDER_H__ diff --git a/intl/icu/source/tools/gennorm2/norms.cpp b/intl/icu/source/tools/gennorm2/norms.cpp new file mode 100644 index 0000000000..9dd8dca977 --- /dev/null +++ b/intl/icu/source/tools/gennorm2/norms.cpp @@ -0,0 +1,324 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +// norms.cpp +// created: 2017jun04 Markus W. Scherer +// (pulled out of n2builder.cpp) + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_NORMALIZATION + +#include <stdio.h> +#include <stdlib.h> +#include "unicode/errorcode.h" +#include "unicode/umutablecptrie.h" +#include "unicode/unistr.h" +#include "unicode/utf16.h" +#include "normalizer2impl.h" +#include "norms.h" +#include "toolutil.h" +#include "uvectr32.h" + +U_NAMESPACE_BEGIN + +void BuilderReorderingBuffer::append(UChar32 c, uint8_t cc) { + if(cc==0 || fLength==0 || ccAt(fLength-1)<=cc) { + if(cc==0) { + fLastStarterIndex=fLength; + } + fArray[fLength++]=(c<<8)|cc; + return; + } + // Let this character bubble back to its canonical order. + int32_t i=fLength-1; + while(i>fLastStarterIndex && ccAt(i)>cc) { + --i; + } + ++i; // after the last starter or prevCC<=cc + // Move this and the following characters forward one to make space. + for(int32_t j=fLength; i<j; --j) { + fArray[j]=fArray[j-1]; + } + fArray[i]=(c<<8)|cc; + ++fLength; + fDidReorder=true; +} + +void BuilderReorderingBuffer::toString(UnicodeString &dest) const { + dest.remove(); + for(int32_t i=0; i<fLength; ++i) { + dest.append(charAt(i)); + } +} + +UChar32 Norm::combine(UChar32 trail) const { + int32_t length; + const CompositionPair *pairs=getCompositionPairs(length); + for(int32_t i=0; i<length; ++i) { + if(trail==pairs[i].trail) { + return pairs[i].composite; + } + if(trail<pairs[i].trail) { + break; + } + } + return U_SENTINEL; +} + +Norms::Norms(UErrorCode &errorCode) { + normTrie = umutablecptrie_open(0, 0, &errorCode); + normMem=utm_open("gennorm2 normalization structs", 10000, 0x110100, sizeof(Norm)); + // Default "inert" Norm struct at index 0. Practically immutable. + norms=allocNorm(); + norms->type=Norm::INERT; +} + +Norms::~Norms() { + umutablecptrie_close(normTrie); + int32_t normsLength=utm_countItems(normMem); + for(int32_t i=1; i<normsLength; ++i) { + delete norms[i].mapping; + delete norms[i].rawMapping; + delete norms[i].compositions; + } + utm_close(normMem); +} + +Norm *Norms::allocNorm() { + Norm *p=(Norm *)utm_alloc(normMem); + norms=(Norm *)utm_getStart(normMem); // in case it got reallocated + return p; +} + +Norm *Norms::getNorm(UChar32 c) { + uint32_t i = umutablecptrie_get(normTrie, c); + if(i==0) { + return nullptr; + } + return norms+i; +} + +const Norm *Norms::getNorm(UChar32 c) const { + uint32_t i = umutablecptrie_get(normTrie, c); + if(i==0) { + return nullptr; + } + return norms+i; +} + +const Norm &Norms::getNormRef(UChar32 c) const { + return norms[umutablecptrie_get(normTrie, c)]; +} + +Norm *Norms::createNorm(UChar32 c) { + uint32_t i=umutablecptrie_get(normTrie, c); + if(i!=0) { + return norms+i; + } else { + /* allocate Norm */ + Norm *p=allocNorm(); + IcuToolErrorCode errorCode("gennorm2/createNorm()"); + umutablecptrie_set(normTrie, c, (uint32_t)(p - norms), errorCode); + return p; + } +} + +void Norms::reorder(UnicodeString &mapping, BuilderReorderingBuffer &buffer) const { + int32_t length=mapping.length(); + U_ASSERT(length<=Normalizer2Impl::MAPPING_LENGTH_MASK); + const char16_t *s=mapping.getBuffer(); + int32_t i=0; + UChar32 c; + while(i<length) { + U16_NEXT(s, i, length, c); + buffer.append(c, getCC(c)); + } + if(buffer.didReorder()) { + buffer.toString(mapping); + } +} + +UBool Norms::combinesWithCCBetween(const Norm &norm, uint8_t lowCC, int32_t highCC) const { + if((highCC-lowCC)>=2) { + int32_t length; + const CompositionPair *pairs=norm.getCompositionPairs(length); + for(int32_t i=0; i<length; ++i) { + uint8_t trailCC=getCC(pairs[i].trail); + if(lowCC<trailCC && trailCC<highCC) { + return true; + } + } + } + return false; +} + +void Norms::enumRanges(Enumerator &e) { + UChar32 start = 0, end; + uint32_t i; + while ((end = umutablecptrie_getRange(normTrie, start, UCPMAP_RANGE_NORMAL, 0, + nullptr, nullptr, &i)) >= 0) { + if (i > 0) { + e.rangeHandler(start, end, norms[i]); + } + start = end + 1; + } +} + +Norms::Enumerator::~Enumerator() {} + +void CompositionBuilder::rangeHandler(UChar32 start, UChar32 end, Norm &norm) { + if(norm.mappingType!=Norm::ROUND_TRIP) { return; } + if(start!=end) { + fprintf(stderr, + "gennorm2 error: same round-trip mapping for " + "more than 1 code point U+%04lX..U+%04lX\n", + (long)start, (long)end); + exit(U_INVALID_FORMAT_ERROR); + } + if(norm.cc!=0) { + fprintf(stderr, + "gennorm2 error: " + "U+%04lX has a round-trip mapping and ccc!=0, " + "not possible in Unicode normalization\n", + (long)start); + exit(U_INVALID_FORMAT_ERROR); + } + // setRoundTripMapping() ensured that there are exactly two code points. + const UnicodeString &m=*norm.mapping; + UChar32 lead=m.char32At(0); + UChar32 trail=m.char32At(m.length()-1); + if(norms.getCC(lead)!=0) { + fprintf(stderr, + "gennorm2 error: " + "U+%04lX's round-trip mapping's starter U+%04lX has ccc!=0, " + "not possible in Unicode normalization\n", + (long)start, (long)lead); + exit(U_INVALID_FORMAT_ERROR); + } + // Flag for trailing character. + norms.createNorm(trail)->combinesBack=true; + // Insert (trail, composite) pair into compositions list for the lead character. + IcuToolErrorCode errorCode("gennorm2/addComposition()"); + Norm *leadNorm=norms.createNorm(lead); + UVector32 *compositions=leadNorm->compositions; + int32_t i; + if(compositions==nullptr) { + compositions=leadNorm->compositions=new UVector32(errorCode); + i=0; // "insert" the first pair at index 0 + } else { + // Insertion sort, and check for duplicate trail characters. + int32_t length; + const CompositionPair *pairs=leadNorm->getCompositionPairs(length); + for(i=0; i<length; ++i) { + if(trail==pairs[i].trail) { + fprintf(stderr, + "gennorm2 error: same round-trip mapping for " + "more than 1 code point (e.g., U+%04lX) to U+%04lX + U+%04lX\n", + (long)start, (long)lead, (long)trail); + exit(U_INVALID_FORMAT_ERROR); + } + if(trail<pairs[i].trail) { + break; + } + } + } + compositions->insertElementAt(trail, 2*i, errorCode); + compositions->insertElementAt(start, 2*i+1, errorCode); +} + +void Decomposer::rangeHandler(UChar32 start, UChar32 end, Norm &norm) { + if(!norm.hasMapping()) { return; } + const UnicodeString &m=*norm.mapping; + UnicodeString *decomposed=nullptr; + const char16_t *s=toUCharPtr(m.getBuffer()); + int32_t length=m.length(); + int32_t prev, i=0; + UChar32 c; + while(i<length) { + prev=i; + U16_NEXT(s, i, length, c); + if(start<=c && c<=end) { + fprintf(stderr, + "gennorm2 error: U+%04lX maps to itself directly or indirectly\n", + (long)c); + exit(U_INVALID_FORMAT_ERROR); + } + const Norm &cNorm=norms.getNormRef(c); + if(cNorm.hasMapping()) { + if(norm.mappingType==Norm::ROUND_TRIP) { + if(prev==0) { + if(cNorm.mappingType!=Norm::ROUND_TRIP) { + fprintf(stderr, + "gennorm2 error: " + "U+%04lX's round-trip mapping's starter " + "U+%04lX one-way-decomposes, " + "not possible in Unicode normalization\n", + (long)start, (long)c); + exit(U_INVALID_FORMAT_ERROR); + } + uint8_t myTrailCC=norms.getCC(m.char32At(i)); + UChar32 cTrailChar=cNorm.mapping->char32At(cNorm.mapping->length()-1); + uint8_t cTrailCC=norms.getCC(cTrailChar); + if(cTrailCC>myTrailCC) { + fprintf(stderr, + "gennorm2 error: " + "U+%04lX's round-trip mapping's starter " + "U+%04lX decomposes and the " + "inner/earlier tccc=%hu > outer/following tccc=%hu, " + "not possible in Unicode normalization\n", + (long)start, (long)c, + (short)cTrailCC, (short)myTrailCC); + exit(U_INVALID_FORMAT_ERROR); + } + } else { + fprintf(stderr, + "gennorm2 error: " + "U+%04lX's round-trip mapping's non-starter " + "U+%04lX decomposes, " + "not possible in Unicode normalization\n", + (long)start, (long)c); + exit(U_INVALID_FORMAT_ERROR); + } + } + if(decomposed==nullptr) { + decomposed=new UnicodeString(m, 0, prev); + } + decomposed->append(*cNorm.mapping); + } else if(Hangul::isHangul(c)) { + char16_t buffer[3]; + int32_t hangulLength=Hangul::decompose(c, buffer); + if(norm.mappingType==Norm::ROUND_TRIP && prev!=0) { + fprintf(stderr, + "gennorm2 error: " + "U+%04lX's round-trip mapping's non-starter " + "U+%04lX decomposes, " + "not possible in Unicode normalization\n", + (long)start, (long)c); + exit(U_INVALID_FORMAT_ERROR); + } + if(decomposed==nullptr) { + decomposed=new UnicodeString(m, 0, prev); + } + decomposed->append(buffer, hangulLength); + } else if(decomposed!=nullptr) { + decomposed->append(m, prev, i-prev); + } + } + if(decomposed!=nullptr) { + if(norm.rawMapping==nullptr) { + // Remember the original mapping when decomposing recursively. + norm.rawMapping=norm.mapping; + } else { + delete norm.mapping; + } + norm.mapping=decomposed; + // Not norm.setMappingCP(); because the original mapping + // is most likely to be encodable as a delta. + didDecompose|=true; + } +} + +U_NAMESPACE_END + +#endif // #if !UCONFIG_NO_NORMALIZATION diff --git a/intl/icu/source/tools/gennorm2/norms.h b/intl/icu/source/tools/gennorm2/norms.h new file mode 100644 index 0000000000..f2778d9509 --- /dev/null +++ b/intl/icu/source/tools/gennorm2/norms.h @@ -0,0 +1,215 @@ +// © 2017 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html + +// norms.h +// created: 2017jun04 Markus W. Scherer +// (pulled out of n2builder.cpp) + +// Storing & manipulating Normalizer2 builder data. + +#ifndef __NORMS_H__ +#define __NORMS_H__ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_NORMALIZATION + +#include "unicode/errorcode.h" +#include "unicode/umutablecptrie.h" +#include "unicode/uniset.h" +#include "unicode/unistr.h" +#include "unicode/utf16.h" +#include "normalizer2impl.h" +#include "toolutil.h" +#include "uvectr32.h" + +U_NAMESPACE_BEGIN + +class BuilderReorderingBuffer { +public: + BuilderReorderingBuffer() : fLength(0), fLastStarterIndex(-1), fDidReorder(false) {} + void reset() { + fLength=0; + fLastStarterIndex=-1; + fDidReorder=false; + } + int32_t length() const { return fLength; } + UBool isEmpty() const { return fLength==0; } + int32_t lastStarterIndex() const { return fLastStarterIndex; } + UChar32 charAt(int32_t i) const { return fArray[i]>>8; } + uint8_t ccAt(int32_t i) const { return (uint8_t)fArray[i]; } + UBool didReorder() const { return fDidReorder; } + + void append(UChar32 c, uint8_t cc); + void toString(UnicodeString &dest) const; + +private: + int32_t fArray[Normalizer2Impl::MAPPING_LENGTH_MASK]; + int32_t fLength; + int32_t fLastStarterIndex; + UBool fDidReorder; +}; + +struct CompositionPair { + CompositionPair(UChar32 t, UChar32 c) : trail(t), composite(c) {} + UChar32 trail, composite; +}; + +struct Norm { + enum MappingType { NONE, REMOVED, ROUND_TRIP, ONE_WAY }; + + UBool hasMapping() const { return mappingType>REMOVED; } + + // Requires hasMapping() and well-formed mapping. + void setMappingCP() { + UChar32 c; + if(!mapping->isEmpty() && mapping->length()==U16_LENGTH(c=mapping->char32At(0))) { + mappingCP=c; + } else { + mappingCP=U_SENTINEL; + } + } + + const CompositionPair *getCompositionPairs(int32_t &length) const { + if(compositions==nullptr) { + length=0; + return nullptr; + } else { + length=compositions->size()/2; + return reinterpret_cast<const CompositionPair *>(compositions->getBuffer()); + } + } + UChar32 combine(UChar32 trail) const; + + UnicodeString *mapping; + UnicodeString *rawMapping; // non-nullptr if the mapping is further decomposed + UChar32 mappingCP; // >=0 if mapping to 1 code point + int32_t mappingPhase; + MappingType mappingType; + + UVector32 *compositions; // (trail, composite) pairs + uint8_t cc, leadCC, trailCC; + UBool combinesBack; + UBool hasCompBoundaryBefore, hasCompBoundaryAfter; + + /** + * Overall type of normalization properties. + * Set after most processing is done. + * + * Corresponds to the rows in the chart on + * https://icu.unicode.org/design/normalization/custom + * in numerical (but reverse visual) order. + * + * YES_NO means composition quick check=yes, decomposition QC=no -- etc. + */ + enum Type { + /** Initial value until most processing is done. */ + UNKNOWN, + /** No mapping, does not combine, ccc=0. */ + INERT, + /** Starter, no mapping, has compositions. */ + YES_YES_COMBINES_FWD, + /** Starter with a round-trip mapping and compositions. */ + YES_NO_COMBINES_FWD, + /** Starter with a round-trip mapping but no compositions. */ + YES_NO_MAPPING_ONLY, + /** Has a one-way mapping which is comp-normalized. */ + NO_NO_COMP_YES, + /** Has a one-way mapping which is not comp-normalized but has a comp boundary before. */ + NO_NO_COMP_BOUNDARY_BEFORE, + /** Has a one-way mapping which does not have a comp boundary before. */ + NO_NO_COMP_NO_MAYBE_CC, + /** Has a one-way mapping to the empty string. */ + NO_NO_EMPTY, + /** Has an algorithmic one-way mapping to a single code point. */ + NO_NO_DELTA, + /** + * Combines both backward and forward, has compositions. + * Allowed, but not normally used. + */ + MAYBE_YES_COMBINES_FWD, + /** Combines only backward. */ + MAYBE_YES_SIMPLE, + /** Non-zero ccc but does not combine backward. */ + YES_YES_WITH_CC + } type; + /** Offset into the type's part of the extra data, or the algorithmic-mapping delta. */ + int32_t offset; + + /** + * Error string set by processing functions that do not have access + * to the code point, deferred for readable reporting. + */ + const char *error; +}; + +class Norms { +public: + Norms(UErrorCode &errorCode); + ~Norms(); + + int32_t length() const { return utm_countItems(normMem); } + const Norm &getNormRefByIndex(int32_t i) const { return norms[i]; } + Norm &getNormRefByIndex(int32_t i) { return norms[i]; } + + Norm *allocNorm(); + /** Returns an existing Norm unit, or nullptr if c has no data. */ + Norm *getNorm(UChar32 c); + const Norm *getNorm(UChar32 c) const; + /** Returns a Norm unit, creating a new one if necessary. */ + Norm *createNorm(UChar32 c); + /** Returns an existing Norm unit, or an immutable empty object if c has no data. */ + const Norm &getNormRef(UChar32 c) const; + uint8_t getCC(UChar32 c) const { return getNormRef(c).cc; } + UBool combinesBack(UChar32 c) const { + return Hangul::isJamoV(c) || Hangul::isJamoT(c) || getNormRef(c).combinesBack; + } + + void reorder(UnicodeString &mapping, BuilderReorderingBuffer &buffer) const; + + // int32_t highCC not uint8_t so that we can pass in 256 as the upper limit. + UBool combinesWithCCBetween(const Norm &norm, uint8_t lowCC, int32_t highCC) const; + + class Enumerator { + public: + Enumerator(Norms &n) : norms(n) {} + virtual ~Enumerator(); + /** Called for enumerated value!=0. */ + virtual void rangeHandler(UChar32 start, UChar32 end, Norm &norm) = 0; + protected: + Norms &norms; + }; + + void enumRanges(Enumerator &e); + + UnicodeSet ccSet, mappingSet; + +private: + Norms(const Norms &other) = delete; + Norms &operator=(const Norms &other) = delete; + + UMutableCPTrie *normTrie; + UToolMemory *normMem; + Norm *norms; +}; + +class CompositionBuilder : public Norms::Enumerator { +public: + CompositionBuilder(Norms &n) : Norms::Enumerator(n) {} + /** Adds a composition mapping for the first character in a round-trip mapping. */ + void rangeHandler(UChar32 start, UChar32 end, Norm &norm) override; +}; + +class Decomposer : public Norms::Enumerator { +public: + Decomposer(Norms &n) : Norms::Enumerator(n), didDecompose(false) {} + /** Decomposes each character of the current mapping. Sets didDecompose if any. */ + void rangeHandler(UChar32 start, UChar32 end, Norm &norm) override; + UBool didDecompose; +}; + +U_NAMESPACE_END + +#endif // #if !UCONFIG_NO_NORMALIZATION + +#endif // __NORMS_H__ diff --git a/intl/icu/source/tools/gennorm2/sources.txt b/intl/icu/source/tools/gennorm2/sources.txt new file mode 100644 index 0000000000..76452d26ea --- /dev/null +++ b/intl/icu/source/tools/gennorm2/sources.txt @@ -0,0 +1,4 @@ +extradata.cpp +gennorm2.cpp +n2builder.cpp +norms.cpp |