From 0ebf5bdf043a27fd3dfb7f92e0cb63d88954c44d Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 19 Apr 2024 03:47:29 +0200 Subject: Adding upstream version 115.8.0esr. Signed-off-by: Daniel Baumann --- intl/icu/source/tools/gencfu/Makefile.in | 96 ++++++ intl/icu/source/tools/gencfu/gencfu.1.in | 93 ++++++ intl/icu/source/tools/gencfu/gencfu.cpp | 332 +++++++++++++++++++++ intl/icu/source/tools/gencfu/gencfu.vcxproj | 80 +++++ .../icu/source/tools/gencfu/gencfu.vcxproj.filters | 22 ++ intl/icu/source/tools/gencfu/sources.txt | 1 + 6 files changed, 624 insertions(+) create mode 100644 intl/icu/source/tools/gencfu/Makefile.in create mode 100644 intl/icu/source/tools/gencfu/gencfu.1.in create mode 100644 intl/icu/source/tools/gencfu/gencfu.cpp create mode 100644 intl/icu/source/tools/gencfu/gencfu.vcxproj create mode 100644 intl/icu/source/tools/gencfu/gencfu.vcxproj.filters create mode 100644 intl/icu/source/tools/gencfu/sources.txt (limited to 'intl/icu/source/tools/gencfu') diff --git a/intl/icu/source/tools/gencfu/Makefile.in b/intl/icu/source/tools/gencfu/Makefile.in new file mode 100644 index 0000000000..6cd8e418db --- /dev/null +++ b/intl/icu/source/tools/gencfu/Makefile.in @@ -0,0 +1,96 @@ +## Makefile.in for ICU - tools/gencfu +## Copyright (C) 2016 and later: Unicode, Inc. and others. +## License & terms of use: http://www.unicode.org/copyright.html +## Copyright (c) 2009-2012 International Business Machines Corporation and +## others. All Rights Reserved. + +## Source directory information +srcdir = @srcdir@ +top_srcdir = @top_srcdir@ + +top_builddir = ../.. + +include $(top_builddir)/icudefs.mk + +## Build directory information +subdir = tools/gencfu + +TARGET_STUB_NAME = gencfu + +SECTION = 1 + +MAN_FILES = $(TARGET_STUB_NAME).$(SECTION) + + +## Extra files to remove for 'make clean' +CLEANFILES = *~ $(DEPS) $(MAN_FILES) + +## Target information +TARGET = $(BINDIR)/$(TARGET_STUB_NAME)$(EXEEXT) + +CPPFLAGS += -I$(top_srcdir)/common -I$(top_srcdir)/i18n -I$(srcdir)/../toolutil +LIBS = $(LIBICUTOOLUTIL) $(LIBICUI18N) $(LIBICUUC) $(DEFAULT_LIBS) $(LIB_M) + +SOURCES = $(shell cat $(srcdir)/sources.txt) +OBJECTS = $(SOURCES:.cpp=.o) + +DEPS = $(OBJECTS:.o=.d) + +## List of phony targets +.PHONY : all all-local install install-local clean clean-local \ +distclean distclean-local dist dist-local check check-local install-man + +## Clear suffix list +.SUFFIXES : + +## List of standard targets +all: all-local +install: install-local +clean: clean-local +distclean : distclean-local +dist: dist-local +check: all check-local + +all-local: $(TARGET) $(MAN_FILES) + +install-local: all-local install-man + $(MKINSTALLDIRS) $(DESTDIR)$(bindir) + $(INSTALL) $(TARGET) $(DESTDIR)$(bindir) + +install-man: $(MAN_FILES) + $(MKINSTALLDIRS) $(DESTDIR)$(mandir)/man$(SECTION) + $(INSTALL_DATA) $? $(DESTDIR)$(mandir)/man$(SECTION) + +dist-local: + +clean-local: + test -z "$(CLEANFILES)" || $(RMV) $(CLEANFILES) + $(RMV) $(TARGET) $(OBJECTS) + +distclean-local: clean-local + $(RMV) Makefile + +check-local: all-local + +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + cd $(top_builddir) \ + && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= $(SHELL) ./config.status + +$(TARGET) : $(OBJECTS) + $(LINK.cc) $(OUTOPT)$@ $^ $(LIBS) + $(POST_BUILD_STEP) + + +%.$(SECTION): $(srcdir)/%.$(SECTION).in + cd $(top_builddir) \ + && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= $(SHELL) ./config.status + + +ifeq (,$(MAKECMDGOALS)) +-include $(DEPS) +else +ifneq ($(patsubst %clean,,$(MAKECMDGOALS)),) +-include $(DEPS) +endif +endif + diff --git a/intl/icu/source/tools/gencfu/gencfu.1.in b/intl/icu/source/tools/gencfu/gencfu.1.in new file mode 100644 index 0000000000..2b3240b2ab --- /dev/null +++ b/intl/icu/source/tools/gencfu/gencfu.1.in @@ -0,0 +1,93 @@ +.\" Hey, Emacs! This is -*-nroff-*- you know... +.\" +.\" gencfu.1: manual page for the gencfu utility +.\" +.\" Copyright (C) 2016 and later: Unicode, Inc. and others. +.\" License & terms of use: http://www.unicode.org/copyright.html +.\" Copyright (C) 2012 International Business Machines Corporation and others +.\" +.TH GENCFU 1 "24 May 2009" "ICU MANPAGE" "ICU @VERSION@ Manual" +.SH NAME +.B gencfu +\- Generates Unicode Confusable data files +.SH SYNOPSIS +.B gencfu +[ +.BR "\-h\fP, \fB\-?\fP, \fB\-\-help" +] +[ +.BR "\-V\fP, \fB\-\-version" +] +[ +.BR "\-c\fP, \fB\-\-copyright" +] +[ +.BR "\-v\fP, \fB\-\-verbose" +] +[ +.BI "\-d\fP, \fB\-\-destdir" " destination" +] +[ +.BI "\-i\fP, \fB\-\-icudatadir" " directory" +] +.BI "\-r\fP, \fB\-\-rules" " rule\-file" +.BI "\-w\fP, \fB\-\-wsrules" " whole\-script\-rule\-file" +.BI "\-o\fP, \fB\-\-out" " output\-file" +.SH DESCRIPTION +.B gencfu +reads confusable character definitions in the input file, which are +plain text files containing confusable character +definitions in the input format defined by Unicode UAX39 for the files +.I confusables.txt +and +.I confusablesWholeScript.txt. +This source (.txt) format is also accepted by ICU spoof detectors. +The files must be encoded in utf-8 format, with or without a BOM. +Normally the output data file has the +.B .cfu +extension. +.SH OPTIONS +.TP +.BR "\-h\fP, \fB\-?\fP, \fB\-\-help" +Print help about usage and exit. +.TP +.BR "\-V\fP, \fB\-\-version" +Print the version of +.B gencfu +and exit. +.TP +.BR "\-c\fP, \fB\-\-copyright" +Embeds the standard ICU copyright into the +.IR output-file . +.TP +.BR "\-v\fP, \fB\-\-verbose" +Display extra informative messages during execution. +.TP +.BI "\-d\fP, \fB\-\-destdir" " destination" +Set the destination directory of the +.IR output-file +to +.IR destination . +.TP +.BI "\-i\fP, \fB\-\-icudatadir" " directory" +Look for any necessary ICU data files in +.IR directory . +For example, the file +.B pnames.icu +must be located when ICU's data is not built as a shared library. +The default ICU data directory is specified by the environment variable +.BR ICU_DATA . +Most configurations of ICU do not require this argument. +.TP +.BI "\-r\fP, \fB\-\-rules" " rule\-file" +The source file to read. +.TP +.BI "\-w\fP, \fB\-\-wsrules" " whole\-script\-rule\-file" +The whole script source file to read. +.TP +.BI "\-o\fP, \fB\-\-out" " output\-file" +The output data file to write. +.SH VERSION +1.0 +.SH COPYRIGHT +Copyright (C) 2009 International Business Machines Corporation and others diff --git a/intl/icu/source/tools/gencfu/gencfu.cpp b/intl/icu/source/tools/gencfu/gencfu.cpp new file mode 100644 index 0000000000..543cd76afa --- /dev/null +++ b/intl/icu/source/tools/gencfu/gencfu.cpp @@ -0,0 +1,332 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +********************************************************************** +* Copyright (C) 2009-2016, International Business Machines +* Corporation and others. All Rights Reserved. +********************************************************************** +* +* File gencfu.c +*/ + +//-------------------------------------------------------------------- +// +// Tool for generating Unicode Confusable data files (.cfu files). +// .cfu files contain the compiled of the confusable data +// derived from the Unicode Consortium data described in +// Unicode UAX 39. +// +// Usage: gencfu [options] -r confusables-file.txt -o output-file.cfu +// +// options: -v verbose +// -? or -h help +// +// The input rule file are plain text files containing confusable character +// definitions in the input format defined by Unicode UAX39 for the files +// confusables.txt. This source (.txt) format +// is also accepted by ICU spoof detectors. The +// files must be encoded in utf-8 format, with or without a BOM. +// +// The script used to compile confusablesWholeScript.txt into the CFU file +// until the Unicode consortium deprecated it. +// +//-------------------------------------------------------------------- + +#include "unicode/utypes.h" +#include "unicode/unistr.h" +#include "unicode/uclean.h" +#include "unicode/udata.h" +#include "unicode/putil.h" + +#include "uoptions.h" +#include "unewdata.h" +#include "ucmndata.h" +#include "uspoof_impl.h" +#include "cmemory.h" + +#include +#include +#include + +U_NAMESPACE_USE + +static char *progName; +static UOption options[]={ + UOPTION_HELP_H, /* 0 */ + UOPTION_HELP_QUESTION_MARK, /* 1 */ + UOPTION_VERBOSE, /* 2 */ + { "rules", nullptr, nullptr, nullptr, 'r', UOPT_REQUIRES_ARG, 0 }, /* 3 */ + { "wsrules", nullptr, nullptr, nullptr, 'w', UOPT_REQUIRES_ARG, 0}, /* 4 */ // deprecated + { "out", nullptr, nullptr, nullptr, 'o', UOPT_REQUIRES_ARG, 0 }, /* 5 */ + UOPTION_ICUDATADIR, /* 6 */ + UOPTION_DESTDIR, /* 7 */ + UOPTION_COPYRIGHT, /* 8 */ + UOPTION_QUIET, /* 9 */ +}; + +void usageAndDie(int retCode) { + printf("Usage: %s [-v] [-options] -r confusablesRules.txt -o output-file\n", progName); + printf("\tRead in Unicode confusable character definitions and write out the binary data\n" + "options:\n" + "\t-h or -? or --help this usage text\n" + "\t-V or --version show a version message\n" + "\t-c or --copyright include a copyright notice\n" + "\t-v or --verbose turn on verbose output\n" + "\t-q or --quiet do not display warnings and progress\n" + "\t-i or --icudatadir directory for locating any needed intermediate data files,\n" + "\t followed by path, defaults to %s\n" + "\t-d or --destdir destination directory, followed by the path\n", + u_getDataDirectory()); + exit (retCode); +} + + +#if UCONFIG_NO_REGULAR_EXPRESSIONS || UCONFIG_NO_NORMALIZATION || UCONFIG_NO_FILE_IO + +/* dummy UDataInfo cf. udata.h */ +static UDataInfo dummyDataInfo = { + sizeof(UDataInfo), + 0, + + U_IS_BIG_ENDIAN, + U_CHARSET_FAMILY, + U_SIZEOF_UCHAR, + 0, + + { 0, 0, 0, 0 }, /* dummy dataFormat */ + { 0, 0, 0, 0 }, /* dummy formatVersion */ + { 0, 0, 0, 0 } /* dummy dataVersion */ +}; + +#else + +// +// Set up the ICU data header, defined in ucmndata.h +// +DataHeader dh ={ + {sizeof(DataHeader), // Struct MappedData + 0xda, + 0x27}, + + { // struct UDataInfo + sizeof(UDataInfo), // size + 0, // reserved + U_IS_BIG_ENDIAN, + U_CHARSET_FAMILY, + U_SIZEOF_UCHAR, + 0, // reserved + + { 0x43, 0x66, 0x75, 0x20 }, // dataFormat="Cfu " + { 0xff, 0, 0, 0 }, // formatVersion. Filled in later with values + // from the builder. The values declared + // here should never appear in any real data. + { 5, 1, 0, 0 } // dataVersion (Unicode version) + }}; + +#endif + +// Forward declaration for function for reading source files. +static const char *readFile(const char *fileName, int32_t *len); + +//---------------------------------------------------------------------------- +// +// main for gencfu +// +//---------------------------------------------------------------------------- +int main(int argc, char **argv) { + UErrorCode status = U_ZERO_ERROR; + const char *confFileName; + const char *outFileName; + const char *outDir = nullptr; + const char *copyright = nullptr; + + // + // Pick up and check the command line arguments, + // using the standard ICU tool utils option handling. + // + U_MAIN_INIT_ARGS(argc, argv); + progName = argv[0]; + argc=u_parseArgs(argc, argv, UPRV_LENGTHOF(options), options); + if(argc<0) { + // Unrecognized option + fprintf(stderr, "error in command line argument \"%s\"\n", argv[-argc]); + usageAndDie(U_ILLEGAL_ARGUMENT_ERROR); + } + + if(options[0].doesOccur || options[1].doesOccur) { + // -? or -h for help. + usageAndDie(0); + } + + if (!(options[3].doesOccur && options[5].doesOccur)) { + fprintf(stderr, "confusables file and output file must all be specified.\n"); + usageAndDie(U_ILLEGAL_ARGUMENT_ERROR); + } + confFileName = options[3].value; + outFileName = options[5].value; + + if (options[6].doesOccur) { + u_setDataDirectory(options[6].value); + } + + status = U_ZERO_ERROR; + + /* Combine the directory with the file name */ + if(options[7].doesOccur) { + outDir = options[7].value; + } + if (options[8].doesOccur) { + copyright = U_COPYRIGHT_STRING; + } + + UBool quiet = false; + if (options[9].doesOccur) { + quiet = true; + } + +#if UCONFIG_NO_REGULAR_EXPRESSIONS || UCONFIG_NO_NORMALIZATION || UCONFIG_NO_FILE_IO + // spoof detection data file parsing is dependent on regular expressions. + // TODO: have the tool return an error status. Requires fixing the ICU data build + // so that it doesn't abort entirely on that error. + + UNewDataMemory *pData; + char msg[1024]; + + /* write message with just the name */ + snprintf(msg, sizeof(msg), "gencfu writes dummy %s because of UCONFIG_NO_REGULAR_EXPRESSIONS and/or UCONFIG_NO_NORMALIZATION and/or UCONFIG_NO_FILE_IO, see uconfig.h", outFileName); + fprintf(stderr, "%s\n", msg); + + /* write the dummy data file */ + pData = udata_create(outDir, nullptr, outFileName, &dummyDataInfo, nullptr, &status); + udata_writeBlock(pData, msg, strlen(msg)); + udata_finish(pData, &status); + return (int)status; + +#else + /* Initialize ICU */ + u_init(&status); + if (U_FAILURE(status)) { + fprintf(stderr, "%s: can not initialize ICU. status = %s\n", + argv[0], u_errorName(status)); + exit(1); + } + status = U_ZERO_ERROR; + + // Read in the confusables source file + + int32_t confusablesLen = 0; + const char *confusables = readFile(confFileName, &confusablesLen); + if (confusables == nullptr) { + printf("gencfu: error reading file \"%s\"\n", confFileName); + exit(-1); + } + + // + // Create the Spoof Detector from the source confusables files. + // This will compile the data. + // + UParseError parseError; + parseError.line = 0; + parseError.offset = 0; + int32_t errType; + USpoofChecker *sc = uspoof_openFromSource(confusables, confusablesLen, + nullptr, 0, + &errType, &parseError, &status); + if (U_FAILURE(status)) { + fprintf(stderr, "gencfu: uspoof_openFromSource error \"%s\" at file %s, line %d, column %d\n", + u_errorName(status), confFileName, (int)parseError.line, (int)parseError.offset); + exit(status); + } + + + // + // Get the compiled rule data from the USpoofChecker. + // + uint32_t outDataSize; + uint8_t *outData; + outDataSize = uspoof_serialize(sc, nullptr, 0, &status); + if (status != U_BUFFER_OVERFLOW_ERROR) { + fprintf(stderr, "gencfu: uspoof_serialize() returned %s\n", u_errorName(status)); + exit(status); + } + status = U_ZERO_ERROR; + outData = new uint8_t[outDataSize]; + uspoof_serialize(sc, outData, outDataSize, &status); + + // Copy the data format version numbers from the spoof data header into the UDataMemory header. + + uprv_memcpy(dh.info.formatVersion, + reinterpret_cast(outData)->fFormatVersion, + sizeof(dh.info.formatVersion)); + + // + // Create the output file + // + size_t bytesWritten; + UNewDataMemory *pData; + pData = udata_create(outDir, nullptr, outFileName, &(dh.info), copyright, &status); + if(U_FAILURE(status)) { + fprintf(stderr, "gencfu: Could not open output file \"%s\", \"%s\"\n", + outFileName, u_errorName(status)); + exit(status); + } + + + // Write the data itself. + udata_writeBlock(pData, outData, outDataSize); + // finish up + bytesWritten = udata_finish(pData, &status); + if(U_FAILURE(status)) { + fprintf(stderr, "gencfu: Error %d writing the output file\n", status); + exit(status); + } + + if (bytesWritten != outDataSize) { + fprintf(stderr, "gencfu: Error writing to output file \"%s\"\n", outFileName); + exit(-1); + } + + uspoof_close(sc); + delete [] outData; + delete [] confusables; + u_cleanup(); + if (!quiet) { + printf("gencfu: tool completed successfully.\n"); + } + return 0; +#endif // UCONFIG_NO_REGULAR_EXPRESSIONS +} + + + // + // Read in a confusables source file + // + static const char *readFile(const char *fileName, int32_t *len) { + char *result; + long fileSize; + FILE *file; + + file = fopen(fileName, "rb"); + if( file == 0 ) { + return nullptr; + } + fseek(file, 0, SEEK_END); + fileSize = ftell(file); + fseek(file, 0, SEEK_SET); + result = new char[fileSize+10]; + if (result==nullptr) { + fclose(file); + return nullptr; + } + + long t = static_cast(fread(result, 1, fileSize, file)); + if (t != fileSize) { + delete [] result; + fclose(file); + return nullptr; + } + result[fileSize]=0; + *len = static_cast(fileSize); + fclose(file); + return result; + } diff --git a/intl/icu/source/tools/gencfu/gencfu.vcxproj b/intl/icu/source/tools/gencfu/gencfu.vcxproj new file mode 100644 index 0000000000..4018d5078d --- /dev/null +++ b/intl/icu/source/tools/gencfu/gencfu.vcxproj @@ -0,0 +1,80 @@ + + + + {691EE0C0-DC57-4A48-8AEE-8ED75EB3A057} + + + Application + false + MultiByte + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + .\$(Platform)\$(Configuration)\ + .\$(Platform)\$(Configuration)\ + + .\x86\$(Configuration)\ + .\x86\$(Configuration)\ + + true + false + + + + + $(OutDir)\gencfu.tlb + + + Level3 + Default + false + ..\..\common;..\..\i18n;..\toolutil;%(AdditionalIncludeDirectories) + $(OutDir)\gencfu.pch + $(OutDir)/ + $(OutDir)/ + $(OutDir)\gencfu.pdb + + + Console + $(OutDir)\gencfu.exe + ..\..\..\$(IcuLibOutputDir);%(AdditionalLibraryDirectories) + + + copy "$(TargetPath)" ..\..\..\$(IcuBinOutputDir) + ..\..\..\$(IcuBinOutputDir)\$(TargetFileName);%(Outputs) + + + + + + true + MultiThreadedDebugDLL + + + icuucd.lib;icuind.lib;icutud.lib;%(AdditionalDependencies) + + + + + + MultiThreadedDLL + true + + + icuuc.lib;icuin.lib;icutu.lib;%(AdditionalDependencies) + + + + + + + + + \ No newline at end of file diff --git a/intl/icu/source/tools/gencfu/gencfu.vcxproj.filters b/intl/icu/source/tools/gencfu/gencfu.vcxproj.filters new file mode 100644 index 0000000000..096a235f7c --- /dev/null +++ b/intl/icu/source/tools/gencfu/gencfu.vcxproj.filters @@ -0,0 +1,22 @@ + + + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + + + Source Files + + + \ No newline at end of file diff --git a/intl/icu/source/tools/gencfu/sources.txt b/intl/icu/source/tools/gencfu/sources.txt new file mode 100644 index 0000000000..e5a05f8451 --- /dev/null +++ b/intl/icu/source/tools/gencfu/sources.txt @@ -0,0 +1 @@ +gencfu.cpp -- cgit v1.2.3