summaryrefslogtreecommitdiffstats
path: root/intl/icu/source/tools/gennorm2
diff options
context:
space:
mode:
Diffstat (limited to 'intl/icu/source/tools/gennorm2')
-rw-r--r--intl/icu/source/tools/gennorm2/Makefile.in81
-rw-r--r--intl/icu/source/tools/gennorm2/gennorm2.cpp286
-rw-r--r--intl/icu/source/tools/gennorm2/gennorm2.vcxproj270
-rw-r--r--intl/icu/source/tools/gennorm2/n2builder.cpp1307
-rw-r--r--intl/icu/source/tools/gennorm2/n2builder.h127
5 files changed, 2071 insertions, 0 deletions
diff --git a/intl/icu/source/tools/gennorm2/Makefile.in b/intl/icu/source/tools/gennorm2/Makefile.in
new file mode 100644
index 000000000..228076346
--- /dev/null
+++ b/intl/icu/source/tools/gennorm2/Makefile.in
@@ -0,0 +1,81 @@
+## Makefile.in for ICU - tools/gennorm2
+## Copyright (C) 2016 and later: Unicode, Inc. and others.
+## License & terms of use: http://www.unicode.org/copyright.html
+## Copyright (c) 2009-2011, International Business Machines Corporation and
+## others. All Rights Reserved.
+## Steven R. Loomis/Markus W. Scherer
+
+## Source directory information
+srcdir = @srcdir@
+top_srcdir = @top_srcdir@
+
+top_builddir = ../..
+
+include $(top_builddir)/icudefs.mk
+
+## Build directory information
+subdir = tools/gennorm2
+
+TARGET_STUB_NAME = gennorm2
+
+## Extra files to remove for 'make clean'
+CLEANFILES = *~ $(DEPS)
+
+## Target information
+TARGET = $(BINDIR)/$(TARGET_STUB_NAME)$(EXEEXT)
+
+CPPFLAGS += -I$(top_srcdir)/common -I$(srcdir)/../toolutil
+LIBS = $(LIBICUTOOLUTIL) $(LIBICUI18N) $(LIBICUUC) $(DEFAULT_LIBS) $(LIB_M)
+
+OBJECTS = gennorm2.o n2builder.o
+
+DEPS = $(OBJECTS:.o=.d)
+
+## List of phony targets
+.PHONY : all all-local install install-local clean clean-local \
+distclean distclean-local dist dist-local check check-local install-man
+
+## Clear suffix list
+.SUFFIXES :
+
+## List of standard targets
+all: all-local
+install: install-local
+clean: clean-local
+distclean : distclean-local
+dist: dist-local
+check: all check-local
+
+all-local: $(TARGET)
+
+install-local: all-local
+ $(MKINSTALLDIRS) $(DESTDIR)$(sbindir)
+ $(INSTALL) $(TARGET) $(DESTDIR)$(sbindir)
+
+dist-local:
+
+clean-local:
+ test -z "$(CLEANFILES)" || $(RMV) $(CLEANFILES)
+ $(RMV) $(TARGET) $(OBJECTS)
+
+distclean-local: clean-local
+ $(RMV) Makefile
+
+check-local: all-local
+
+Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
+ cd $(top_builddir) \
+ && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= $(SHELL) ./config.status
+
+$(TARGET) : $(OBJECTS)
+ $(LINK.cc) $(OUTOPT)$@ $^ $(LIBS)
+ $(POST_BUILD_STEP)
+
+
+ifeq (,$(MAKECMDGOALS))
+-include $(DEPS)
+else
+ifneq ($(patsubst %clean,,$(MAKECMDGOALS)),)
+-include $(DEPS)
+endif
+endif
diff --git a/intl/icu/source/tools/gennorm2/gennorm2.cpp b/intl/icu/source/tools/gennorm2/gennorm2.cpp
new file mode 100644
index 000000000..9c85e0bc9
--- /dev/null
+++ b/intl/icu/source/tools/gennorm2/gennorm2.cpp
@@ -0,0 +1,286 @@
+// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// License & terms of use: http://www.unicode.org/copyright.html
+/*
+*******************************************************************************
+*
+* Copyright (C) 2009-2014, International Business Machines
+* Corporation and others. All Rights Reserved.
+*
+*******************************************************************************
+* file name: gennorm2.cpp
+* encoding: US-ASCII
+* tab size: 8 (not used)
+* indentation:4
+*
+* created on: 2009nov25
+* created by: Markus W. Scherer
+*
+* This program reads text files that define Unicode normalization,
+* parses them, and builds a binary data file.
+*/
+
+#include "unicode/utypes.h"
+#include "n2builder.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "unicode/errorcode.h"
+#include "unicode/localpointer.h"
+#include "unicode/putil.h"
+#include "unicode/uchar.h"
+#include "unicode/unistr.h"
+#include "charstr.h"
+#include "normalizer2impl.h"
+#include "toolutil.h"
+#include "uoptions.h"
+#include "uparse.h"
+
+#if UCONFIG_NO_NORMALIZATION
+#include "unewdata.h"
+#endif
+
+U_NAMESPACE_BEGIN
+
+UBool beVerbose=FALSE, haveCopyright=TRUE;
+
+U_DEFINE_LOCAL_OPEN_POINTER(LocalStdioFilePointer, FILE, fclose);
+
+#if !UCONFIG_NO_NORMALIZATION
+void parseFile(FILE *f, Normalizer2DataBuilder &builder);
+#endif
+
+/* -------------------------------------------------------------------------- */
+
+enum {
+ HELP_H,
+ HELP_QUESTION_MARK,
+ VERBOSE,
+ COPYRIGHT,
+ SOURCEDIR,
+ OUTPUT_FILENAME,
+ UNICODE_VERSION,
+ WRITE_C_SOURCE,
+ OPT_FAST
+};
+
+static UOption options[]={
+ UOPTION_HELP_H,
+ UOPTION_HELP_QUESTION_MARK,
+ UOPTION_VERBOSE,
+ UOPTION_COPYRIGHT,
+ UOPTION_SOURCEDIR,
+ UOPTION_DEF("output", 'o', UOPT_REQUIRES_ARG),
+ UOPTION_DEF("unicode", 'u', UOPT_REQUIRES_ARG),
+ UOPTION_DEF("csource", '\1', UOPT_NO_ARG),
+ UOPTION_DEF("fast", '\1', UOPT_NO_ARG)
+};
+
+extern "C" int
+main(int argc, char* argv[]) {
+ U_MAIN_INIT_ARGS(argc, argv);
+
+ /* preset then read command line options */
+ options[SOURCEDIR].value="";
+ argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[HELP_H]), options);
+
+ /* error handling, printing usage message */
+ if(argc<0) {
+ fprintf(stderr,
+ "error in command line argument \"%s\"\n",
+ argv[-argc]);
+ }
+ if(!options[OUTPUT_FILENAME].doesOccur) {
+ argc=-1;
+ }
+ if( argc<2 ||
+ options[HELP_H].doesOccur || options[HELP_QUESTION_MARK].doesOccur
+ ) {
+ /*
+ * Broken into chunks because the C89 standard says the minimum
+ * required supported string length is 509 bytes.
+ */
+ fprintf(stderr,
+ "Usage: %s [-options] infiles+ -o outputfilename\n"
+ "\n"
+ "Reads the infiles with normalization data and\n"
+ "creates a binary or C source file (outputfilename) with the data.\n"
+ "\n",
+ argv[0]);
+ fprintf(stderr,
+ "Options:\n"
+ "\t-h or -? or --help this usage text\n"
+ "\t-v or --verbose verbose output\n"
+ "\t-c or --copyright include a copyright notice\n"
+ "\t-u or --unicode Unicode version, followed by the version like 5.2.0\n");
+ fprintf(stderr,
+ "\t-s or --sourcedir source directory, followed by the path\n"
+ "\t-o or --output output filename\n"
+ "\t --csource writes a C source file with initializers\n");
+ fprintf(stderr,
+ "\t --fast optimize the data for fast normalization,\n"
+ "\t which might increase its size (Writes fully decomposed\n"
+ "\t regular mappings instead of delta mappings.\n"
+ "\t You should measure the runtime speed to make sure that\n"
+ "\t this is a good trade-off.)\n");
+ return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR;
+ }
+
+ beVerbose=options[VERBOSE].doesOccur;
+ haveCopyright=options[COPYRIGHT].doesOccur;
+
+ IcuToolErrorCode errorCode("gennorm2/main()");
+
+#if UCONFIG_NO_NORMALIZATION
+
+ fprintf(stderr,
+ "gennorm2 writes a dummy binary data file "
+ "because UCONFIG_NO_NORMALIZATION is set, \n"
+ "see icu/source/common/unicode/uconfig.h\n");
+ udata_createDummy(NULL, NULL, options[OUTPUT_FILENAME].value, errorCode);
+ // Should not return an error since this is the expected behaviour if UCONFIG_NO_NORMALIZATION is on.
+ // return U_UNSUPPORTED_ERROR;
+ return 0;
+
+#else
+
+ LocalPointer<Normalizer2DataBuilder> builder(new Normalizer2DataBuilder(errorCode), errorCode);
+ errorCode.assertSuccess();
+
+ if(options[UNICODE_VERSION].doesOccur) {
+ builder->setUnicodeVersion(options[UNICODE_VERSION].value);
+ }
+
+ if(options[OPT_FAST].doesOccur) {
+ builder->setOptimization(Normalizer2DataBuilder::OPTIMIZE_FAST);
+ }
+
+ // prepare the filename beginning with the source dir
+ CharString filename(options[SOURCEDIR].value, errorCode);
+ int32_t pathLength=filename.length();
+ if( pathLength>0 &&
+ filename[pathLength-1]!=U_FILE_SEP_CHAR &&
+ filename[pathLength-1]!=U_FILE_ALT_SEP_CHAR
+ ) {
+ filename.append(U_FILE_SEP_CHAR, errorCode);
+ pathLength=filename.length();
+ }
+
+ for(int i=1; i<argc; ++i) {
+ printf("gennorm2: processing %s\n", argv[i]);
+ filename.append(argv[i], errorCode);
+ LocalStdioFilePointer f(fopen(filename.data(), "r"));
+ if(f==NULL) {
+ fprintf(stderr, "gennorm2 error: unable to open %s\n", filename.data());
+ exit(U_FILE_ACCESS_ERROR);
+ }
+ builder->setOverrideHandling(Normalizer2DataBuilder::OVERRIDE_PREVIOUS);
+ parseFile(f.getAlias(), *builder);
+ filename.truncate(pathLength);
+ }
+
+ if(options[WRITE_C_SOURCE].doesOccur) {
+ builder->writeCSourceFile(options[OUTPUT_FILENAME].value);
+ } else {
+ builder->writeBinaryFile(options[OUTPUT_FILENAME].value);
+ }
+
+ return errorCode.get();
+
+#endif
+}
+
+#if !UCONFIG_NO_NORMALIZATION
+
+void parseFile(FILE *f, Normalizer2DataBuilder &builder) {
+ IcuToolErrorCode errorCode("gennorm2/parseFile()");
+ char line[300];
+ uint32_t startCP, endCP;
+ while(NULL!=fgets(line, (int)sizeof(line), f)) {
+ char *comment=(char *)strchr(line, '#');
+ if(comment!=NULL) {
+ *comment=0;
+ }
+ u_rtrim(line);
+ if(line[0]==0) {
+ continue; // skip empty and comment-only lines
+ }
+ if(line[0]=='*') {
+ const char *s=u_skipWhitespace(line+1);
+ if(0==strncmp(s, "Unicode", 7)) {
+ s=u_skipWhitespace(s+7);
+ builder.setUnicodeVersion(s);
+ }
+ continue; // reserved syntax
+ }
+ const char *delimiter;
+ int32_t rangeLength=
+ u_parseCodePointRangeAnyTerminator(line, &startCP, &endCP, &delimiter, errorCode);
+ if(errorCode.isFailure()) {
+ fprintf(stderr, "gennorm2 error: parsing code point range from %s\n", line);
+ exit(errorCode.reset());
+ }
+ delimiter=u_skipWhitespace(delimiter);
+ if(*delimiter==':') {
+ const char *s=u_skipWhitespace(delimiter+1);
+ char *end;
+ unsigned long value=strtoul(s, &end, 10);
+ if(end<=s || *u_skipWhitespace(end)!=0 || value>=0xff) {
+ fprintf(stderr, "gennorm2 error: parsing ccc from %s\n", line);
+ exit(U_PARSE_ERROR);
+ }
+ for(UChar32 c=(UChar32)startCP; c<=(UChar32)endCP; ++c) {
+ builder.setCC(c, (uint8_t)value);
+ }
+ continue;
+ }
+ if(*delimiter=='-') {
+ if(*u_skipWhitespace(delimiter+1)!=0) {
+ fprintf(stderr, "gennorm2 error: parsing remove-mapping %s\n", line);
+ exit(U_PARSE_ERROR);
+ }
+ for(UChar32 c=(UChar32)startCP; c<=(UChar32)endCP; ++c) {
+ builder.removeMapping(c);
+ }
+ continue;
+ }
+ if(*delimiter=='=' || *delimiter=='>') {
+ UChar uchars[Normalizer2Impl::MAPPING_LENGTH_MASK];
+ int32_t length=u_parseString(delimiter+1, uchars, UPRV_LENGTHOF(uchars), NULL, errorCode);
+ if(errorCode.isFailure()) {
+ fprintf(stderr, "gennorm2 error: parsing mapping string from %s\n", line);
+ exit(errorCode.reset());
+ }
+ UnicodeString mapping(FALSE, uchars, length);
+ if(*delimiter=='=') {
+ if(rangeLength!=1) {
+ fprintf(stderr,
+ "gennorm2 error: round-trip mapping for more than 1 code point on %s\n",
+ line);
+ exit(U_PARSE_ERROR);
+ }
+ builder.setRoundTripMapping((UChar32)startCP, mapping);
+ } else {
+ for(UChar32 c=(UChar32)startCP; c<=(UChar32)endCP; ++c) {
+ builder.setOneWayMapping(c, mapping);
+ }
+ }
+ continue;
+ }
+ fprintf(stderr, "gennorm2 error: unrecognized data line %s\n", line);
+ exit(U_PARSE_ERROR);
+ }
+}
+
+#endif // !UCONFIG_NO_NORMALIZATION
+
+U_NAMESPACE_END
+
+/*
+ * Hey, Emacs, please set the following:
+ *
+ * Local Variables:
+ * indent-tabs-mode: nil
+ * End:
+ *
+ */
diff --git a/intl/icu/source/tools/gennorm2/gennorm2.vcxproj b/intl/icu/source/tools/gennorm2/gennorm2.vcxproj
new file mode 100644
index 000000000..99625ebde
--- /dev/null
+++ b/intl/icu/source/tools/gennorm2/gennorm2.vcxproj
@@ -0,0 +1,270 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <ItemGroup Label="ProjectConfigurations">
+ <ProjectConfiguration Include="Debug|Win32">
+ <Configuration>Debug</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Debug|x64">
+ <Configuration>Debug</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|Win32">
+ <Configuration>Release</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|x64">
+ <Configuration>Release</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ </ItemGroup>
+ <PropertyGroup Label="Globals">
+ <ProjectGuid>{C7891A65-80AB-4245-912E-5F1E17B0E6C4}</ProjectGuid>
+ <RootNamespace>gennorm2</RootNamespace>
+ <Keyword>Win32Proj</Keyword>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <CharacterSet>Unicode</CharacterSet>
+ <PlatformToolset>v140</PlatformToolset>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <CharacterSet>Unicode</CharacterSet>
+ <WholeProgramOptimization>true</WholeProgramOptimization>
+ <PlatformToolset>v140</PlatformToolset>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <CharacterSet>Unicode</CharacterSet>
+ <PlatformToolset>v140</PlatformToolset>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <CharacterSet>Unicode</CharacterSet>
+ <WholeProgramOptimization>true</WholeProgramOptimization>
+ <PlatformToolset>v140</PlatformToolset>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+ <ImportGroup Label="ExtensionSettings">
+ </ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <PropertyGroup Label="UserMacros" />
+ <PropertyGroup>
+ <_ProjectFileVersion>10.0.30319.1</_ProjectFileVersion>
+ <OutDir Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">.\x86\Release\</OutDir>
+ <IntDir Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">.\x86\Release\</IntDir>
+ <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">false</LinkIncremental>
+ <OutDir Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">.\x86\Debug\</OutDir>
+ <IntDir Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">.\x86\Debug\</IntDir>
+ <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</LinkIncremental>
+ <OutDir Condition="'$(Configuration)|$(Platform)'=='Release|x64'">.\x64\Release\</OutDir>
+ <IntDir Condition="'$(Configuration)|$(Platform)'=='Release|x64'">.\x64\Release\</IntDir>
+ <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Release|x64'">false</LinkIncremental>
+ <OutDir Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">.\x64\Debug\</OutDir>
+ <IntDir Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">.\x64\Debug\</IntDir>
+ <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</LinkIncremental>
+ </PropertyGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <CustomBuildStep>
+ <Command>copy "$(TargetPath)" ..\..\..\bin
+</Command>
+ <Outputs>..\..\..\bin\$(TargetFileName);%(Outputs)</Outputs>
+ </CustomBuildStep>
+ <ClCompile>
+ <AdditionalIncludeDirectories>..\..\common;..\toolutil;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+ <PreprocessorDefinitions>WIN32;NDEBUG;_CRT_SECURE_NO_DEPRECATE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <Optimization>MaxSpeed</Optimization>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <StringPooling>true</StringPooling>
+ <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <DisableLanguageExtensions>false</DisableLanguageExtensions>
+ <TreatWChar_tAsBuiltInType>true</TreatWChar_tAsBuiltInType>
+ <PrecompiledHeaderOutputFile>.\x86\Release\gennorm2.pch</PrecompiledHeaderOutputFile>
+ <AssemblerListingLocation>.\x86\Release\</AssemblerListingLocation>
+ <ObjectFileName>.\x86\Release\</ObjectFileName>
+ <ProgramDataBaseFileName>.\x86\Release\</ProgramDataBaseFileName>
+ <WarningLevel>Level3</WarningLevel>
+ <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+ <SuppressStartupBanner>true</SuppressStartupBanner>
+ <CompileAs>Default</CompileAs>
+ </ClCompile>
+ <ResourceCompile>
+ <PreprocessorDefinitions>NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <Culture>0x0409</Culture>
+ </ResourceCompile>
+ <Link>
+ <OutputFile>.\x86\Release\gennorm2.exe</OutputFile>
+ <SuppressStartupBanner>true</SuppressStartupBanner>
+ <ProgramDatabaseFile>.\x86\Release\gennorm2.pdb</ProgramDatabaseFile>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <SubSystem>Console</SubSystem>
+ <OptimizeReferences>true</OptimizeReferences>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <TargetMachine>MachineX86</TargetMachine>
+ <RandomizedBaseAddress>false</RandomizedBaseAddress>
+ <DataExecutionPrevention>false</DataExecutionPrevention>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <CustomBuildStep>
+ <Command>copy "$(TargetPath)" ..\..\..\bin
+</Command>
+ <Outputs>..\..\..\bin\$(TargetFileName);%(Outputs)</Outputs>
+ </CustomBuildStep>
+ <ClCompile>
+ <Optimization>Disabled</Optimization>
+ <AdditionalIncludeDirectories>..\..\common;..\toolutil;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+ <PreprocessorDefinitions>WIN32;_DEBUG;_CRT_SECURE_NO_DEPRECATE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <MinimalRebuild>true</MinimalRebuild>
+ <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
+ <StringPooling>true</StringPooling>
+ <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
+ <BufferSecurityCheck>true</BufferSecurityCheck>
+ <DisableLanguageExtensions>false</DisableLanguageExtensions>
+ <TreatWChar_tAsBuiltInType>true</TreatWChar_tAsBuiltInType>
+ <PrecompiledHeaderOutputFile>.\x86\Debug\gennorm2.pch</PrecompiledHeaderOutputFile>
+ <AssemblerListingLocation>.\x86\Debug\</AssemblerListingLocation>
+ <ObjectFileName>.\x86\Debug\</ObjectFileName>
+ <ProgramDataBaseFileName>.\x86\Debug\</ProgramDataBaseFileName>
+ <BrowseInformation>true</BrowseInformation>
+ <WarningLevel>Level3</WarningLevel>
+ <SuppressStartupBanner>true</SuppressStartupBanner>
+ <DebugInformationFormat>EditAndContinue</DebugInformationFormat>
+ <CompileAs>Default</CompileAs>
+ </ClCompile>
+ <ResourceCompile>
+ <PreprocessorDefinitions>_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <Culture>0x0409</Culture>
+ </ResourceCompile>
+ <Link>
+ <OutputFile>.\x86\Debug\gennorm2.exe</OutputFile>
+ <SuppressStartupBanner>true</SuppressStartupBanner>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <ProgramDatabaseFile>.\x86\Debug\gennorm2.pdb</ProgramDatabaseFile>
+ <SubSystem>Console</SubSystem>
+ <TargetMachine>MachineX86</TargetMachine>
+ <RandomizedBaseAddress>false</RandomizedBaseAddress>
+ <DataExecutionPrevention>false</DataExecutionPrevention>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <CustomBuildStep>
+ <Command>copy "$(TargetPath)" ..\..\..\bin64
+</Command>
+ <Outputs>..\..\..\bin64\$(TargetFileName);%(Outputs)</Outputs>
+ </CustomBuildStep>
+ <ClCompile>
+ <AdditionalIncludeDirectories>..\..\common;..\toolutil;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+ <PreprocessorDefinitions>WIN64;WIN32;NDEBUG;_CRT_SECURE_NO_DEPRECATE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <Optimization>MaxSpeed</Optimization>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <StringPooling>true</StringPooling>
+ <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <DisableLanguageExtensions>false</DisableLanguageExtensions>
+ <TreatWChar_tAsBuiltInType>true</TreatWChar_tAsBuiltInType>
+ <PrecompiledHeaderOutputFile>.\x64\Release\gennorm2.pch</PrecompiledHeaderOutputFile>
+ <AssemblerListingLocation>.\x64\Release\</AssemblerListingLocation>
+ <ObjectFileName>.\x64\Release\</ObjectFileName>
+ <ProgramDataBaseFileName>.\x64\Release\</ProgramDataBaseFileName>
+ <WarningLevel>Level3</WarningLevel>
+ <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+ <SuppressStartupBanner>true</SuppressStartupBanner>
+ <CompileAs>Default</CompileAs>
+ </ClCompile>
+ <ResourceCompile>
+ <PreprocessorDefinitions>NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <Culture>0x0409</Culture>
+ </ResourceCompile>
+ <Link>
+ <OutputFile>.\x64\Release\gennorm2.exe</OutputFile>
+ <SuppressStartupBanner>true</SuppressStartupBanner>
+ <ProgramDatabaseFile>.\x64\Release\gennorm2.pdb</ProgramDatabaseFile>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <SubSystem>Console</SubSystem>
+ <OptimizeReferences>true</OptimizeReferences>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <TargetMachine>MachineX64</TargetMachine>
+ <RandomizedBaseAddress>false</RandomizedBaseAddress>
+ <DataExecutionPrevention>false</DataExecutionPrevention>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <CustomBuildStep>
+ <Command>copy "$(TargetPath)" ..\..\..\bin64
+</Command>
+ <Outputs>..\..\..\bin64\$(TargetFileName);%(Outputs)</Outputs>
+ </CustomBuildStep>
+ <ClCompile>
+ <Optimization>Disabled</Optimization>
+ <AdditionalIncludeDirectories>..\..\common;..\toolutil;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+ <PreprocessorDefinitions>WIN64;WIN32;_DEBUG;_CRT_SECURE_NO_DEPRECATE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <MinimalRebuild>true</MinimalRebuild>
+ <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
+ <StringPooling>true</StringPooling>
+ <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
+ <BufferSecurityCheck>true</BufferSecurityCheck>
+ <DisableLanguageExtensions>false</DisableLanguageExtensions>
+ <TreatWChar_tAsBuiltInType>true</TreatWChar_tAsBuiltInType>
+ <PrecompiledHeaderOutputFile>.\x64\Debug\gennorm2.pch</PrecompiledHeaderOutputFile>
+ <AssemblerListingLocation>.\x64\Debug\</AssemblerListingLocation>
+ <ObjectFileName>.\x64\Debug\</ObjectFileName>
+ <ProgramDataBaseFileName>.\x64\Debug\</ProgramDataBaseFileName>
+ <BrowseInformation>true</BrowseInformation>
+ <WarningLevel>Level3</WarningLevel>
+ <SuppressStartupBanner>true</SuppressStartupBanner>
+ <DebugInformationFormat>EditAndContinue</DebugInformationFormat>
+ <CompileAs>Default</CompileAs>
+ </ClCompile>
+ <ResourceCompile>
+ <PreprocessorDefinitions>_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <Culture>0x0409</Culture>
+ </ResourceCompile>
+ <Link>
+ <OutputFile>.\x64\Debug\gennorm2.exe</OutputFile>
+ <SuppressStartupBanner>true</SuppressStartupBanner>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <ProgramDatabaseFile>.\x64\Debug\gennorm2.pdb</ProgramDatabaseFile>
+ <SubSystem>Console</SubSystem>
+ <TargetMachine>MachineX64</TargetMachine>
+ <RandomizedBaseAddress>false</RandomizedBaseAddress>
+ <DataExecutionPrevention>false</DataExecutionPrevention>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemGroup>
+ <ClCompile Include="gennorm2.cpp" />
+ <ClCompile Include="n2builder.cpp" />
+ </ItemGroup>
+ <ItemGroup>
+ <ClInclude Include="n2builder.h" />
+ </ItemGroup>
+ <ItemGroup>
+ <ProjectReference Include="..\..\common\common.vcxproj">
+ <Project>{73c0a65b-d1f2-4de1-b3a6-15dad2c23f3d}</Project>
+ <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
+ </ProjectReference>
+ <ProjectReference Include="..\toolutil\toolutil.vcxproj">
+ <Project>{6b231032-3cb5-4eed-9210-810d666a23a0}</Project>
+ <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
+ </ProjectReference>
+ </ItemGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+ <ImportGroup Label="ExtensionTargets">
+ </ImportGroup>
+</Project> \ No newline at end of file
diff --git a/intl/icu/source/tools/gennorm2/n2builder.cpp b/intl/icu/source/tools/gennorm2/n2builder.cpp
new file mode 100644
index 000000000..54ad860f3
--- /dev/null
+++ b/intl/icu/source/tools/gennorm2/n2builder.cpp
@@ -0,0 +1,1307 @@
+// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// License & terms of use: http://www.unicode.org/copyright.html
+/*
+*******************************************************************************
+*
+* Copyright (C) 2009-2016, International Business Machines
+* Corporation and others. All Rights Reserved.
+*
+*******************************************************************************
+* file name: n2builder.cpp
+* encoding: US-ASCII
+* tab size: 8 (not used)
+* indentation:4
+*
+* created on: 2009nov25
+* created by: Markus W. Scherer
+*
+* Builds Normalizer2 data and writes a binary .nrm file.
+* For the file format see source/common/normalizer2impl.h.
+*/
+
+#include "unicode/utypes.h"
+#include "n2builder.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#if U_HAVE_STD_STRING
+#include <vector>
+#endif
+#include "unicode/errorcode.h"
+#include "unicode/localpointer.h"
+#include "unicode/putil.h"
+#include "unicode/udata.h"
+#include "unicode/uniset.h"
+#include "unicode/unistr.h"
+#include "unicode/ustring.h"
+#include "charstr.h"
+#include "hash.h"
+#include "normalizer2impl.h"
+#include "toolutil.h"
+#include "unewdata.h"
+#include "utrie2.h"
+#include "uvectr32.h"
+#include "writesrc.h"
+
+#if !UCONFIG_NO_NORMALIZATION
+
+/* UDataInfo cf. udata.h */
+static UDataInfo dataInfo={
+ sizeof(UDataInfo),
+ 0,
+
+ U_IS_BIG_ENDIAN,
+ U_CHARSET_FAMILY,
+ U_SIZEOF_UCHAR,
+ 0,
+
+ { 0x4e, 0x72, 0x6d, 0x32 }, /* dataFormat="Nrm2" */
+ { 2, 0, 0, 0 }, /* formatVersion */
+ { 5, 2, 0, 0 } /* dataVersion (Unicode version) */
+};
+
+U_NAMESPACE_BEGIN
+
+class HangulIterator {
+public:
+ struct Range {
+ UChar32 start, limit;
+ uint16_t norm16;
+ };
+
+ HangulIterator() : rangeIndex(0) {}
+ const Range *nextRange() {
+ if(rangeIndex<UPRV_LENGTHOF(ranges)) {
+ return ranges+rangeIndex++;
+ } else {
+ return NULL;
+ }
+ }
+ void reset() { rangeIndex=0; }
+private:
+ static const Range ranges[4];
+ int32_t rangeIndex;
+};
+
+const HangulIterator::Range HangulIterator::ranges[4]={
+ { Hangul::JAMO_L_BASE, Hangul::JAMO_L_BASE+Hangul::JAMO_L_COUNT, 1 },
+ { Hangul::JAMO_V_BASE, Hangul::JAMO_V_BASE+Hangul::JAMO_V_COUNT, Normalizer2Impl::JAMO_VT },
+ // JAMO_T_BASE+1: not U+11A7
+ { Hangul::JAMO_T_BASE+1, Hangul::JAMO_T_BASE+Hangul::JAMO_T_COUNT, Normalizer2Impl::JAMO_VT },
+ { Hangul::HANGUL_BASE, Hangul::HANGUL_BASE+Hangul::HANGUL_COUNT, 0 }, // will become minYesNo
+};
+
+struct CompositionPair {
+ CompositionPair(UChar32 t, UChar32 c) : trail(t), composite(c) {}
+ UChar32 trail, composite;
+};
+
+struct Norm {
+ enum MappingType { NONE, REMOVED, ROUND_TRIP, ONE_WAY };
+
+ UBool hasMapping() const { return mappingType>REMOVED; }
+
+ // Requires hasMapping() and well-formed mapping.
+ void setMappingCP() {
+ UChar32 c;
+ if(!mapping->isEmpty() && mapping->length()==U16_LENGTH(c=mapping->char32At(0))) {
+ mappingCP=c;
+ } else {
+ mappingCP=U_SENTINEL;
+ }
+ }
+
+ const CompositionPair *getCompositionPairs(int32_t &length) const {
+ if(compositions==NULL) {
+ length=0;
+ return NULL;
+ } else {
+ length=compositions->size()/2;
+ return reinterpret_cast<const CompositionPair *>(compositions->getBuffer());
+ }
+ }
+
+ UnicodeString *mapping;
+ UnicodeString *rawMapping; // non-NULL if the mapping is further decomposed
+ UChar32 mappingCP; // >=0 if mapping to 1 code point
+ int32_t mappingPhase;
+ MappingType mappingType;
+
+ UVector32 *compositions; // (trail, composite) pairs
+ uint8_t cc;
+ UBool combinesBack;
+ UBool hasNoCompBoundaryAfter;
+
+ enum OffsetType {
+ OFFSET_NONE,
+ // Composition for back-combining character. Allowed, but not normally used.
+ OFFSET_MAYBE_YES,
+ // Composition for a starter that does not have a decomposition mapping.
+ OFFSET_YES_YES,
+ // Round-trip mapping & composition for a starter.
+ OFFSET_YES_NO_MAPPING_AND_COMPOSITION,
+ // Round-trip mapping for a starter that itself does not combine-forward.
+ OFFSET_YES_NO_MAPPING_ONLY,
+ // One-way mapping.
+ OFFSET_NO_NO,
+ // Delta for an algorithmic one-way mapping.
+ OFFSET_DELTA
+ };
+ enum { OFFSET_SHIFT=4, OFFSET_MASK=(1<<OFFSET_SHIFT)-1 };
+ int32_t offset;
+};
+
+class Normalizer2DBEnumerator {
+public:
+ Normalizer2DBEnumerator(Normalizer2DataBuilder &b) : builder(b) {}
+ virtual ~Normalizer2DBEnumerator() {}
+ virtual UBool rangeHandler(UChar32 start, UChar32 end, uint32_t value) = 0;
+ Normalizer2DBEnumerator *ptr() { return this; }
+protected:
+ Normalizer2DataBuilder &builder;
+};
+
+U_CDECL_BEGIN
+
+static UBool U_CALLCONV
+enumRangeHandler(const void *context, UChar32 start, UChar32 end, uint32_t value) {
+ return ((Normalizer2DBEnumerator *)context)->rangeHandler(start, end, value);
+}
+
+U_CDECL_END
+
+Normalizer2DataBuilder::Normalizer2DataBuilder(UErrorCode &errorCode) :
+ phase(0), overrideHandling(OVERRIDE_PREVIOUS), optimization(OPTIMIZE_NORMAL),
+ norm16TrieLength(0) {
+ memset(unicodeVersion, 0, sizeof(unicodeVersion));
+ normTrie=utrie2_open(0, 0, &errorCode);
+ normMem=utm_open("gennorm2 normalization structs", 10000, 0x110100, sizeof(Norm));
+ norms=allocNorm(); // unused Norm struct at index 0
+ memset(indexes, 0, sizeof(indexes));
+ memset(smallFCD, 0, sizeof(smallFCD));
+}
+
+Normalizer2DataBuilder::~Normalizer2DataBuilder() {
+ utrie2_close(normTrie);
+ int32_t normsLength=utm_countItems(normMem);
+ for(int32_t i=1; i<normsLength; ++i) {
+ delete norms[i].mapping;
+ delete norms[i].rawMapping;
+ delete norms[i].compositions;
+ }
+ utm_close(normMem);
+ utrie2_close(norm16Trie);
+}
+
+void
+Normalizer2DataBuilder::setUnicodeVersion(const char *v) {
+ UVersionInfo nullVersion={ 0, 0, 0, 0 };
+ UVersionInfo version;
+ u_versionFromString(version, v);
+ if( 0!=memcmp(version, unicodeVersion, U_MAX_VERSION_LENGTH) &&
+ 0!=memcmp(nullVersion, unicodeVersion, U_MAX_VERSION_LENGTH)
+ ) {
+ char buffer[U_MAX_VERSION_STRING_LENGTH];
+ u_versionToString(unicodeVersion, buffer);
+ fprintf(stderr, "gennorm2 error: multiple inconsistent Unicode version numbers %s vs. %s\n",
+ buffer, v);
+ exit(U_ILLEGAL_ARGUMENT_ERROR);
+ }
+ memcpy(unicodeVersion, version, U_MAX_VERSION_LENGTH);
+}
+
+Norm *Normalizer2DataBuilder::allocNorm() {
+ Norm *p=(Norm *)utm_alloc(normMem);
+ norms=(Norm *)utm_getStart(normMem); // in case it got reallocated
+ return p;
+}
+
+/* get an existing Norm unit */
+Norm *Normalizer2DataBuilder::getNorm(UChar32 c) {
+ uint32_t i=utrie2_get32(normTrie, c);
+ if(i==0) {
+ return NULL;
+ }
+ return norms+i;
+}
+
+const Norm &Normalizer2DataBuilder::getNormRef(UChar32 c) const {
+ return norms[utrie2_get32(normTrie, c)];
+}
+
+/*
+ * get or create a Norm unit;
+ * get or create the intermediate trie entries for it as well
+ */
+Norm *Normalizer2DataBuilder::createNorm(UChar32 c) {
+ uint32_t i=utrie2_get32(normTrie, c);
+ if(i!=0) {
+ return norms+i;
+ } else {
+ /* allocate Norm */
+ Norm *p=allocNorm();
+ IcuToolErrorCode errorCode("gennorm2/createNorm()");
+ utrie2_set32(normTrie, c, (uint32_t)(p-norms), errorCode);
+ return p;
+ }
+}
+
+Norm *Normalizer2DataBuilder::checkNormForMapping(Norm *p, UChar32 c) {
+ if(p!=NULL) {
+ if(p->mappingType!=Norm::NONE) {
+ if( overrideHandling==OVERRIDE_NONE ||
+ (overrideHandling==OVERRIDE_PREVIOUS && p->mappingPhase==phase)
+ ) {
+ fprintf(stderr,
+ "error in gennorm2 phase %d: "
+ "not permitted to override mapping for U+%04lX from phase %d\n",
+ (int)phase, (long)c, (int)p->mappingPhase);
+ exit(U_INVALID_FORMAT_ERROR);
+ }
+ delete p->mapping;
+ p->mapping=NULL;
+ }
+ p->mappingPhase=phase;
+ }
+ return p;
+}
+
+void Normalizer2DataBuilder::setOverrideHandling(OverrideHandling oh) {
+ overrideHandling=oh;
+ ++phase;
+}
+
+void Normalizer2DataBuilder::setCC(UChar32 c, uint8_t cc) {
+ createNorm(c)->cc=cc;
+}
+
+uint8_t Normalizer2DataBuilder::getCC(UChar32 c) const {
+ return getNormRef(c).cc;
+}
+
+static UBool isWellFormed(const UnicodeString &s) {
+ UErrorCode errorCode=U_ZERO_ERROR;
+ u_strToUTF8(NULL, 0, NULL, s.getBuffer(), s.length(), &errorCode);
+ return U_SUCCESS(errorCode) || errorCode==U_BUFFER_OVERFLOW_ERROR;
+}
+
+void Normalizer2DataBuilder::setOneWayMapping(UChar32 c, const UnicodeString &m) {
+ if(!isWellFormed(m)) {
+ fprintf(stderr,
+ "error in gennorm2 phase %d: "
+ "illegal one-way mapping from U+%04lX to malformed string\n",
+ (int)phase, (long)c);
+ exit(U_INVALID_FORMAT_ERROR);
+ }
+ Norm *p=checkNormForMapping(createNorm(c), c);
+ p->mapping=new UnicodeString(m);
+ p->mappingType=Norm::ONE_WAY;
+ p->setMappingCP();
+}
+
+void Normalizer2DataBuilder::setRoundTripMapping(UChar32 c, const UnicodeString &m) {
+ if(U_IS_SURROGATE(c)) {
+ fprintf(stderr,
+ "error in gennorm2 phase %d: "
+ "illegal round-trip mapping from surrogate code point U+%04lX\n",
+ (int)phase, (long)c);
+ exit(U_INVALID_FORMAT_ERROR);
+ }
+ if(!isWellFormed(m)) {
+ fprintf(stderr,
+ "error in gennorm2 phase %d: "
+ "illegal round-trip mapping from U+%04lX to malformed string\n",
+ (int)phase, (long)c);
+ exit(U_INVALID_FORMAT_ERROR);
+ }
+ int32_t numCP=u_countChar32(m.getBuffer(), m.length());
+ if(numCP!=2) {
+ fprintf(stderr,
+ "error in gennorm2 phase %d: "
+ "illegal round-trip mapping from U+%04lX to %d!=2 code points\n",
+ (int)phase, (long)c, (int)numCP);
+ exit(U_INVALID_FORMAT_ERROR);
+ }
+ Norm *p=checkNormForMapping(createNorm(c), c);
+ p->mapping=new UnicodeString(m);
+ p->mappingType=Norm::ROUND_TRIP;
+ p->mappingCP=U_SENTINEL;
+}
+
+void Normalizer2DataBuilder::removeMapping(UChar32 c) {
+ Norm *p=checkNormForMapping(getNorm(c), c);
+ if(p!=NULL) {
+ p->mappingType=Norm::REMOVED;
+ }
+}
+
+class CompositionBuilder : public Normalizer2DBEnumerator {
+public:
+ CompositionBuilder(Normalizer2DataBuilder &b) : Normalizer2DBEnumerator(b) {}
+ virtual UBool rangeHandler(UChar32 start, UChar32 end, uint32_t value) {
+ builder.addComposition(start, end, value);
+ return TRUE;
+ }
+};
+
+void
+Normalizer2DataBuilder::addComposition(UChar32 start, UChar32 end, uint32_t value) {
+ if(norms[value].mappingType==Norm::ROUND_TRIP) {
+ if(start!=end) {
+ fprintf(stderr,
+ "gennorm2 error: same round-trip mapping for "
+ "more than 1 code point U+%04lX..U+%04lX\n",
+ (long)start, (long)end);
+ exit(U_INVALID_FORMAT_ERROR);
+ }
+ if(norms[value].cc!=0) {
+ fprintf(stderr,
+ "gennorm2 error: "
+ "U+%04lX has a round-trip mapping and ccc!=0, "
+ "not possible in Unicode normalization\n",
+ (long)start);
+ exit(U_INVALID_FORMAT_ERROR);
+ }
+ // setRoundTripMapping() ensured that there are exactly two code points.
+ const UnicodeString &m=*norms[value].mapping;
+ UChar32 lead=m.char32At(0);
+ UChar32 trail=m.char32At(m.length()-1);
+ if(getCC(lead)!=0) {
+ fprintf(stderr,
+ "gennorm2 error: "
+ "U+%04lX's round-trip mapping's starter U+%04lX has ccc!=0, "
+ "not possible in Unicode normalization\n",
+ (long)start, (long)lead);
+ exit(U_INVALID_FORMAT_ERROR);
+ }
+ // Flag for trailing character.
+ createNorm(trail)->combinesBack=TRUE;
+ // Insert (trail, composite) pair into compositions list for the lead character.
+ IcuToolErrorCode errorCode("gennorm2/addComposition()");
+ Norm *leadNorm=createNorm(lead);
+ UVector32 *compositions=leadNorm->compositions;
+ int32_t i;
+ if(compositions==NULL) {
+ compositions=leadNorm->compositions=new UVector32(errorCode);
+ i=0; // "insert" the first pair at index 0
+ } else {
+ // Insertion sort, and check for duplicate trail characters.
+ int32_t length;
+ const CompositionPair *pairs=leadNorm->getCompositionPairs(length);
+ for(i=0; i<length; ++i) {
+ if(trail==pairs[i].trail) {
+ fprintf(stderr,
+ "gennorm2 error: same round-trip mapping for "
+ "more than 1 code point (e.g., U+%04lX) to U+%04lX + U+%04lX\n",
+ (long)start, (long)lead, (long)trail);
+ exit(U_INVALID_FORMAT_ERROR);
+ }
+ if(trail<pairs[i].trail) {
+ break;
+ }
+ }
+ }
+ compositions->insertElementAt(trail, 2*i, errorCode);
+ compositions->insertElementAt(start, 2*i+1, errorCode);
+ }
+}
+
+UBool Normalizer2DataBuilder::combinesWithCCBetween(const Norm &norm,
+ uint8_t lowCC, uint8_t highCC) const {
+ if((highCC-lowCC)>=2) {
+ int32_t length;
+ const CompositionPair *pairs=norm.getCompositionPairs(length);
+ for(int32_t i=0; i<length; ++i) {
+ uint8_t trailCC=getCC(pairs[i].trail);
+ if(lowCC<trailCC && trailCC<highCC) {
+ return TRUE;
+ }
+ }
+ }
+ return FALSE;
+}
+
+UChar32 Normalizer2DataBuilder::combine(const Norm &norm, UChar32 trail) const {
+ int32_t length;
+ const CompositionPair *pairs=norm.getCompositionPairs(length);
+ for(int32_t i=0; i<length; ++i) {
+ if(trail==pairs[i].trail) {
+ return pairs[i].composite;
+ }
+ if(trail<pairs[i].trail) {
+ break;
+ }
+ }
+ return U_SENTINEL;
+}
+
+class Decomposer : public Normalizer2DBEnumerator {
+public:
+ Decomposer(Normalizer2DataBuilder &b) : Normalizer2DBEnumerator(b), didDecompose(FALSE) {}
+ virtual UBool rangeHandler(UChar32 start, UChar32 end, uint32_t value) {
+ didDecompose|=builder.decompose(start, end, value);
+ return TRUE;
+ }
+ UBool didDecompose;
+};
+
+UBool
+Normalizer2DataBuilder::decompose(UChar32 start, UChar32 end, uint32_t value) {
+ if(norms[value].hasMapping()) {
+ Norm &norm=norms[value];
+ const UnicodeString &m=*norm.mapping;
+ UnicodeString *decomposed=NULL;
+ const UChar *s=m.getBuffer();
+ int32_t length=m.length();
+ int32_t prev, i=0;
+ UChar32 c;
+ while(i<length) {
+ prev=i;
+ U16_NEXT(s, i, length, c);
+ if(start<=c && c<=end) {
+ fprintf(stderr,
+ "gennorm2 error: U+%04lX maps to itself directly or indirectly\n",
+ (long)c);
+ exit(U_INVALID_FORMAT_ERROR);
+ }
+ const Norm &cNorm=getNormRef(c);
+ if(cNorm.hasMapping()) {
+ if(norm.mappingType==Norm::ROUND_TRIP) {
+ if(prev==0) {
+ if(cNorm.mappingType!=Norm::ROUND_TRIP) {
+ fprintf(stderr,
+ "gennorm2 error: "
+ "U+%04lX's round-trip mapping's starter "
+ "U+%04lX one-way-decomposes, "
+ "not possible in Unicode normalization\n",
+ (long)start, (long)c);
+ exit(U_INVALID_FORMAT_ERROR);
+ }
+ uint8_t myTrailCC=getCC(m.char32At(i));
+ UChar32 cTrailChar=cNorm.mapping->char32At(cNorm.mapping->length()-1);
+ uint8_t cTrailCC=getCC(cTrailChar);
+ if(cTrailCC>myTrailCC) {
+ fprintf(stderr,
+ "gennorm2 error: "
+ "U+%04lX's round-trip mapping's starter "
+ "U+%04lX decomposes and the "
+ "inner/earlier tccc=%hu > outer/following tccc=%hu, "
+ "not possible in Unicode normalization\n",
+ (long)start, (long)c,
+ (short)cTrailCC, (short)myTrailCC);
+ exit(U_INVALID_FORMAT_ERROR);
+ }
+ } else {
+ fprintf(stderr,
+ "gennorm2 error: "
+ "U+%04lX's round-trip mapping's non-starter "
+ "U+%04lX decomposes, "
+ "not possible in Unicode normalization\n",
+ (long)start, (long)c);
+ exit(U_INVALID_FORMAT_ERROR);
+ }
+ }
+ if(decomposed==NULL) {
+ decomposed=new UnicodeString(m, 0, prev);
+ }
+ decomposed->append(*cNorm.mapping);
+ } else if(Hangul::isHangul(c)) {
+ UChar buffer[3];
+ int32_t hangulLength=Hangul::decompose(c, buffer);
+ if(norm.mappingType==Norm::ROUND_TRIP && prev!=0) {
+ fprintf(stderr,
+ "gennorm2 error: "
+ "U+%04lX's round-trip mapping's non-starter "
+ "U+%04lX decomposes, "
+ "not possible in Unicode normalization\n",
+ (long)start, (long)c);
+ exit(U_INVALID_FORMAT_ERROR);
+ }
+ if(decomposed==NULL) {
+ decomposed=new UnicodeString(m, 0, prev);
+ }
+ decomposed->append(buffer, hangulLength);
+ } else if(decomposed!=NULL) {
+ decomposed->append(m, prev, i-prev);
+ }
+ }
+ if(decomposed!=NULL) {
+ if(norm.rawMapping==NULL) {
+ // Remember the original mapping when decomposing recursively.
+ norm.rawMapping=norm.mapping;
+ } else {
+ delete norm.mapping;
+ }
+ norm.mapping=decomposed;
+ // Not norm.setMappingCP(); because the original mapping
+ // is most likely to be encodable as a delta.
+ return TRUE;
+ }
+ }
+ return FALSE;
+}
+
+class BuilderReorderingBuffer {
+public:
+ BuilderReorderingBuffer() : fLength(0), fLastStarterIndex(-1), fDidReorder(FALSE) {}
+ void reset() {
+ fLength=0;
+ fLastStarterIndex=-1;
+ fDidReorder=FALSE;
+ }
+ int32_t length() const { return fLength; }
+ UBool isEmpty() const { return fLength==0; }
+ int32_t lastStarterIndex() const { return fLastStarterIndex; }
+ UChar32 charAt(int32_t i) const { return fArray[i]>>8; }
+ uint8_t ccAt(int32_t i) const { return (uint8_t)fArray[i]; }
+ UBool didReorder() const { return fDidReorder; }
+ void append(UChar32 c, uint8_t cc) {
+ if(cc==0 || fLength==0 || ccAt(fLength-1)<=cc) {
+ if(cc==0) {
+ fLastStarterIndex=fLength;
+ }
+ fArray[fLength++]=(c<<8)|cc;
+ return;
+ }
+ // Let this character bubble back to its canonical order.
+ int32_t i=fLength-1;
+ while(i>fLastStarterIndex && ccAt(i)>cc) {
+ --i;
+ }
+ ++i; // after the last starter or prevCC<=cc
+ // Move this and the following characters forward one to make space.
+ for(int32_t j=fLength; i<j; --j) {
+ fArray[j]=fArray[j-1];
+ }
+ fArray[i]=(c<<8)|cc;
+ ++fLength;
+ fDidReorder=TRUE;
+ }
+ void toString(UnicodeString &dest) {
+ dest.remove();
+ for(int32_t i=0; i<fLength; ++i) {
+ dest.append(charAt(i));
+ }
+ }
+ void setComposite(UChar32 composite, int32_t combMarkIndex) {
+ fArray[fLastStarterIndex]=composite<<8;
+ // Remove the combining mark that contributed to the composite.
+ --fLength;
+ while(combMarkIndex<fLength) {
+ fArray[combMarkIndex]=fArray[combMarkIndex+1];
+ ++combMarkIndex;
+ }
+ }
+private:
+ int32_t fArray[Normalizer2Impl::MAPPING_LENGTH_MASK];
+ int32_t fLength;
+ int32_t fLastStarterIndex;
+ UBool fDidReorder;
+};
+
+void
+Normalizer2DataBuilder::reorder(Norm *p, BuilderReorderingBuffer &buffer) {
+ UnicodeString &m=*p->mapping;
+ int32_t length=m.length();
+ if(length>Normalizer2Impl::MAPPING_LENGTH_MASK) {
+ return; // writeMapping() will complain about it and print the code point.
+ }
+ const UChar *s=m.getBuffer();
+ int32_t i=0;
+ UChar32 c;
+ while(i<length) {
+ U16_NEXT(s, i, length, c);
+ buffer.append(c, getCC(c));
+ }
+ if(buffer.didReorder()) {
+ buffer.toString(m);
+ }
+}
+
+/*
+ * Computes the flag for the last code branch in Normalizer2Impl::hasCompBoundaryAfter().
+ * A starter character with a mapping does not have a composition boundary after it
+ * if the character itself combines-forward (which is tested by the caller of this function),
+ * or it is deleted (mapped to the empty string),
+ * or its mapping contains no starter,
+ * or the last starter combines-forward.
+ */
+UBool Normalizer2DataBuilder::hasNoCompBoundaryAfter(BuilderReorderingBuffer &buffer) {
+ if(buffer.isEmpty()) {
+ return TRUE; // maps-to-empty-string is no boundary of any kind
+ }
+ int32_t lastStarterIndex=buffer.lastStarterIndex();
+ if(lastStarterIndex<0) {
+ return TRUE; // no starter
+ }
+ UChar32 starter=buffer.charAt(lastStarterIndex);
+ if( Hangul::isJamoL(starter) ||
+ (Hangul::isJamoV(starter) &&
+ 0<lastStarterIndex && Hangul::isJamoL(buffer.charAt(lastStarterIndex-1)))
+ ) {
+ // A Jamo leading consonant or an LV pair combines-forward if it is at the end,
+ // otherwise it is blocked.
+ return lastStarterIndex==buffer.length()-1;
+ }
+ // Note: There can be no Hangul syllable in the fully decomposed mapping.
+ const Norm *starterNorm=&getNormRef(starter);
+ if(starterNorm->compositions==NULL) {
+ return FALSE; // the last starter does not combine forward
+ }
+ // Compose as far as possible, and see if further compositions are possible.
+ uint8_t prevCC=0;
+ for(int32_t combMarkIndex=lastStarterIndex+1; combMarkIndex<buffer.length();) {
+ uint8_t cc=buffer.ccAt(combMarkIndex); // !=0 because after last starter
+ if(combinesWithCCBetween(*starterNorm, prevCC, cc)) {
+ return TRUE;
+ }
+ if( prevCC<cc &&
+ (starter=combine(*starterNorm, buffer.charAt(combMarkIndex)))>=0
+ ) {
+ buffer.setComposite(starter, combMarkIndex);
+ starterNorm=&getNormRef(starter);
+ if(starterNorm->compositions==NULL) {
+ return FALSE; // the composite does not combine further
+ }
+ } else {
+ prevCC=cc;
+ ++combMarkIndex;
+ }
+ }
+ // TRUE if the final, forward-combining starter is at the end.
+ return prevCC==0;
+}
+
+// Requires p->hasMapping().
+// Returns the offset of the "first unit" from the beginning of the extraData for c.
+// That is the same as the length of the optional data for the raw mapping and the ccc/lccc word.
+int32_t Normalizer2DataBuilder::writeMapping(UChar32 c, const Norm *p, UnicodeString &dataString) {
+ UnicodeString &m=*p->mapping;
+ int32_t length=m.length();
+ if(length>Normalizer2Impl::MAPPING_LENGTH_MASK) {
+ fprintf(stderr,
+ "gennorm2 error: "
+ "mapping for U+%04lX longer than maximum of %d\n",
+ (long)c, Normalizer2Impl::MAPPING_LENGTH_MASK);
+ exit(U_INVALID_FORMAT_ERROR);
+ }
+ int32_t leadCC, trailCC;
+ if(length==0) {
+ leadCC=trailCC=0;
+ } else {
+ leadCC=getCC(m.char32At(0));
+ trailCC=getCC(m.char32At(length-1));
+ }
+ if(c<Normalizer2Impl::MIN_CCC_LCCC_CP && (p->cc!=0 || leadCC!=0)) {
+ fprintf(stderr,
+ "gennorm2 error: "
+ "U+%04lX below U+0300 has ccc!=0 or lccc!=0, not supported by ICU\n",
+ (long)c);
+ exit(U_INVALID_FORMAT_ERROR);
+ }
+ // Write small-FCD data.
+ if((leadCC|trailCC)!=0) {
+ UChar32 lead= c<=0xffff ? c : U16_LEAD(c);
+ smallFCD[lead>>8]|=(uint8_t)1<<((lead>>5)&7);
+ }
+ // Write the mapping & raw mapping extraData.
+ int32_t firstUnit=length|(trailCC<<8);
+ int32_t preMappingLength=0;
+ if(p->rawMapping!=NULL) {
+ UnicodeString &rm=*p->rawMapping;
+ int32_t rmLength=rm.length();
+ if(rmLength>Normalizer2Impl::MAPPING_LENGTH_MASK) {
+ fprintf(stderr,
+ "gennorm2 error: "
+ "raw mapping for U+%04lX longer than maximum of %d\n",
+ (long)c, Normalizer2Impl::MAPPING_LENGTH_MASK);
+ exit(U_INVALID_FORMAT_ERROR);
+ }
+ UChar rm0=rm.charAt(0);
+ if( rmLength==length-1 &&
+ // 99: overlong substring lengths get pinned to remainder lengths anyway
+ 0==rm.compare(1, 99, m, 2, 99) &&
+ rm0>Normalizer2Impl::MAPPING_LENGTH_MASK
+ ) {
+ // Compression:
+ // rawMapping=rm0+mapping.substring(2) -> store only rm0
+ //
+ // The raw mapping is the same as the final mapping after replacing
+ // the final mapping's first two code units with the raw mapping's first one.
+ // In this case, we store only that first unit, rm0.
+ // This helps with a few hundred mappings.
+ dataString.append(rm0);
+ preMappingLength=1;
+ } else {
+ // Store the raw mapping with its length.
+ dataString.append(rm);
+ dataString.append((UChar)rmLength);
+ preMappingLength=rmLength+1;
+ }
+ firstUnit|=Normalizer2Impl::MAPPING_HAS_RAW_MAPPING;
+ }
+ int32_t cccLccc=p->cc|(leadCC<<8);
+ if(cccLccc!=0) {
+ dataString.append((UChar)cccLccc);
+ ++preMappingLength;
+ firstUnit|=Normalizer2Impl::MAPPING_HAS_CCC_LCCC_WORD;
+ }
+ if(p->hasNoCompBoundaryAfter) {
+ firstUnit|=Normalizer2Impl::MAPPING_NO_COMP_BOUNDARY_AFTER;
+ }
+ dataString.append((UChar)firstUnit);
+ dataString.append(m);
+ return preMappingLength;
+}
+
+// Requires p->compositions!=NULL.
+void Normalizer2DataBuilder::writeCompositions(UChar32 c, const Norm *p, UnicodeString &dataString) {
+ if(p->cc!=0) {
+ fprintf(stderr,
+ "gennorm2 error: "
+ "U+%04lX combines-forward and has ccc!=0, not possible in Unicode normalization\n",
+ (long)c);
+ exit(U_INVALID_FORMAT_ERROR);
+ }
+ int32_t length;
+ const CompositionPair *pairs=p->getCompositionPairs(length);
+ for(int32_t i=0; i<length; ++i) {
+ const CompositionPair &pair=pairs[i];
+ // 22 bits for the composite character and whether it combines forward.
+ UChar32 compositeAndFwd=pair.composite<<1;
+ if(getNormRef(pair.composite).compositions!=NULL) {
+ compositeAndFwd|=1; // The composite character also combines-forward.
+ }
+ // Encode most pairs in two units and some in three.
+ int32_t firstUnit, secondUnit, thirdUnit;
+ if(pair.trail<Normalizer2Impl::COMP_1_TRAIL_LIMIT) {
+ if(compositeAndFwd<=0xffff) {
+ firstUnit=pair.trail<<1;
+ secondUnit=compositeAndFwd;
+ thirdUnit=-1;
+ } else {
+ firstUnit=(pair.trail<<1)|Normalizer2Impl::COMP_1_TRIPLE;
+ secondUnit=compositeAndFwd>>16;
+ thirdUnit=compositeAndFwd;
+ }
+ } else {
+ firstUnit=(Normalizer2Impl::COMP_1_TRAIL_LIMIT+
+ (pair.trail>>Normalizer2Impl::COMP_1_TRAIL_SHIFT))|
+ Normalizer2Impl::COMP_1_TRIPLE;
+ secondUnit=(pair.trail<<Normalizer2Impl::COMP_2_TRAIL_SHIFT)|
+ (compositeAndFwd>>16);
+ thirdUnit=compositeAndFwd;
+ }
+ // Set the high bit of the first unit if this is the last composition pair.
+ if(i==(length-1)) {
+ firstUnit|=Normalizer2Impl::COMP_1_LAST_TUPLE;
+ }
+ dataString.append((UChar)firstUnit).append((UChar)secondUnit);
+ if(thirdUnit>=0) {
+ dataString.append((UChar)thirdUnit);
+ }
+ }
+}
+
+class ExtraDataWriter : public Normalizer2DBEnumerator {
+public:
+ ExtraDataWriter(Normalizer2DataBuilder &b) :
+ Normalizer2DBEnumerator(b),
+ yesYesCompositions(1000, (UChar32)0xffff, 2), // 0=inert, 1=Jamo L, 2=start of compositions
+ yesNoMappingsAndCompositions(1000, (UChar32)0, 1) {} // 0=Hangul, 1=start of normal data
+ virtual UBool rangeHandler(UChar32 start, UChar32 end, uint32_t value) {
+ if(value!=0) {
+ if(start!=end) {
+ fprintf(stderr,
+ "gennorm2 error: unexpected shared data for "
+ "multiple code points U+%04lX..U+%04lX\n",
+ (long)start, (long)end);
+ exit(U_INTERNAL_PROGRAM_ERROR);
+ }
+ builder.writeExtraData(start, value, *this);
+ }
+ return TRUE;
+ }
+ UnicodeString maybeYesCompositions;
+ UnicodeString yesYesCompositions;
+ UnicodeString yesNoMappingsAndCompositions;
+ UnicodeString yesNoMappingsOnly;
+ UnicodeString noNoMappings;
+ Hashtable previousNoNoMappings; // If constructed in runtime code, pass in UErrorCode.
+};
+
+void Normalizer2DataBuilder::writeExtraData(UChar32 c, uint32_t value, ExtraDataWriter &writer) {
+ Norm *p=norms+value;
+ if(!p->hasMapping()) {
+ // Write small-FCD data.
+ // There is similar code in writeMapping() for characters that do have a mapping.
+ if(c<Normalizer2Impl::MIN_CCC_LCCC_CP && p->cc!=0) {
+ fprintf(stderr,
+ "gennorm2 error: "
+ "U+%04lX below U+0300 has ccc!=0, not supported by ICU\n",
+ (long)c);
+ exit(U_INVALID_FORMAT_ERROR);
+ }
+ if(p->cc!=0) {
+ UChar32 lead= c<=0xffff ? c : U16_LEAD(c);
+ smallFCD[lead>>8]|=(uint8_t)1<<((lead>>5)&7);
+ }
+ }
+ if(p->combinesBack) {
+ if(p->hasMapping()) {
+ fprintf(stderr,
+ "gennorm2 error: "
+ "U+%04lX combines-back and decomposes, not possible in Unicode normalization\n",
+ (long)c);
+ exit(U_INVALID_FORMAT_ERROR);
+ }
+ if(p->compositions!=NULL) {
+ p->offset=
+ (writer.maybeYesCompositions.length()<<Norm::OFFSET_SHIFT)|
+ Norm::OFFSET_MAYBE_YES;
+ writeCompositions(c, p, writer.maybeYesCompositions);
+ }
+ } else if(!p->hasMapping()) {
+ if(p->compositions!=NULL) {
+ p->offset=
+ (writer.yesYesCompositions.length()<<Norm::OFFSET_SHIFT)|
+ Norm::OFFSET_YES_YES;
+ writeCompositions(c, p, writer.yesYesCompositions);
+ }
+ } else if(p->mappingType==Norm::ROUND_TRIP) {
+ if(p->compositions!=NULL) {
+ int32_t offset=writer.yesNoMappingsAndCompositions.length()+
+ writeMapping(c, p, writer.yesNoMappingsAndCompositions);
+ p->offset=(offset<<Norm::OFFSET_SHIFT)|Norm::OFFSET_YES_NO_MAPPING_AND_COMPOSITION;
+ writeCompositions(c, p, writer.yesNoMappingsAndCompositions);
+ } else {
+ int32_t offset=writer.yesNoMappingsOnly.length()+
+ writeMapping(c, p, writer.yesNoMappingsOnly);
+ p->offset=(offset<<Norm::OFFSET_SHIFT)|Norm::OFFSET_YES_NO_MAPPING_ONLY;
+ }
+ } else /* one-way */ {
+ if(p->compositions!=NULL) {
+ fprintf(stderr,
+ "gennorm2 error: "
+ "U+%04lX combines-forward and has a one-way mapping, "
+ "not possible in Unicode normalization\n",
+ (long)c);
+ exit(U_INVALID_FORMAT_ERROR);
+ }
+ if(p->cc==0 && optimization!=OPTIMIZE_FAST) {
+ // Try a compact, algorithmic encoding.
+ // Only for ccc=0, because we can't store additional information
+ // and we do not recursively follow an algorithmic encoding for access to the ccc.
+ //
+ // Also, if hasNoCompBoundaryAfter is set, we can only use the algorithmic encoding
+ // if the mappingCP decomposes further, to ensure that there is a place to store it.
+ // We want to see that the final mapping does not have exactly 1 code point,
+ // or else we would have to recursively ensure that the final mapping is stored
+ // in normal extraData.
+ if(p->mappingCP>=0 && (!p->hasNoCompBoundaryAfter || 1!=p->mapping->countChar32())) {
+ int32_t delta=p->mappingCP-c;
+ if(-Normalizer2Impl::MAX_DELTA<=delta && delta<=Normalizer2Impl::MAX_DELTA) {
+ p->offset=(delta<<Norm::OFFSET_SHIFT)|Norm::OFFSET_DELTA;
+ }
+ }
+ }
+ if(p->offset==0) {
+ int32_t oldNoNoLength=writer.noNoMappings.length();
+ int32_t offset=oldNoNoLength+writeMapping(c, p, writer.noNoMappings);
+ UnicodeString newMapping=writer.noNoMappings.tempSubString(oldNoNoLength);
+ int32_t previousOffset=writer.previousNoNoMappings.geti(newMapping);
+ if(previousOffset!=0) {
+ // Duplicate, remove the new units and point to the old ones.
+ writer.noNoMappings.truncate(oldNoNoLength);
+ p->offset=((previousOffset-1)<<Norm::OFFSET_SHIFT)|Norm::OFFSET_NO_NO;
+ } else {
+ // Enter this new mapping into the hashtable, avoiding value 0 which is "not found".
+ IcuToolErrorCode errorCode("gennorm2/writeExtraData()/Hashtable.puti()");
+ writer.previousNoNoMappings.puti(newMapping, offset+1, errorCode);
+ p->offset=(offset<<Norm::OFFSET_SHIFT)|Norm::OFFSET_NO_NO;
+ }
+ }
+ }
+}
+
+class Norm16Writer : public Normalizer2DBEnumerator {
+public:
+ Norm16Writer(Normalizer2DataBuilder &b) : Normalizer2DBEnumerator(b) {}
+ virtual UBool rangeHandler(UChar32 start, UChar32 end, uint32_t value) {
+ builder.writeNorm16(start, end, value);
+ return TRUE;
+ }
+};
+
+void Normalizer2DataBuilder::writeNorm16(UChar32 start, UChar32 end, uint32_t value) {
+ if(value!=0) {
+ const Norm *p=norms+value;
+ int32_t offset=p->offset>>Norm::OFFSET_SHIFT;
+ int32_t norm16=0;
+ UBool isDecompNo=FALSE;
+ UBool isCompNoMaybe=FALSE;
+ switch(p->offset&Norm::OFFSET_MASK) {
+ case Norm::OFFSET_NONE:
+ // No mapping, no compositions list.
+ if(p->combinesBack) {
+ norm16=Normalizer2Impl::MIN_NORMAL_MAYBE_YES+p->cc;
+ isDecompNo=(UBool)(p->cc!=0);
+ isCompNoMaybe=TRUE;
+ } else if(p->cc!=0) {
+ norm16=Normalizer2Impl::MIN_YES_YES_WITH_CC-1+p->cc;
+ isDecompNo=isCompNoMaybe=TRUE;
+ }
+ break;
+ case Norm::OFFSET_MAYBE_YES:
+ norm16=indexes[Normalizer2Impl::IX_MIN_MAYBE_YES]+offset;
+ isCompNoMaybe=TRUE;
+ break;
+ case Norm::OFFSET_YES_YES:
+ norm16=offset;
+ break;
+ case Norm::OFFSET_YES_NO_MAPPING_AND_COMPOSITION:
+ norm16=indexes[Normalizer2Impl::IX_MIN_YES_NO]+offset;
+ isDecompNo=TRUE;
+ break;
+ case Norm::OFFSET_YES_NO_MAPPING_ONLY:
+ norm16=indexes[Normalizer2Impl::IX_MIN_YES_NO_MAPPINGS_ONLY]+offset;
+ isDecompNo=TRUE;
+ break;
+ case Norm::OFFSET_NO_NO:
+ norm16=indexes[Normalizer2Impl::IX_MIN_NO_NO]+offset;
+ isDecompNo=isCompNoMaybe=TRUE;
+ break;
+ case Norm::OFFSET_DELTA:
+ norm16=getCenterNoNoDelta()+offset;
+ isDecompNo=isCompNoMaybe=TRUE;
+ break;
+ default: // Should not occur.
+ exit(U_INTERNAL_PROGRAM_ERROR);
+ }
+ IcuToolErrorCode errorCode("gennorm2/writeNorm16()");
+ utrie2_setRange32(norm16Trie, start, end, (uint32_t)norm16, TRUE, errorCode);
+ if(isDecompNo && start<indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]) {
+ indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]=start;
+ }
+ if(isCompNoMaybe && start<indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]) {
+ indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]=start;
+ }
+ }
+}
+
+void Normalizer2DataBuilder::setHangulData() {
+ HangulIterator hi;
+ const HangulIterator::Range *range;
+ // Check that none of the Hangul/Jamo code points have data.
+ while((range=hi.nextRange())!=NULL) {
+ for(UChar32 c=range->start; c<range->limit; ++c) {
+ if(utrie2_get32(norm16Trie, c)!=0) {
+ fprintf(stderr,
+ "gennorm2 error: "
+ "illegal mapping/composition/ccc data for Hangul or Jamo U+%04lX\n",
+ (long)c);
+ exit(U_INVALID_FORMAT_ERROR);
+ }
+ }
+ }
+ // Set data for algorithmic runtime handling.
+ IcuToolErrorCode errorCode("gennorm2/setHangulData()");
+ hi.reset();
+ while((range=hi.nextRange())!=NULL) {
+ uint16_t norm16=range->norm16;
+ if(norm16==0) {
+ norm16=(uint16_t)indexes[Normalizer2Impl::IX_MIN_YES_NO]; // Hangul LV/LVT encoded as minYesNo
+ if(range->start<indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]) {
+ indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]=range->start;
+ }
+ } else {
+ if(range->start<indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]) { // Jamo V/T are maybeYes
+ indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]=range->start;
+ }
+ }
+ utrie2_setRange32(norm16Trie, range->start, range->limit-1, norm16, TRUE, errorCode);
+ errorCode.assertSuccess();
+ }
+}
+
+U_CDECL_BEGIN
+
+static UBool U_CALLCONV
+enumRangeMaxValue(const void *context, UChar32 /*start*/, UChar32 /*end*/, uint32_t value) {
+ uint32_t *pMaxValue=(uint32_t *)context;
+ if(value>*pMaxValue) {
+ *pMaxValue=value;
+ }
+ return TRUE;
+}
+
+U_CDECL_END
+
+void Normalizer2DataBuilder::processData() {
+ IcuToolErrorCode errorCode("gennorm2/processData()");
+ norm16Trie=utrie2_open(0, 0, errorCode);
+ errorCode.assertSuccess();
+
+ utrie2_enum(normTrie, NULL, enumRangeHandler, CompositionBuilder(*this).ptr());
+
+ Decomposer decomposer(*this);
+ do {
+ decomposer.didDecompose=FALSE;
+ utrie2_enum(normTrie, NULL, enumRangeHandler, &decomposer);
+ } while(decomposer.didDecompose);
+
+ BuilderReorderingBuffer buffer;
+ int32_t normsLength=utm_countItems(normMem);
+ for(int32_t i=1; i<normsLength; ++i) {
+ // Set the hasNoCompBoundaryAfter flag for use by the last code branch
+ // in Normalizer2Impl::hasCompBoundaryAfter().
+ // For details see the comments on hasNoCompBoundaryAfter(buffer).
+ const Norm &norm=norms[i];
+ if(norm.hasMapping()) {
+ if(norm.compositions!=NULL) {
+ norms[i].hasNoCompBoundaryAfter=TRUE;
+ } else {
+ buffer.reset();
+ reorder(norms+i, buffer);
+ norms[i].hasNoCompBoundaryAfter=hasNoCompBoundaryAfter(buffer);
+ }
+ }
+ }
+
+ indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]=0x110000;
+ indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]=0x110000;
+
+ ExtraDataWriter extraDataWriter(*this);
+ utrie2_enum(normTrie, NULL, enumRangeHandler, &extraDataWriter);
+
+ extraData=extraDataWriter.maybeYesCompositions;
+ extraData.append(extraDataWriter.yesYesCompositions).
+ append(extraDataWriter.yesNoMappingsAndCompositions).
+ append(extraDataWriter.yesNoMappingsOnly).
+ append(extraDataWriter.noNoMappings);
+ // Pad to even length for 4-byte alignment of following data.
+ if(extraData.length()&1) {
+ extraData.append((UChar)0);
+ }
+
+ indexes[Normalizer2Impl::IX_MIN_YES_NO]=
+ extraDataWriter.yesYesCompositions.length();
+ indexes[Normalizer2Impl::IX_MIN_YES_NO_MAPPINGS_ONLY]=
+ indexes[Normalizer2Impl::IX_MIN_YES_NO]+
+ extraDataWriter.yesNoMappingsAndCompositions.length();
+ indexes[Normalizer2Impl::IX_MIN_NO_NO]=
+ indexes[Normalizer2Impl::IX_MIN_YES_NO_MAPPINGS_ONLY]+
+ extraDataWriter.yesNoMappingsOnly.length();
+ indexes[Normalizer2Impl::IX_LIMIT_NO_NO]=
+ indexes[Normalizer2Impl::IX_MIN_NO_NO]+
+ extraDataWriter.noNoMappings.length();
+ indexes[Normalizer2Impl::IX_MIN_MAYBE_YES]=
+ Normalizer2Impl::MIN_NORMAL_MAYBE_YES-
+ extraDataWriter.maybeYesCompositions.length();
+
+ int32_t minNoNoDelta=getCenterNoNoDelta()-Normalizer2Impl::MAX_DELTA;
+ if(indexes[Normalizer2Impl::IX_LIMIT_NO_NO]>minNoNoDelta) {
+ fprintf(stderr,
+ "gennorm2 error: "
+ "data structure overflow, too much mapping composition data\n");
+ exit(U_BUFFER_OVERFLOW_ERROR);
+ }
+
+ utrie2_enum(normTrie, NULL, enumRangeHandler, Norm16Writer(*this).ptr());
+
+ setHangulData();
+
+ // Look for the "worst" norm16 value of any supplementary code point
+ // corresponding to a lead surrogate, and set it as that surrogate's value.
+ // Enables quick check inner loops to look at only code units.
+ //
+ // We could be more sophisticated:
+ // We could collect a bit set for whether there are values in the different
+ // norm16 ranges (yesNo, maybeYes, yesYesWithCC etc.)
+ // and select the best value that only breaks the composition and/or decomposition
+ // inner loops if necessary.
+ // However, that seems like overkill for an optimization for supplementary characters.
+ for(UChar lead=0xd800; lead<0xdc00; ++lead) {
+ uint32_t maxValue=utrie2_get32(norm16Trie, lead);
+ utrie2_enumForLeadSurrogate(norm16Trie, lead, NULL, enumRangeMaxValue, &maxValue);
+ if( maxValue>=(uint32_t)indexes[Normalizer2Impl::IX_LIMIT_NO_NO] &&
+ maxValue>(uint32_t)indexes[Normalizer2Impl::IX_MIN_NO_NO]
+ ) {
+ // Set noNo ("worst" value) if it got into "less-bad" maybeYes or ccc!=0.
+ // Otherwise it might end up at something like JAMO_VT which stays in
+ // the inner decomposition quick check loop.
+ maxValue=(uint32_t)indexes[Normalizer2Impl::IX_LIMIT_NO_NO]-1;
+ }
+ utrie2_set32ForLeadSurrogateCodeUnit(norm16Trie, lead, maxValue, errorCode);
+ }
+
+ // Adjust supplementary minimum code points to break quick check loops at their lead surrogates.
+ // For an empty data file, minCP=0x110000 turns into 0xdc00 (first trail surrogate)
+ // which is harmless.
+ // As a result, the minimum code points are always BMP code points.
+ int32_t minCP=indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP];
+ if(minCP>=0x10000) {
+ indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]=U16_LEAD(minCP);
+ }
+ minCP=indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP];
+ if(minCP>=0x10000) {
+ indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]=U16_LEAD(minCP);
+ }
+
+ utrie2_freeze(norm16Trie, UTRIE2_16_VALUE_BITS, errorCode);
+ norm16TrieLength=utrie2_serialize(norm16Trie, NULL, 0, errorCode);
+ if(errorCode.get()!=U_BUFFER_OVERFLOW_ERROR) {
+ fprintf(stderr, "gennorm2 error: unable to freeze/serialize the normalization trie - %s\n",
+ errorCode.errorName());
+ exit(errorCode.reset());
+ }
+ errorCode.reset();
+
+ int32_t offset=(int32_t)sizeof(indexes);
+ indexes[Normalizer2Impl::IX_NORM_TRIE_OFFSET]=offset;
+ offset+=norm16TrieLength;
+ indexes[Normalizer2Impl::IX_EXTRA_DATA_OFFSET]=offset;
+ offset+=extraData.length()*2;
+ indexes[Normalizer2Impl::IX_SMALL_FCD_OFFSET]=offset;
+ offset+=sizeof(smallFCD);
+ int32_t totalSize=offset;
+ for(int32_t i=Normalizer2Impl::IX_RESERVED3_OFFSET; i<=Normalizer2Impl::IX_TOTAL_SIZE; ++i) {
+ indexes[i]=totalSize;
+ }
+
+ if(beVerbose) {
+ printf("size of normalization trie: %5ld bytes\n", (long)norm16TrieLength);
+ printf("size of 16-bit extra data: %5ld uint16_t\n", (long)extraData.length());
+ printf("size of small-FCD data: %5ld bytes\n", (long)sizeof(smallFCD));
+ printf("size of binary data file contents: %5ld bytes\n", (long)totalSize);
+ printf("minDecompNoCodePoint: U+%04lX\n", (long)indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]);
+ printf("minCompNoMaybeCodePoint: U+%04lX\n", (long)indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]);
+ printf("minYesNo: 0x%04x\n", (int)indexes[Normalizer2Impl::IX_MIN_YES_NO]);
+ printf("minYesNoMappingsOnly: 0x%04x\n", (int)indexes[Normalizer2Impl::IX_MIN_YES_NO_MAPPINGS_ONLY]);
+ printf("minNoNo: 0x%04x\n", (int)indexes[Normalizer2Impl::IX_MIN_NO_NO]);
+ printf("limitNoNo: 0x%04x\n", (int)indexes[Normalizer2Impl::IX_LIMIT_NO_NO]);
+ printf("minMaybeYes: 0x%04x\n", (int)indexes[Normalizer2Impl::IX_MIN_MAYBE_YES]);
+ }
+
+ UVersionInfo nullVersion={ 0, 0, 0, 0 };
+ if(0==memcmp(nullVersion, unicodeVersion, 4)) {
+ u_versionFromString(unicodeVersion, U_UNICODE_VERSION);
+ }
+ memcpy(dataInfo.dataVersion, unicodeVersion, 4);
+}
+
+void Normalizer2DataBuilder::writeBinaryFile(const char *filename) {
+ processData();
+
+ IcuToolErrorCode errorCode("gennorm2/writeBinaryFile()");
+ LocalArray<uint8_t> norm16TrieBytes(new uint8_t[norm16TrieLength]);
+ utrie2_serialize(norm16Trie, norm16TrieBytes.getAlias(), norm16TrieLength, errorCode);
+ errorCode.assertSuccess();
+
+ UNewDataMemory *pData=
+ udata_create(NULL, NULL, filename, &dataInfo,
+ haveCopyright ? U_COPYRIGHT_STRING : NULL, errorCode);
+ if(errorCode.isFailure()) {
+ fprintf(stderr, "gennorm2 error: unable to create the output file %s - %s\n",
+ filename, errorCode.errorName());
+ exit(errorCode.reset());
+ }
+ udata_writeBlock(pData, indexes, sizeof(indexes));
+ udata_writeBlock(pData, norm16TrieBytes.getAlias(), norm16TrieLength);
+ udata_writeUString(pData, extraData.getBuffer(), extraData.length());
+ udata_writeBlock(pData, smallFCD, sizeof(smallFCD));
+ int32_t writtenSize=udata_finish(pData, errorCode);
+ if(errorCode.isFailure()) {
+ fprintf(stderr, "gennorm2: error %s writing the output file\n", errorCode.errorName());
+ exit(errorCode.reset());
+ }
+ int32_t totalSize=indexes[Normalizer2Impl::IX_TOTAL_SIZE];
+ if(writtenSize!=totalSize) {
+ fprintf(stderr, "gennorm2 error: written size %ld != calculated size %ld\n",
+ (long)writtenSize, (long)totalSize);
+ exit(U_INTERNAL_PROGRAM_ERROR);
+ }
+}
+
+void
+Normalizer2DataBuilder::writeCSourceFile(const char *filename) {
+ processData();
+
+ IcuToolErrorCode errorCode("gennorm2/writeCSourceFile()");
+ const char *basename=findBasename(filename);
+ CharString path(filename, (int32_t)(basename-filename), errorCode);
+ CharString dataName(basename, errorCode);
+ const char *extension=strrchr(basename, '.');
+ if(extension!=NULL) {
+ dataName.truncate((int32_t)(extension-basename));
+ }
+ errorCode.assertSuccess();
+
+ LocalArray<uint8_t> norm16TrieBytes(new uint8_t[norm16TrieLength]);
+ utrie2_serialize(norm16Trie, norm16TrieBytes.getAlias(), norm16TrieLength, errorCode);
+ errorCode.assertSuccess();
+
+ FILE *f=usrc_create(path.data(), basename, "icu/source/tools/gennorm2/n2builder.cpp");
+ if(f==NULL) {
+ fprintf(stderr, "gennorm2/writeCSourceFile() error: unable to create the output file %s\n",
+ filename);
+ exit(U_FILE_ACCESS_ERROR);
+ return;
+ }
+ fputs("#ifdef INCLUDED_FROM_NORMALIZER2_CPP\n\n", f);
+ char line[100];
+ sprintf(line, "static const UVersionInfo %s_formatVersion={", dataName.data());
+ usrc_writeArray(f, line, dataInfo.formatVersion, 8, 4, "};\n");
+ sprintf(line, "static const UVersionInfo %s_dataVersion={", dataName.data());
+ usrc_writeArray(f, line, dataInfo.dataVersion, 8, 4, "};\n\n");
+ sprintf(line, "static const int32_t %s_indexes[Normalizer2Impl::IX_COUNT]={\n",
+ dataName.data());
+ usrc_writeArray(f,
+ line,
+ indexes, 32, Normalizer2Impl::IX_COUNT,
+ "\n};\n\n");
+ sprintf(line, "static const uint16_t %s_trieIndex[%%ld]={\n", dataName.data());
+ usrc_writeUTrie2Arrays(f,
+ line, NULL,
+ norm16Trie,
+ "\n};\n\n");
+ sprintf(line, "static const uint16_t %s_extraData[%%ld]={\n", dataName.data());
+ usrc_writeArray(f,
+ line,
+ extraData.getBuffer(), 16, extraData.length(),
+ "\n};\n\n");
+ sprintf(line, "static const uint8_t %s_smallFCD[%%ld]={\n", dataName.data());
+ usrc_writeArray(f,
+ line,
+ smallFCD, 8, sizeof(smallFCD),
+ "\n};\n\n");
+ /*fputs( // TODO
+ "static const UCaseProps %s_singleton={\n"
+ " NULL,\n"
+ " %s_indexes,\n"
+ " %s_extraData,\n"
+ " %s_smallFCD,\n",
+ f);*/
+ sprintf(line, "static const UTrie2 %s_trie={\n", dataName.data());
+ char line2[100];
+ sprintf(line2, "%s_trieIndex", dataName.data());
+ usrc_writeUTrie2Struct(f,
+ line,
+ norm16Trie, line2, NULL,
+ "};\n");
+ fputs("\n#endif // INCLUDED_FROM_NORMALIZER2_CPP\n", f);
+ fclose(f);
+}
+
+U_NAMESPACE_END
+
+#endif /* #if !UCONFIG_NO_NORMALIZATION */
+
+/*
+ * Hey, Emacs, please set the following:
+ *
+ * Local Variables:
+ * indent-tabs-mode: nil
+ * End:
+ */
diff --git a/intl/icu/source/tools/gennorm2/n2builder.h b/intl/icu/source/tools/gennorm2/n2builder.h
new file mode 100644
index 000000000..6f8b8afff
--- /dev/null
+++ b/intl/icu/source/tools/gennorm2/n2builder.h
@@ -0,0 +1,127 @@
+// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// License & terms of use: http://www.unicode.org/copyright.html
+/*
+*******************************************************************************
+*
+* Copyright (C) 2009-2014, International Business Machines
+* Corporation and others. All Rights Reserved.
+*
+*******************************************************************************
+* file name: n2builder.h
+* encoding: US-ASCII
+* tab size: 8 (not used)
+* indentation:4
+*
+* created on: 2009nov25
+* created by: Markus W. Scherer
+*/
+
+#ifndef __N2BUILDER_H__
+#define __N2BUILDER_H__
+
+#include "unicode/utypes.h"
+
+#if !UCONFIG_NO_NORMALIZATION
+
+#include "unicode/errorcode.h"
+#include "unicode/unistr.h"
+#include "normalizer2impl.h" // for IX_COUNT
+#include "toolutil.h"
+#include "utrie2.h"
+
+U_NAMESPACE_BEGIN
+
+extern UBool beVerbose, haveCopyright;
+
+struct Norm;
+
+class BuilderReorderingBuffer;
+class ExtraDataWriter;
+
+class Normalizer2DataBuilder {
+public:
+ Normalizer2DataBuilder(UErrorCode &errorCode);
+ ~Normalizer2DataBuilder();
+
+ enum OverrideHandling {
+ OVERRIDE_NONE,
+ OVERRIDE_ANY,
+ OVERRIDE_PREVIOUS
+ };
+
+ void setOverrideHandling(OverrideHandling oh);
+
+ enum Optimization {
+ OPTIMIZE_NORMAL,
+ OPTIMIZE_FAST
+ };
+
+ void setOptimization(Optimization opt) { optimization=opt; }
+
+ void setCC(UChar32 c, uint8_t cc);
+ void setOneWayMapping(UChar32 c, const UnicodeString &m);
+ void setRoundTripMapping(UChar32 c, const UnicodeString &m);
+ void removeMapping(UChar32 c);
+
+ void setUnicodeVersion(const char *v);
+
+ void writeBinaryFile(const char *filename);
+ void writeCSourceFile(const char *filename);
+
+private:
+ friend class CompositionBuilder;
+ friend class Decomposer;
+ friend class ExtraDataWriter;
+ friend class Norm16Writer;
+
+ // No copy constructor nor assignment operator.
+ Normalizer2DataBuilder(const Normalizer2DataBuilder &other);
+ Normalizer2DataBuilder &operator=(const Normalizer2DataBuilder &other);
+
+ Norm *allocNorm();
+ Norm *getNorm(UChar32 c);
+ Norm *createNorm(UChar32 c);
+ Norm *checkNormForMapping(Norm *p, UChar32 c); // check for permitted overrides
+
+ const Norm &getNormRef(UChar32 c) const;
+ uint8_t getCC(UChar32 c) const;
+ UBool combinesWithCCBetween(const Norm &norm, uint8_t lowCC, uint8_t highCC) const;
+ UChar32 combine(const Norm &norm, UChar32 trail) const;
+
+ void addComposition(UChar32 start, UChar32 end, uint32_t value);
+ UBool decompose(UChar32 start, UChar32 end, uint32_t value);
+ void reorder(Norm *p, BuilderReorderingBuffer &buffer);
+ UBool hasNoCompBoundaryAfter(BuilderReorderingBuffer &buffer);
+ void setHangulData();
+ int32_t writeMapping(UChar32 c, const Norm *p, UnicodeString &dataString);
+ void writeCompositions(UChar32 c, const Norm *p, UnicodeString &dataString);
+ void writeExtraData(UChar32 c, uint32_t value, ExtraDataWriter &writer);
+ int32_t getCenterNoNoDelta() {
+ return indexes[Normalizer2Impl::IX_MIN_MAYBE_YES]-Normalizer2Impl::MAX_DELTA-1;
+ }
+ void writeNorm16(UChar32 start, UChar32 end, uint32_t value);
+ void processData();
+
+ UTrie2 *normTrie;
+ UToolMemory *normMem;
+ Norm *norms;
+
+ int32_t phase;
+ OverrideHandling overrideHandling;
+
+ Optimization optimization;
+
+ int32_t indexes[Normalizer2Impl::IX_COUNT];
+ UTrie2 *norm16Trie;
+ int32_t norm16TrieLength;
+ UnicodeString extraData;
+ uint8_t smallFCD[0x100];
+
+ UVersionInfo unicodeVersion;
+};
+
+U_NAMESPACE_END
+
+#endif // #if !UCONFIG_NO_NORMALIZATION
+
+#endif // __N2BUILDER_H__
generated by cgit v1.2.3 (git 2.25.1) at 2024-10-20 00:28:43 +0000