diff options
Diffstat (limited to 'intl/icu/source/tools/gennorm2/n2builder.cpp')
| -rw-r--r-- | intl/icu/source/tools/gennorm2/n2builder.cpp | 1307 |
1 files changed, 1307 insertions, 0 deletions
diff --git a/intl/icu/source/tools/gennorm2/n2builder.cpp b/intl/icu/source/tools/gennorm2/n2builder.cpp new file mode 100644 index 000000000..54ad860f3 --- /dev/null +++ b/intl/icu/source/tools/gennorm2/n2builder.cpp @@ -0,0 +1,1307 @@ +// Copyright (C) 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* +* Copyright (C) 2009-2016, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: n2builder.cpp +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2009nov25 +* created by: Markus W. Scherer +* +* Builds Normalizer2 data and writes a binary .nrm file. +* For the file format see source/common/normalizer2impl.h. +*/ + +#include "unicode/utypes.h" +#include "n2builder.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#if U_HAVE_STD_STRING +#include <vector> +#endif +#include "unicode/errorcode.h" +#include "unicode/localpointer.h" +#include "unicode/putil.h" +#include "unicode/udata.h" +#include "unicode/uniset.h" +#include "unicode/unistr.h" +#include "unicode/ustring.h" +#include "charstr.h" +#include "hash.h" +#include "normalizer2impl.h" +#include "toolutil.h" +#include "unewdata.h" +#include "utrie2.h" +#include "uvectr32.h" +#include "writesrc.h" + +#if !UCONFIG_NO_NORMALIZATION + +/* UDataInfo cf. udata.h */ +static UDataInfo dataInfo={ + sizeof(UDataInfo), + 0, + + U_IS_BIG_ENDIAN, + U_CHARSET_FAMILY, + U_SIZEOF_UCHAR, + 0, + + { 0x4e, 0x72, 0x6d, 0x32 }, /* dataFormat="Nrm2" */ + { 2, 0, 0, 0 }, /* formatVersion */ + { 5, 2, 0, 0 } /* dataVersion (Unicode version) */ +}; + +U_NAMESPACE_BEGIN + +class HangulIterator { +public: + struct Range { + UChar32 start, limit; + uint16_t norm16; + }; + + HangulIterator() : rangeIndex(0) {} + const Range *nextRange() { + if(rangeIndex<UPRV_LENGTHOF(ranges)) { + return ranges+rangeIndex++; + } else { + return NULL; + } + } + void reset() { rangeIndex=0; } +private: + static const Range ranges[4]; + int32_t rangeIndex; +}; + +const HangulIterator::Range HangulIterator::ranges[4]={ + { Hangul::JAMO_L_BASE, Hangul::JAMO_L_BASE+Hangul::JAMO_L_COUNT, 1 }, + { Hangul::JAMO_V_BASE, Hangul::JAMO_V_BASE+Hangul::JAMO_V_COUNT, Normalizer2Impl::JAMO_VT }, + // JAMO_T_BASE+1: not U+11A7 + { Hangul::JAMO_T_BASE+1, Hangul::JAMO_T_BASE+Hangul::JAMO_T_COUNT, Normalizer2Impl::JAMO_VT }, + { Hangul::HANGUL_BASE, Hangul::HANGUL_BASE+Hangul::HANGUL_COUNT, 0 }, // will become minYesNo +}; + +struct CompositionPair { + CompositionPair(UChar32 t, UChar32 c) : trail(t), composite(c) {} + UChar32 trail, composite; +}; + +struct Norm { + enum MappingType { NONE, REMOVED, ROUND_TRIP, ONE_WAY }; + + UBool hasMapping() const { return mappingType>REMOVED; } + + // Requires hasMapping() and well-formed mapping. + void setMappingCP() { + UChar32 c; + if(!mapping->isEmpty() && mapping->length()==U16_LENGTH(c=mapping->char32At(0))) { + mappingCP=c; + } else { + mappingCP=U_SENTINEL; + } + } + + const CompositionPair *getCompositionPairs(int32_t &length) const { + if(compositions==NULL) { + length=0; + return NULL; + } else { + length=compositions->size()/2; + return reinterpret_cast<const CompositionPair *>(compositions->getBuffer()); + } + } + + UnicodeString *mapping; + UnicodeString *rawMapping; // non-NULL if the mapping is further decomposed + UChar32 mappingCP; // >=0 if mapping to 1 code point + int32_t mappingPhase; + MappingType mappingType; + + UVector32 *compositions; // (trail, composite) pairs + uint8_t cc; + UBool combinesBack; + UBool hasNoCompBoundaryAfter; + + enum OffsetType { + OFFSET_NONE, + // Composition for back-combining character. Allowed, but not normally used. + OFFSET_MAYBE_YES, + // Composition for a starter that does not have a decomposition mapping. + OFFSET_YES_YES, + // Round-trip mapping & composition for a starter. + OFFSET_YES_NO_MAPPING_AND_COMPOSITION, + // Round-trip mapping for a starter that itself does not combine-forward. + OFFSET_YES_NO_MAPPING_ONLY, + // One-way mapping. + OFFSET_NO_NO, + // Delta for an algorithmic one-way mapping. + OFFSET_DELTA + }; + enum { OFFSET_SHIFT=4, OFFSET_MASK=(1<<OFFSET_SHIFT)-1 }; + int32_t offset; +}; + +class Normalizer2DBEnumerator { +public: + Normalizer2DBEnumerator(Normalizer2DataBuilder &b) : builder(b) {} + virtual ~Normalizer2DBEnumerator() {} + virtual UBool rangeHandler(UChar32 start, UChar32 end, uint32_t value) = 0; + Normalizer2DBEnumerator *ptr() { return this; } +protected: + Normalizer2DataBuilder &builder; +}; + +U_CDECL_BEGIN + +static UBool U_CALLCONV +enumRangeHandler(const void *context, UChar32 start, UChar32 end, uint32_t value) { + return ((Normalizer2DBEnumerator *)context)->rangeHandler(start, end, value); +} + +U_CDECL_END + +Normalizer2DataBuilder::Normalizer2DataBuilder(UErrorCode &errorCode) : + phase(0), overrideHandling(OVERRIDE_PREVIOUS), optimization(OPTIMIZE_NORMAL), + norm16TrieLength(0) { + memset(unicodeVersion, 0, sizeof(unicodeVersion)); + normTrie=utrie2_open(0, 0, &errorCode); + normMem=utm_open("gennorm2 normalization structs", 10000, 0x110100, sizeof(Norm)); + norms=allocNorm(); // unused Norm struct at index 0 + memset(indexes, 0, sizeof(indexes)); + memset(smallFCD, 0, sizeof(smallFCD)); +} + +Normalizer2DataBuilder::~Normalizer2DataBuilder() { + utrie2_close(normTrie); + int32_t normsLength=utm_countItems(normMem); + for(int32_t i=1; i<normsLength; ++i) { + delete norms[i].mapping; + delete norms[i].rawMapping; + delete norms[i].compositions; + } + utm_close(normMem); + utrie2_close(norm16Trie); +} + +void +Normalizer2DataBuilder::setUnicodeVersion(const char *v) { + UVersionInfo nullVersion={ 0, 0, 0, 0 }; + UVersionInfo version; + u_versionFromString(version, v); + if( 0!=memcmp(version, unicodeVersion, U_MAX_VERSION_LENGTH) && + 0!=memcmp(nullVersion, unicodeVersion, U_MAX_VERSION_LENGTH) + ) { + char buffer[U_MAX_VERSION_STRING_LENGTH]; + u_versionToString(unicodeVersion, buffer); + fprintf(stderr, "gennorm2 error: multiple inconsistent Unicode version numbers %s vs. %s\n", + buffer, v); + exit(U_ILLEGAL_ARGUMENT_ERROR); + } + memcpy(unicodeVersion, version, U_MAX_VERSION_LENGTH); +} + +Norm *Normalizer2DataBuilder::allocNorm() { + Norm *p=(Norm *)utm_alloc(normMem); + norms=(Norm *)utm_getStart(normMem); // in case it got reallocated + return p; +} + +/* get an existing Norm unit */ +Norm *Normalizer2DataBuilder::getNorm(UChar32 c) { + uint32_t i=utrie2_get32(normTrie, c); + if(i==0) { + return NULL; + } + return norms+i; +} + +const Norm &Normalizer2DataBuilder::getNormRef(UChar32 c) const { + return norms[utrie2_get32(normTrie, c)]; +} + +/* + * get or create a Norm unit; + * get or create the intermediate trie entries for it as well + */ +Norm *Normalizer2DataBuilder::createNorm(UChar32 c) { + uint32_t i=utrie2_get32(normTrie, c); + if(i!=0) { + return norms+i; + } else { + /* allocate Norm */ + Norm *p=allocNorm(); + IcuToolErrorCode errorCode("gennorm2/createNorm()"); + utrie2_set32(normTrie, c, (uint32_t)(p-norms), errorCode); + return p; + } +} + +Norm *Normalizer2DataBuilder::checkNormForMapping(Norm *p, UChar32 c) { + if(p!=NULL) { + if(p->mappingType!=Norm::NONE) { + if( overrideHandling==OVERRIDE_NONE || + (overrideHandling==OVERRIDE_PREVIOUS && p->mappingPhase==phase) + ) { + fprintf(stderr, + "error in gennorm2 phase %d: " + "not permitted to override mapping for U+%04lX from phase %d\n", + (int)phase, (long)c, (int)p->mappingPhase); + exit(U_INVALID_FORMAT_ERROR); + } + delete p->mapping; + p->mapping=NULL; + } + p->mappingPhase=phase; + } + return p; +} + +void Normalizer2DataBuilder::setOverrideHandling(OverrideHandling oh) { + overrideHandling=oh; + ++phase; +} + +void Normalizer2DataBuilder::setCC(UChar32 c, uint8_t cc) { + createNorm(c)->cc=cc; +} + +uint8_t Normalizer2DataBuilder::getCC(UChar32 c) const { + return getNormRef(c).cc; +} + +static UBool isWellFormed(const UnicodeString &s) { + UErrorCode errorCode=U_ZERO_ERROR; + u_strToUTF8(NULL, 0, NULL, s.getBuffer(), s.length(), &errorCode); + return U_SUCCESS(errorCode) || errorCode==U_BUFFER_OVERFLOW_ERROR; +} + +void Normalizer2DataBuilder::setOneWayMapping(UChar32 c, const UnicodeString &m) { + if(!isWellFormed(m)) { + fprintf(stderr, + "error in gennorm2 phase %d: " + "illegal one-way mapping from U+%04lX to malformed string\n", + (int)phase, (long)c); + exit(U_INVALID_FORMAT_ERROR); + } + Norm *p=checkNormForMapping(createNorm(c), c); + p->mapping=new UnicodeString(m); + p->mappingType=Norm::ONE_WAY; + p->setMappingCP(); +} + +void Normalizer2DataBuilder::setRoundTripMapping(UChar32 c, const UnicodeString &m) { + if(U_IS_SURROGATE(c)) { + fprintf(stderr, + "error in gennorm2 phase %d: " + "illegal round-trip mapping from surrogate code point U+%04lX\n", + (int)phase, (long)c); + exit(U_INVALID_FORMAT_ERROR); + } + if(!isWellFormed(m)) { + fprintf(stderr, + "error in gennorm2 phase %d: " + "illegal round-trip mapping from U+%04lX to malformed string\n", + (int)phase, (long)c); + exit(U_INVALID_FORMAT_ERROR); + } + int32_t numCP=u_countChar32(m.getBuffer(), m.length()); + if(numCP!=2) { + fprintf(stderr, + "error in gennorm2 phase %d: " + "illegal round-trip mapping from U+%04lX to %d!=2 code points\n", + (int)phase, (long)c, (int)numCP); + exit(U_INVALID_FORMAT_ERROR); + } + Norm *p=checkNormForMapping(createNorm(c), c); + p->mapping=new UnicodeString(m); + p->mappingType=Norm::ROUND_TRIP; + p->mappingCP=U_SENTINEL; +} + +void Normalizer2DataBuilder::removeMapping(UChar32 c) { + Norm *p=checkNormForMapping(getNorm(c), c); + if(p!=NULL) { + p->mappingType=Norm::REMOVED; + } +} + +class CompositionBuilder : public Normalizer2DBEnumerator { +public: + CompositionBuilder(Normalizer2DataBuilder &b) : Normalizer2DBEnumerator(b) {} + virtual UBool rangeHandler(UChar32 start, UChar32 end, uint32_t value) { + builder.addComposition(start, end, value); + return TRUE; + } +}; + +void +Normalizer2DataBuilder::addComposition(UChar32 start, UChar32 end, uint32_t value) { + if(norms[value].mappingType==Norm::ROUND_TRIP) { + if(start!=end) { + fprintf(stderr, + "gennorm2 error: same round-trip mapping for " + "more than 1 code point U+%04lX..U+%04lX\n", + (long)start, (long)end); + exit(U_INVALID_FORMAT_ERROR); + } + if(norms[value].cc!=0) { + fprintf(stderr, + "gennorm2 error: " + "U+%04lX has a round-trip mapping and ccc!=0, " + "not possible in Unicode normalization\n", + (long)start); + exit(U_INVALID_FORMAT_ERROR); + } + // setRoundTripMapping() ensured that there are exactly two code points. + const UnicodeString &m=*norms[value].mapping; + UChar32 lead=m.char32At(0); + UChar32 trail=m.char32At(m.length()-1); + if(getCC(lead)!=0) { + fprintf(stderr, + "gennorm2 error: " + "U+%04lX's round-trip mapping's starter U+%04lX has ccc!=0, " + "not possible in Unicode normalization\n", + (long)start, (long)lead); + exit(U_INVALID_FORMAT_ERROR); + } + // Flag for trailing character. + createNorm(trail)->combinesBack=TRUE; + // Insert (trail, composite) pair into compositions list for the lead character. + IcuToolErrorCode errorCode("gennorm2/addComposition()"); + Norm *leadNorm=createNorm(lead); + UVector32 *compositions=leadNorm->compositions; + int32_t i; + if(compositions==NULL) { + compositions=leadNorm->compositions=new UVector32(errorCode); + i=0; // "insert" the first pair at index 0 + } else { + // Insertion sort, and check for duplicate trail characters. + int32_t length; + const CompositionPair *pairs=leadNorm->getCompositionPairs(length); + for(i=0; i<length; ++i) { + if(trail==pairs[i].trail) { + fprintf(stderr, + "gennorm2 error: same round-trip mapping for " + "more than 1 code point (e.g., U+%04lX) to U+%04lX + U+%04lX\n", + (long)start, (long)lead, (long)trail); + exit(U_INVALID_FORMAT_ERROR); + } + if(trail<pairs[i].trail) { + break; + } + } + } + compositions->insertElementAt(trail, 2*i, errorCode); + compositions->insertElementAt(start, 2*i+1, errorCode); + } +} + +UBool Normalizer2DataBuilder::combinesWithCCBetween(const Norm &norm, + uint8_t lowCC, uint8_t highCC) const { + if((highCC-lowCC)>=2) { + int32_t length; + const CompositionPair *pairs=norm.getCompositionPairs(length); + for(int32_t i=0; i<length; ++i) { + uint8_t trailCC=getCC(pairs[i].trail); + if(lowCC<trailCC && trailCC<highCC) { + return TRUE; + } + } + } + return FALSE; +} + +UChar32 Normalizer2DataBuilder::combine(const Norm &norm, UChar32 trail) const { + int32_t length; + const CompositionPair *pairs=norm.getCompositionPairs(length); + for(int32_t i=0; i<length; ++i) { + if(trail==pairs[i].trail) { + return pairs[i].composite; + } + if(trail<pairs[i].trail) { + break; + } + } + return U_SENTINEL; +} + +class Decomposer : public Normalizer2DBEnumerator { +public: + Decomposer(Normalizer2DataBuilder &b) : Normalizer2DBEnumerator(b), didDecompose(FALSE) {} + virtual UBool rangeHandler(UChar32 start, UChar32 end, uint32_t value) { + didDecompose|=builder.decompose(start, end, value); + return TRUE; + } + UBool didDecompose; +}; + +UBool +Normalizer2DataBuilder::decompose(UChar32 start, UChar32 end, uint32_t value) { + if(norms[value].hasMapping()) { + Norm &norm=norms[value]; + const UnicodeString &m=*norm.mapping; + UnicodeString *decomposed=NULL; + const UChar *s=m.getBuffer(); + int32_t length=m.length(); + int32_t prev, i=0; + UChar32 c; + while(i<length) { + prev=i; + U16_NEXT(s, i, length, c); + if(start<=c && c<=end) { + fprintf(stderr, + "gennorm2 error: U+%04lX maps to itself directly or indirectly\n", + (long)c); + exit(U_INVALID_FORMAT_ERROR); + } + const Norm &cNorm=getNormRef(c); + if(cNorm.hasMapping()) { + if(norm.mappingType==Norm::ROUND_TRIP) { + if(prev==0) { + if(cNorm.mappingType!=Norm::ROUND_TRIP) { + fprintf(stderr, + "gennorm2 error: " + "U+%04lX's round-trip mapping's starter " + "U+%04lX one-way-decomposes, " + "not possible in Unicode normalization\n", + (long)start, (long)c); + exit(U_INVALID_FORMAT_ERROR); + } + uint8_t myTrailCC=getCC(m.char32At(i)); + UChar32 cTrailChar=cNorm.mapping->char32At(cNorm.mapping->length()-1); + uint8_t cTrailCC=getCC(cTrailChar); + if(cTrailCC>myTrailCC) { + fprintf(stderr, + "gennorm2 error: " + "U+%04lX's round-trip mapping's starter " + "U+%04lX decomposes and the " + "inner/earlier tccc=%hu > outer/following tccc=%hu, " + "not possible in Unicode normalization\n", + (long)start, (long)c, + (short)cTrailCC, (short)myTrailCC); + exit(U_INVALID_FORMAT_ERROR); + } + } else { + fprintf(stderr, + "gennorm2 error: " + "U+%04lX's round-trip mapping's non-starter " + "U+%04lX decomposes, " + "not possible in Unicode normalization\n", + (long)start, (long)c); + exit(U_INVALID_FORMAT_ERROR); + } + } + if(decomposed==NULL) { + decomposed=new UnicodeString(m, 0, prev); + } + decomposed->append(*cNorm.mapping); + } else if(Hangul::isHangul(c)) { + UChar buffer[3]; + int32_t hangulLength=Hangul::decompose(c, buffer); + if(norm.mappingType==Norm::ROUND_TRIP && prev!=0) { + fprintf(stderr, + "gennorm2 error: " + "U+%04lX's round-trip mapping's non-starter " + "U+%04lX decomposes, " + "not possible in Unicode normalization\n", + (long)start, (long)c); + exit(U_INVALID_FORMAT_ERROR); + } + if(decomposed==NULL) { + decomposed=new UnicodeString(m, 0, prev); + } + decomposed->append(buffer, hangulLength); + } else if(decomposed!=NULL) { + decomposed->append(m, prev, i-prev); + } + } + if(decomposed!=NULL) { + if(norm.rawMapping==NULL) { + // Remember the original mapping when decomposing recursively. + norm.rawMapping=norm.mapping; + } else { + delete norm.mapping; + } + norm.mapping=decomposed; + // Not norm.setMappingCP(); because the original mapping + // is most likely to be encodable as a delta. + return TRUE; + } + } + return FALSE; +} + +class BuilderReorderingBuffer { +public: + BuilderReorderingBuffer() : fLength(0), fLastStarterIndex(-1), fDidReorder(FALSE) {} + void reset() { + fLength=0; + fLastStarterIndex=-1; + fDidReorder=FALSE; + } + int32_t length() const { return fLength; } + UBool isEmpty() const { return fLength==0; } + int32_t lastStarterIndex() const { return fLastStarterIndex; } + UChar32 charAt(int32_t i) const { return fArray[i]>>8; } + uint8_t ccAt(int32_t i) const { return (uint8_t)fArray[i]; } + UBool didReorder() const { return fDidReorder; } + void append(UChar32 c, uint8_t cc) { + if(cc==0 || fLength==0 || ccAt(fLength-1)<=cc) { + if(cc==0) { + fLastStarterIndex=fLength; + } + fArray[fLength++]=(c<<8)|cc; + return; + } + // Let this character bubble back to its canonical order. + int32_t i=fLength-1; + while(i>fLastStarterIndex && ccAt(i)>cc) { + --i; + } + ++i; // after the last starter or prevCC<=cc + // Move this and the following characters forward one to make space. + for(int32_t j=fLength; i<j; --j) { + fArray[j]=fArray[j-1]; + } + fArray[i]=(c<<8)|cc; + ++fLength; + fDidReorder=TRUE; + } + void toString(UnicodeString &dest) { + dest.remove(); + for(int32_t i=0; i<fLength; ++i) { + dest.append(charAt(i)); + } + } + void setComposite(UChar32 composite, int32_t combMarkIndex) { + fArray[fLastStarterIndex]=composite<<8; + // Remove the combining mark that contributed to the composite. + --fLength; + while(combMarkIndex<fLength) { + fArray[combMarkIndex]=fArray[combMarkIndex+1]; + ++combMarkIndex; + } + } +private: + int32_t fArray[Normalizer2Impl::MAPPING_LENGTH_MASK]; + int32_t fLength; + int32_t fLastStarterIndex; + UBool fDidReorder; +}; + +void +Normalizer2DataBuilder::reorder(Norm *p, BuilderReorderingBuffer &buffer) { + UnicodeString &m=*p->mapping; + int32_t length=m.length(); + if(length>Normalizer2Impl::MAPPING_LENGTH_MASK) { + return; // writeMapping() will complain about it and print the code point. + } + const UChar *s=m.getBuffer(); + int32_t i=0; + UChar32 c; + while(i<length) { + U16_NEXT(s, i, length, c); + buffer.append(c, getCC(c)); + } + if(buffer.didReorder()) { + buffer.toString(m); + } +} + +/* + * Computes the flag for the last code branch in Normalizer2Impl::hasCompBoundaryAfter(). + * A starter character with a mapping does not have a composition boundary after it + * if the character itself combines-forward (which is tested by the caller of this function), + * or it is deleted (mapped to the empty string), + * or its mapping contains no starter, + * or the last starter combines-forward. + */ +UBool Normalizer2DataBuilder::hasNoCompBoundaryAfter(BuilderReorderingBuffer &buffer) { + if(buffer.isEmpty()) { + return TRUE; // maps-to-empty-string is no boundary of any kind + } + int32_t lastStarterIndex=buffer.lastStarterIndex(); + if(lastStarterIndex<0) { + return TRUE; // no starter + } + UChar32 starter=buffer.charAt(lastStarterIndex); + if( Hangul::isJamoL(starter) || + (Hangul::isJamoV(starter) && + 0<lastStarterIndex && Hangul::isJamoL(buffer.charAt(lastStarterIndex-1))) + ) { + // A Jamo leading consonant or an LV pair combines-forward if it is at the end, + // otherwise it is blocked. + return lastStarterIndex==buffer.length()-1; + } + // Note: There can be no Hangul syllable in the fully decomposed mapping. + const Norm *starterNorm=&getNormRef(starter); + if(starterNorm->compositions==NULL) { + return FALSE; // the last starter does not combine forward + } + // Compose as far as possible, and see if further compositions are possible. + uint8_t prevCC=0; + for(int32_t combMarkIndex=lastStarterIndex+1; combMarkIndex<buffer.length();) { + uint8_t cc=buffer.ccAt(combMarkIndex); // !=0 because after last starter + if(combinesWithCCBetween(*starterNorm, prevCC, cc)) { + return TRUE; + } + if( prevCC<cc && + (starter=combine(*starterNorm, buffer.charAt(combMarkIndex)))>=0 + ) { + buffer.setComposite(starter, combMarkIndex); + starterNorm=&getNormRef(starter); + if(starterNorm->compositions==NULL) { + return FALSE; // the composite does not combine further + } + } else { + prevCC=cc; + ++combMarkIndex; + } + } + // TRUE if the final, forward-combining starter is at the end. + return prevCC==0; +} + +// Requires p->hasMapping(). +// Returns the offset of the "first unit" from the beginning of the extraData for c. +// That is the same as the length of the optional data for the raw mapping and the ccc/lccc word. +int32_t Normalizer2DataBuilder::writeMapping(UChar32 c, const Norm *p, UnicodeString &dataString) { + UnicodeString &m=*p->mapping; + int32_t length=m.length(); + if(length>Normalizer2Impl::MAPPING_LENGTH_MASK) { + fprintf(stderr, + "gennorm2 error: " + "mapping for U+%04lX longer than maximum of %d\n", + (long)c, Normalizer2Impl::MAPPING_LENGTH_MASK); + exit(U_INVALID_FORMAT_ERROR); + } + int32_t leadCC, trailCC; + if(length==0) { + leadCC=trailCC=0; + } else { + leadCC=getCC(m.char32At(0)); + trailCC=getCC(m.char32At(length-1)); + } + if(c<Normalizer2Impl::MIN_CCC_LCCC_CP && (p->cc!=0 || leadCC!=0)) { + fprintf(stderr, + "gennorm2 error: " + "U+%04lX below U+0300 has ccc!=0 or lccc!=0, not supported by ICU\n", + (long)c); + exit(U_INVALID_FORMAT_ERROR); + } + // Write small-FCD data. + if((leadCC|trailCC)!=0) { + UChar32 lead= c<=0xffff ? c : U16_LEAD(c); + smallFCD[lead>>8]|=(uint8_t)1<<((lead>>5)&7); + } + // Write the mapping & raw mapping extraData. + int32_t firstUnit=length|(trailCC<<8); + int32_t preMappingLength=0; + if(p->rawMapping!=NULL) { + UnicodeString &rm=*p->rawMapping; + int32_t rmLength=rm.length(); + if(rmLength>Normalizer2Impl::MAPPING_LENGTH_MASK) { + fprintf(stderr, + "gennorm2 error: " + "raw mapping for U+%04lX longer than maximum of %d\n", + (long)c, Normalizer2Impl::MAPPING_LENGTH_MASK); + exit(U_INVALID_FORMAT_ERROR); + } + UChar rm0=rm.charAt(0); + if( rmLength==length-1 && + // 99: overlong substring lengths get pinned to remainder lengths anyway + 0==rm.compare(1, 99, m, 2, 99) && + rm0>Normalizer2Impl::MAPPING_LENGTH_MASK + ) { + // Compression: + // rawMapping=rm0+mapping.substring(2) -> store only rm0 + // + // The raw mapping is the same as the final mapping after replacing + // the final mapping's first two code units with the raw mapping's first one. + // In this case, we store only that first unit, rm0. + // This helps with a few hundred mappings. + dataString.append(rm0); + preMappingLength=1; + } else { + // Store the raw mapping with its length. + dataString.append(rm); + dataString.append((UChar)rmLength); + preMappingLength=rmLength+1; + } + firstUnit|=Normalizer2Impl::MAPPING_HAS_RAW_MAPPING; + } + int32_t cccLccc=p->cc|(leadCC<<8); + if(cccLccc!=0) { + dataString.append((UChar)cccLccc); + ++preMappingLength; + firstUnit|=Normalizer2Impl::MAPPING_HAS_CCC_LCCC_WORD; + } + if(p->hasNoCompBoundaryAfter) { + firstUnit|=Normalizer2Impl::MAPPING_NO_COMP_BOUNDARY_AFTER; + } + dataString.append((UChar)firstUnit); + dataString.append(m); + return preMappingLength; +} + +// Requires p->compositions!=NULL. +void Normalizer2DataBuilder::writeCompositions(UChar32 c, const Norm *p, UnicodeString &dataString) { + if(p->cc!=0) { + fprintf(stderr, + "gennorm2 error: " + "U+%04lX combines-forward and has ccc!=0, not possible in Unicode normalization\n", + (long)c); + exit(U_INVALID_FORMAT_ERROR); + } + int32_t length; + const CompositionPair *pairs=p->getCompositionPairs(length); + for(int32_t i=0; i<length; ++i) { + const CompositionPair &pair=pairs[i]; + // 22 bits for the composite character and whether it combines forward. + UChar32 compositeAndFwd=pair.composite<<1; + if(getNormRef(pair.composite).compositions!=NULL) { + compositeAndFwd|=1; // The composite character also combines-forward. + } + // Encode most pairs in two units and some in three. + int32_t firstUnit, secondUnit, thirdUnit; + if(pair.trail<Normalizer2Impl::COMP_1_TRAIL_LIMIT) { + if(compositeAndFwd<=0xffff) { + firstUnit=pair.trail<<1; + secondUnit=compositeAndFwd; + thirdUnit=-1; + } else { + firstUnit=(pair.trail<<1)|Normalizer2Impl::COMP_1_TRIPLE; + secondUnit=compositeAndFwd>>16; + thirdUnit=compositeAndFwd; + } + } else { + firstUnit=(Normalizer2Impl::COMP_1_TRAIL_LIMIT+ + (pair.trail>>Normalizer2Impl::COMP_1_TRAIL_SHIFT))| + Normalizer2Impl::COMP_1_TRIPLE; + secondUnit=(pair.trail<<Normalizer2Impl::COMP_2_TRAIL_SHIFT)| + (compositeAndFwd>>16); + thirdUnit=compositeAndFwd; + } + // Set the high bit of the first unit if this is the last composition pair. + if(i==(length-1)) { + firstUnit|=Normalizer2Impl::COMP_1_LAST_TUPLE; + } + dataString.append((UChar)firstUnit).append((UChar)secondUnit); + if(thirdUnit>=0) { + dataString.append((UChar)thirdUnit); + } + } +} + +class ExtraDataWriter : public Normalizer2DBEnumerator { +public: + ExtraDataWriter(Normalizer2DataBuilder &b) : + Normalizer2DBEnumerator(b), + yesYesCompositions(1000, (UChar32)0xffff, 2), // 0=inert, 1=Jamo L, 2=start of compositions + yesNoMappingsAndCompositions(1000, (UChar32)0, 1) {} // 0=Hangul, 1=start of normal data + virtual UBool rangeHandler(UChar32 start, UChar32 end, uint32_t value) { + if(value!=0) { + if(start!=end) { + fprintf(stderr, + "gennorm2 error: unexpected shared data for " + "multiple code points U+%04lX..U+%04lX\n", + (long)start, (long)end); + exit(U_INTERNAL_PROGRAM_ERROR); + } + builder.writeExtraData(start, value, *this); + } + return TRUE; + } + UnicodeString maybeYesCompositions; + UnicodeString yesYesCompositions; + UnicodeString yesNoMappingsAndCompositions; + UnicodeString yesNoMappingsOnly; + UnicodeString noNoMappings; + Hashtable previousNoNoMappings; // If constructed in runtime code, pass in UErrorCode. +}; + +void Normalizer2DataBuilder::writeExtraData(UChar32 c, uint32_t value, ExtraDataWriter &writer) { + Norm *p=norms+value; + if(!p->hasMapping()) { + // Write small-FCD data. + // There is similar code in writeMapping() for characters that do have a mapping. + if(c<Normalizer2Impl::MIN_CCC_LCCC_CP && p->cc!=0) { + fprintf(stderr, + "gennorm2 error: " + "U+%04lX below U+0300 has ccc!=0, not supported by ICU\n", + (long)c); + exit(U_INVALID_FORMAT_ERROR); + } + if(p->cc!=0) { + UChar32 lead= c<=0xffff ? c : U16_LEAD(c); + smallFCD[lead>>8]|=(uint8_t)1<<((lead>>5)&7); + } + } + if(p->combinesBack) { + if(p->hasMapping()) { + fprintf(stderr, + "gennorm2 error: " + "U+%04lX combines-back and decomposes, not possible in Unicode normalization\n", + (long)c); + exit(U_INVALID_FORMAT_ERROR); + } + if(p->compositions!=NULL) { + p->offset= + (writer.maybeYesCompositions.length()<<Norm::OFFSET_SHIFT)| + Norm::OFFSET_MAYBE_YES; + writeCompositions(c, p, writer.maybeYesCompositions); + } + } else if(!p->hasMapping()) { + if(p->compositions!=NULL) { + p->offset= + (writer.yesYesCompositions.length()<<Norm::OFFSET_SHIFT)| + Norm::OFFSET_YES_YES; + writeCompositions(c, p, writer.yesYesCompositions); + } + } else if(p->mappingType==Norm::ROUND_TRIP) { + if(p->compositions!=NULL) { + int32_t offset=writer.yesNoMappingsAndCompositions.length()+ + writeMapping(c, p, writer.yesNoMappingsAndCompositions); + p->offset=(offset<<Norm::OFFSET_SHIFT)|Norm::OFFSET_YES_NO_MAPPING_AND_COMPOSITION; + writeCompositions(c, p, writer.yesNoMappingsAndCompositions); + } else { + int32_t offset=writer.yesNoMappingsOnly.length()+ + writeMapping(c, p, writer.yesNoMappingsOnly); + p->offset=(offset<<Norm::OFFSET_SHIFT)|Norm::OFFSET_YES_NO_MAPPING_ONLY; + } + } else /* one-way */ { + if(p->compositions!=NULL) { + fprintf(stderr, + "gennorm2 error: " + "U+%04lX combines-forward and has a one-way mapping, " + "not possible in Unicode normalization\n", + (long)c); + exit(U_INVALID_FORMAT_ERROR); + } + if(p->cc==0 && optimization!=OPTIMIZE_FAST) { + // Try a compact, algorithmic encoding. + // Only for ccc=0, because we can't store additional information + // and we do not recursively follow an algorithmic encoding for access to the ccc. + // + // Also, if hasNoCompBoundaryAfter is set, we can only use the algorithmic encoding + // if the mappingCP decomposes further, to ensure that there is a place to store it. + // We want to see that the final mapping does not have exactly 1 code point, + // or else we would have to recursively ensure that the final mapping is stored + // in normal extraData. + if(p->mappingCP>=0 && (!p->hasNoCompBoundaryAfter || 1!=p->mapping->countChar32())) { + int32_t delta=p->mappingCP-c; + if(-Normalizer2Impl::MAX_DELTA<=delta && delta<=Normalizer2Impl::MAX_DELTA) { + p->offset=(delta<<Norm::OFFSET_SHIFT)|Norm::OFFSET_DELTA; + } + } + } + if(p->offset==0) { + int32_t oldNoNoLength=writer.noNoMappings.length(); + int32_t offset=oldNoNoLength+writeMapping(c, p, writer.noNoMappings); + UnicodeString newMapping=writer.noNoMappings.tempSubString(oldNoNoLength); + int32_t previousOffset=writer.previousNoNoMappings.geti(newMapping); + if(previousOffset!=0) { + // Duplicate, remove the new units and point to the old ones. + writer.noNoMappings.truncate(oldNoNoLength); + p->offset=((previousOffset-1)<<Norm::OFFSET_SHIFT)|Norm::OFFSET_NO_NO; + } else { + // Enter this new mapping into the hashtable, avoiding value 0 which is "not found". + IcuToolErrorCode errorCode("gennorm2/writeExtraData()/Hashtable.puti()"); + writer.previousNoNoMappings.puti(newMapping, offset+1, errorCode); + p->offset=(offset<<Norm::OFFSET_SHIFT)|Norm::OFFSET_NO_NO; + } + } + } +} + +class Norm16Writer : public Normalizer2DBEnumerator { +public: + Norm16Writer(Normalizer2DataBuilder &b) : Normalizer2DBEnumerator(b) {} + virtual UBool rangeHandler(UChar32 start, UChar32 end, uint32_t value) { + builder.writeNorm16(start, end, value); + return TRUE; + } +}; + +void Normalizer2DataBuilder::writeNorm16(UChar32 start, UChar32 end, uint32_t value) { + if(value!=0) { + const Norm *p=norms+value; + int32_t offset=p->offset>>Norm::OFFSET_SHIFT; + int32_t norm16=0; + UBool isDecompNo=FALSE; + UBool isCompNoMaybe=FALSE; + switch(p->offset&Norm::OFFSET_MASK) { + case Norm::OFFSET_NONE: + // No mapping, no compositions list. + if(p->combinesBack) { + norm16=Normalizer2Impl::MIN_NORMAL_MAYBE_YES+p->cc; + isDecompNo=(UBool)(p->cc!=0); + isCompNoMaybe=TRUE; + } else if(p->cc!=0) { + norm16=Normalizer2Impl::MIN_YES_YES_WITH_CC-1+p->cc; + isDecompNo=isCompNoMaybe=TRUE; + } + break; + case Norm::OFFSET_MAYBE_YES: + norm16=indexes[Normalizer2Impl::IX_MIN_MAYBE_YES]+offset; + isCompNoMaybe=TRUE; + break; + case Norm::OFFSET_YES_YES: + norm16=offset; + break; + case Norm::OFFSET_YES_NO_MAPPING_AND_COMPOSITION: + norm16=indexes[Normalizer2Impl::IX_MIN_YES_NO]+offset; + isDecompNo=TRUE; + break; + case Norm::OFFSET_YES_NO_MAPPING_ONLY: + norm16=indexes[Normalizer2Impl::IX_MIN_YES_NO_MAPPINGS_ONLY]+offset; + isDecompNo=TRUE; + break; + case Norm::OFFSET_NO_NO: + norm16=indexes[Normalizer2Impl::IX_MIN_NO_NO]+offset; + isDecompNo=isCompNoMaybe=TRUE; + break; + case Norm::OFFSET_DELTA: + norm16=getCenterNoNoDelta()+offset; + isDecompNo=isCompNoMaybe=TRUE; + break; + default: // Should not occur. + exit(U_INTERNAL_PROGRAM_ERROR); + } + IcuToolErrorCode errorCode("gennorm2/writeNorm16()"); + utrie2_setRange32(norm16Trie, start, end, (uint32_t)norm16, TRUE, errorCode); + if(isDecompNo && start<indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]) { + indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]=start; + } + if(isCompNoMaybe && start<indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]) { + indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]=start; + } + } +} + +void Normalizer2DataBuilder::setHangulData() { + HangulIterator hi; + const HangulIterator::Range *range; + // Check that none of the Hangul/Jamo code points have data. + while((range=hi.nextRange())!=NULL) { + for(UChar32 c=range->start; c<range->limit; ++c) { + if(utrie2_get32(norm16Trie, c)!=0) { + fprintf(stderr, + "gennorm2 error: " + "illegal mapping/composition/ccc data for Hangul or Jamo U+%04lX\n", + (long)c); + exit(U_INVALID_FORMAT_ERROR); + } + } + } + // Set data for algorithmic runtime handling. + IcuToolErrorCode errorCode("gennorm2/setHangulData()"); + hi.reset(); + while((range=hi.nextRange())!=NULL) { + uint16_t norm16=range->norm16; + if(norm16==0) { + norm16=(uint16_t)indexes[Normalizer2Impl::IX_MIN_YES_NO]; // Hangul LV/LVT encoded as minYesNo + if(range->start<indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]) { + indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]=range->start; + } + } else { + if(range->start<indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]) { // Jamo V/T are maybeYes + indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]=range->start; + } + } + utrie2_setRange32(norm16Trie, range->start, range->limit-1, norm16, TRUE, errorCode); + errorCode.assertSuccess(); + } +} + +U_CDECL_BEGIN + +static UBool U_CALLCONV +enumRangeMaxValue(const void *context, UChar32 /*start*/, UChar32 /*end*/, uint32_t value) { + uint32_t *pMaxValue=(uint32_t *)context; + if(value>*pMaxValue) { + *pMaxValue=value; + } + return TRUE; +} + +U_CDECL_END + +void Normalizer2DataBuilder::processData() { + IcuToolErrorCode errorCode("gennorm2/processData()"); + norm16Trie=utrie2_open(0, 0, errorCode); + errorCode.assertSuccess(); + + utrie2_enum(normTrie, NULL, enumRangeHandler, CompositionBuilder(*this).ptr()); + + Decomposer decomposer(*this); + do { + decomposer.didDecompose=FALSE; + utrie2_enum(normTrie, NULL, enumRangeHandler, &decomposer); + } while(decomposer.didDecompose); + + BuilderReorderingBuffer buffer; + int32_t normsLength=utm_countItems(normMem); + for(int32_t i=1; i<normsLength; ++i) { + // Set the hasNoCompBoundaryAfter flag for use by the last code branch + // in Normalizer2Impl::hasCompBoundaryAfter(). + // For details see the comments on hasNoCompBoundaryAfter(buffer). + const Norm &norm=norms[i]; + if(norm.hasMapping()) { + if(norm.compositions!=NULL) { + norms[i].hasNoCompBoundaryAfter=TRUE; + } else { + buffer.reset(); + reorder(norms+i, buffer); + norms[i].hasNoCompBoundaryAfter=hasNoCompBoundaryAfter(buffer); + } + } + } + + indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]=0x110000; + indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]=0x110000; + + ExtraDataWriter extraDataWriter(*this); + utrie2_enum(normTrie, NULL, enumRangeHandler, &extraDataWriter); + + extraData=extraDataWriter.maybeYesCompositions; + extraData.append(extraDataWriter.yesYesCompositions). + append(extraDataWriter.yesNoMappingsAndCompositions). + append(extraDataWriter.yesNoMappingsOnly). + append(extraDataWriter.noNoMappings); + // Pad to even length for 4-byte alignment of following data. + if(extraData.length()&1) { + extraData.append((UChar)0); + } + + indexes[Normalizer2Impl::IX_MIN_YES_NO]= + extraDataWriter.yesYesCompositions.length(); + indexes[Normalizer2Impl::IX_MIN_YES_NO_MAPPINGS_ONLY]= + indexes[Normalizer2Impl::IX_MIN_YES_NO]+ + extraDataWriter.yesNoMappingsAndCompositions.length(); + indexes[Normalizer2Impl::IX_MIN_NO_NO]= + indexes[Normalizer2Impl::IX_MIN_YES_NO_MAPPINGS_ONLY]+ + extraDataWriter.yesNoMappingsOnly.length(); + indexes[Normalizer2Impl::IX_LIMIT_NO_NO]= + indexes[Normalizer2Impl::IX_MIN_NO_NO]+ + extraDataWriter.noNoMappings.length(); + indexes[Normalizer2Impl::IX_MIN_MAYBE_YES]= + Normalizer2Impl::MIN_NORMAL_MAYBE_YES- + extraDataWriter.maybeYesCompositions.length(); + + int32_t minNoNoDelta=getCenterNoNoDelta()-Normalizer2Impl::MAX_DELTA; + if(indexes[Normalizer2Impl::IX_LIMIT_NO_NO]>minNoNoDelta) { + fprintf(stderr, + "gennorm2 error: " + "data structure overflow, too much mapping composition data\n"); + exit(U_BUFFER_OVERFLOW_ERROR); + } + + utrie2_enum(normTrie, NULL, enumRangeHandler, Norm16Writer(*this).ptr()); + + setHangulData(); + + // Look for the "worst" norm16 value of any supplementary code point + // corresponding to a lead surrogate, and set it as that surrogate's value. + // Enables quick check inner loops to look at only code units. + // + // We could be more sophisticated: + // We could collect a bit set for whether there are values in the different + // norm16 ranges (yesNo, maybeYes, yesYesWithCC etc.) + // and select the best value that only breaks the composition and/or decomposition + // inner loops if necessary. + // However, that seems like overkill for an optimization for supplementary characters. + for(UChar lead=0xd800; lead<0xdc00; ++lead) { + uint32_t maxValue=utrie2_get32(norm16Trie, lead); + utrie2_enumForLeadSurrogate(norm16Trie, lead, NULL, enumRangeMaxValue, &maxValue); + if( maxValue>=(uint32_t)indexes[Normalizer2Impl::IX_LIMIT_NO_NO] && + maxValue>(uint32_t)indexes[Normalizer2Impl::IX_MIN_NO_NO] + ) { + // Set noNo ("worst" value) if it got into "less-bad" maybeYes or ccc!=0. + // Otherwise it might end up at something like JAMO_VT which stays in + // the inner decomposition quick check loop. + maxValue=(uint32_t)indexes[Normalizer2Impl::IX_LIMIT_NO_NO]-1; + } + utrie2_set32ForLeadSurrogateCodeUnit(norm16Trie, lead, maxValue, errorCode); + } + + // Adjust supplementary minimum code points to break quick check loops at their lead surrogates. + // For an empty data file, minCP=0x110000 turns into 0xdc00 (first trail surrogate) + // which is harmless. + // As a result, the minimum code points are always BMP code points. + int32_t minCP=indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]; + if(minCP>=0x10000) { + indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]=U16_LEAD(minCP); + } + minCP=indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]; + if(minCP>=0x10000) { + indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]=U16_LEAD(minCP); + } + + utrie2_freeze(norm16Trie, UTRIE2_16_VALUE_BITS, errorCode); + norm16TrieLength=utrie2_serialize(norm16Trie, NULL, 0, errorCode); + if(errorCode.get()!=U_BUFFER_OVERFLOW_ERROR) { + fprintf(stderr, "gennorm2 error: unable to freeze/serialize the normalization trie - %s\n", + errorCode.errorName()); + exit(errorCode.reset()); + } + errorCode.reset(); + + int32_t offset=(int32_t)sizeof(indexes); + indexes[Normalizer2Impl::IX_NORM_TRIE_OFFSET]=offset; + offset+=norm16TrieLength; + indexes[Normalizer2Impl::IX_EXTRA_DATA_OFFSET]=offset; + offset+=extraData.length()*2; + indexes[Normalizer2Impl::IX_SMALL_FCD_OFFSET]=offset; + offset+=sizeof(smallFCD); + int32_t totalSize=offset; + for(int32_t i=Normalizer2Impl::IX_RESERVED3_OFFSET; i<=Normalizer2Impl::IX_TOTAL_SIZE; ++i) { + indexes[i]=totalSize; + } + + if(beVerbose) { + printf("size of normalization trie: %5ld bytes\n", (long)norm16TrieLength); + printf("size of 16-bit extra data: %5ld uint16_t\n", (long)extraData.length()); + printf("size of small-FCD data: %5ld bytes\n", (long)sizeof(smallFCD)); + printf("size of binary data file contents: %5ld bytes\n", (long)totalSize); + printf("minDecompNoCodePoint: U+%04lX\n", (long)indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]); + printf("minCompNoMaybeCodePoint: U+%04lX\n", (long)indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]); + printf("minYesNo: 0x%04x\n", (int)indexes[Normalizer2Impl::IX_MIN_YES_NO]); + printf("minYesNoMappingsOnly: 0x%04x\n", (int)indexes[Normalizer2Impl::IX_MIN_YES_NO_MAPPINGS_ONLY]); + printf("minNoNo: 0x%04x\n", (int)indexes[Normalizer2Impl::IX_MIN_NO_NO]); + printf("limitNoNo: 0x%04x\n", (int)indexes[Normalizer2Impl::IX_LIMIT_NO_NO]); + printf("minMaybeYes: 0x%04x\n", (int)indexes[Normalizer2Impl::IX_MIN_MAYBE_YES]); + } + + UVersionInfo nullVersion={ 0, 0, 0, 0 }; + if(0==memcmp(nullVersion, unicodeVersion, 4)) { + u_versionFromString(unicodeVersion, U_UNICODE_VERSION); + } + memcpy(dataInfo.dataVersion, unicodeVersion, 4); +} + +void Normalizer2DataBuilder::writeBinaryFile(const char *filename) { + processData(); + + IcuToolErrorCode errorCode("gennorm2/writeBinaryFile()"); + LocalArray<uint8_t> norm16TrieBytes(new uint8_t[norm16TrieLength]); + utrie2_serialize(norm16Trie, norm16TrieBytes.getAlias(), norm16TrieLength, errorCode); + errorCode.assertSuccess(); + + UNewDataMemory *pData= + udata_create(NULL, NULL, filename, &dataInfo, + haveCopyright ? U_COPYRIGHT_STRING : NULL, errorCode); + if(errorCode.isFailure()) { + fprintf(stderr, "gennorm2 error: unable to create the output file %s - %s\n", + filename, errorCode.errorName()); + exit(errorCode.reset()); + } + udata_writeBlock(pData, indexes, sizeof(indexes)); + udata_writeBlock(pData, norm16TrieBytes.getAlias(), norm16TrieLength); + udata_writeUString(pData, extraData.getBuffer(), extraData.length()); + udata_writeBlock(pData, smallFCD, sizeof(smallFCD)); + int32_t writtenSize=udata_finish(pData, errorCode); + if(errorCode.isFailure()) { + fprintf(stderr, "gennorm2: error %s writing the output file\n", errorCode.errorName()); + exit(errorCode.reset()); + } + int32_t totalSize=indexes[Normalizer2Impl::IX_TOTAL_SIZE]; + if(writtenSize!=totalSize) { + fprintf(stderr, "gennorm2 error: written size %ld != calculated size %ld\n", + (long)writtenSize, (long)totalSize); + exit(U_INTERNAL_PROGRAM_ERROR); + } +} + +void +Normalizer2DataBuilder::writeCSourceFile(const char *filename) { + processData(); + + IcuToolErrorCode errorCode("gennorm2/writeCSourceFile()"); + const char *basename=findBasename(filename); + CharString path(filename, (int32_t)(basename-filename), errorCode); + CharString dataName(basename, errorCode); + const char *extension=strrchr(basename, '.'); + if(extension!=NULL) { + dataName.truncate((int32_t)(extension-basename)); + } + errorCode.assertSuccess(); + + LocalArray<uint8_t> norm16TrieBytes(new uint8_t[norm16TrieLength]); + utrie2_serialize(norm16Trie, norm16TrieBytes.getAlias(), norm16TrieLength, errorCode); + errorCode.assertSuccess(); + + FILE *f=usrc_create(path.data(), basename, "icu/source/tools/gennorm2/n2builder.cpp"); + if(f==NULL) { + fprintf(stderr, "gennorm2/writeCSourceFile() error: unable to create the output file %s\n", + filename); + exit(U_FILE_ACCESS_ERROR); + return; + } + fputs("#ifdef INCLUDED_FROM_NORMALIZER2_CPP\n\n", f); + char line[100]; + sprintf(line, "static const UVersionInfo %s_formatVersion={", dataName.data()); + usrc_writeArray(f, line, dataInfo.formatVersion, 8, 4, "};\n"); + sprintf(line, "static const UVersionInfo %s_dataVersion={", dataName.data()); + usrc_writeArray(f, line, dataInfo.dataVersion, 8, 4, "};\n\n"); + sprintf(line, "static const int32_t %s_indexes[Normalizer2Impl::IX_COUNT]={\n", + dataName.data()); + usrc_writeArray(f, + line, + indexes, 32, Normalizer2Impl::IX_COUNT, + "\n};\n\n"); + sprintf(line, "static const uint16_t %s_trieIndex[%%ld]={\n", dataName.data()); + usrc_writeUTrie2Arrays(f, + line, NULL, + norm16Trie, + "\n};\n\n"); + sprintf(line, "static const uint16_t %s_extraData[%%ld]={\n", dataName.data()); + usrc_writeArray(f, + line, + extraData.getBuffer(), 16, extraData.length(), + "\n};\n\n"); + sprintf(line, "static const uint8_t %s_smallFCD[%%ld]={\n", dataName.data()); + usrc_writeArray(f, + line, + smallFCD, 8, sizeof(smallFCD), + "\n};\n\n"); + /*fputs( // TODO + "static const UCaseProps %s_singleton={\n" + " NULL,\n" + " %s_indexes,\n" + " %s_extraData,\n" + " %s_smallFCD,\n", + f);*/ + sprintf(line, "static const UTrie2 %s_trie={\n", dataName.data()); + char line2[100]; + sprintf(line2, "%s_trieIndex", dataName.data()); + usrc_writeUTrie2Struct(f, + line, + norm16Trie, line2, NULL, + "};\n"); + fputs("\n#endif // INCLUDED_FROM_NORMALIZER2_CPP\n", f); + fclose(f); +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_NORMALIZATION */ + +/* + * Hey, Emacs, please set the following: + * + * Local Variables: + * indent-tabs-mode: nil + * End: + */ |