summaryrefslogtreecommitdiffstats
path: root/intl/unicharutil/tools
diff options
context:
space:
mode:
Diffstat (limited to 'intl/unicharutil/tools')
-rw-r--r--intl/unicharutil/tools/MUTTUCData.txt208
-rw-r--r--intl/unicharutil/tools/README.txt3
-rw-r--r--intl/unicharutil/tools/UCDATAREADME.txt207
-rw-r--r--intl/unicharutil/tools/data/case.datbin0 -> 16592 bytes
-rw-r--r--intl/unicharutil/tools/data/cmbcl.datbin0 -> 1388 bytes
-rw-r--r--intl/unicharutil/tools/data/ctype.datbin0 -> 19196 bytes
-rw-r--r--intl/unicharutil/tools/data/decomp.datbin0 -> 16272 bytes
-rw-r--r--intl/unicharutil/tools/data/num.datbin0 -> 2976 bytes
-rw-r--r--intl/unicharutil/tools/format.txt243
-rwxr-xr-xintl/unicharutil/tools/genSpecialCasingData.pl287
-rwxr-xr-xintl/unicharutil/tools/genUnicodePropertyData.pl943
-rw-r--r--intl/unicharutil/tools/moz.build12
-rw-r--r--intl/unicharutil/tools/ucgendat.c1457
13 files changed, 3360 insertions, 0 deletions
diff --git a/intl/unicharutil/tools/MUTTUCData.txt b/intl/unicharutil/tools/MUTTUCData.txt
new file mode 100644
index 000000000..7d44e290a
--- /dev/null
+++ b/intl/unicharutil/tools/MUTTUCData.txt
@@ -0,0 +1,208 @@
+#
+# $Id: MUTTUCData.txt,v 1.1 1999/01/08 00:19:19 ftang%netscape.com Exp $
+#
+# Copyright 1996, 1997, 1998 Computing Research Labs,
+# New Mexico State University
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE COMPUTING RESEARCH LAB OR NEW MEXICO STATE UNIVERSITY BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT
+# OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
+# THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+#
+# Implementation specific character properties.
+#
+#
+# Space, other.
+#
+0009;;Ss;;;;;;;;;;;;
+000A;;Ss;;;;;;;;;;;;
+000B;;Ss;;;;;;;;;;;;
+000C;;Ss;;;;;;;;;;;;
+000D;;Ss;;;;;;;;;;;;
+#
+# Non-breaking.
+#
+00A0;;Nb;;;;;;;;;;;;
+2007;;Nb;;;;;;;;;;;;
+2011;;Nb;;;;;;;;;;;;
+FEFF;;Nb;;;;;;;;;;;;
+#
+# Symmetric.
+#
+0028;;Sy;;;;;;;;;;;;
+0029;;Sy;;;;;;;;;;;;
+005B;;Sy;;;;;;;;;;;;
+005D;;Sy;;;;;;;;;;;;
+007B;;Sy;;;;;;;;;;;;
+007D;;Sy;;;;;;;;;;;;
+00AB;;Sy;;;;;;;;;;;;
+00BB;;Sy;;;;;;;;;;;;
+0F3A;;Sy;;;;;;;;;;;;
+0F3B;;Sy;;;;;;;;;;;;
+0F3C;;Sy;;;;;;;;;;;;
+0F3D;;Sy;;;;;;;;;;;;
+0F3E;;Sy;;;;;;;;;;;;
+0F3F;;Sy;;;;;;;;;;;;
+2018;;Sy;;;;;;;;;;;;
+2019;;Sy;;;;;;;;;;;;
+201A;;Sy;;;;;;;;;;;;
+201B;;Sy;;;;;;;;;;;;
+201C;;Sy;;;;;;;;;;;;
+201D;;Sy;;;;;;;;;;;;
+201E;;Sy;;;;;;;;;;;;
+201F;;Sy;;;;;;;;;;;;
+2039;;Sy;;;;;;;;;;;;
+203A;;Sy;;;;;;;;;;;;
+2045;;Sy;;;;;;;;;;;;
+2046;;Sy;;;;;;;;;;;;
+207D;;Sy;;;;;;;;;;;;
+207E;;Sy;;;;;;;;;;;;
+208D;;Sy;;;;;;;;;;;;
+208E;;Sy;;;;;;;;;;;;
+2329;;Sy;;;;;;;;;;;;
+232A;;Sy;;;;;;;;;;;;
+3008;;Sy;;;;;;;;;;;;
+3009;;Sy;;;;;;;;;;;;
+300A;;Sy;;;;;;;;;;;;
+300B;;Sy;;;;;;;;;;;;
+300C;;Sy;;;;;;;;;;;;
+300D;;Sy;;;;;;;;;;;;
+300E;;Sy;;;;;;;;;;;;
+300F;;Sy;;;;;;;;;;;;
+3010;;Sy;;;;;;;;;;;;
+3011;;Sy;;;;;;;;;;;;
+3014;;Sy;;;;;;;;;;;;
+3015;;Sy;;;;;;;;;;;;
+3016;;Sy;;;;;;;;;;;;
+3017;;Sy;;;;;;;;;;;;
+3018;;Sy;;;;;;;;;;;;
+3019;;Sy;;;;;;;;;;;;
+301A;;Sy;;;;;;;;;;;;
+301B;;Sy;;;;;;;;;;;;
+301D;;Sy;;;;;;;;;;;;
+301E;;Sy;;;;;;;;;;;;
+FD3E;;Sy;;;;;;;;;;;;
+FD3F;;Sy;;;;;;;;;;;;
+FE35;;Sy;;;;;;;;;;;;
+FE36;;Sy;;;;;;;;;;;;
+FE37;;Sy;;;;;;;;;;;;
+FE38;;Sy;;;;;;;;;;;;
+FE39;;Sy;;;;;;;;;;;;
+FE3A;;Sy;;;;;;;;;;;;
+FE3B;;Sy;;;;;;;;;;;;
+FE3C;;Sy;;;;;;;;;;;;
+FE3D;;Sy;;;;;;;;;;;;
+FE3E;;Sy;;;;;;;;;;;;
+FE3F;;Sy;;;;;;;;;;;;
+FE40;;Sy;;;;;;;;;;;;
+FE41;;Sy;;;;;;;;;;;;
+FE42;;Sy;;;;;;;;;;;;
+FE43;;Sy;;;;;;;;;;;;
+FE44;;Sy;;;;;;;;;;;;
+FE59;;Sy;;;;;;;;;;;;
+FE5A;;Sy;;;;;;;;;;;;
+FE5B;;Sy;;;;;;;;;;;;
+FE5C;;Sy;;;;;;;;;;;;
+FE5D;;Sy;;;;;;;;;;;;
+FE5E;;Sy;;;;;;;;;;;;
+FF08;;Sy;;;;;;;;;;;;
+FF09;;Sy;;;;;;;;;;;;
+FF3B;;Sy;;;;;;;;;;;;
+FF3D;;Sy;;;;;;;;;;;;
+FF5B;;Sy;;;;;;;;;;;;
+FF5D;;Sy;;;;;;;;;;;;
+FF62;;Sy;;;;;;;;;;;;
+FF63;;Sy;;;;;;;;;;;;
+#
+# Hex digit.
+#
+0030;;Hd;;;;;;;;;;;;
+0031;;Hd;;;;;;;;;;;;
+0032;;Hd;;;;;;;;;;;;
+0033;;Hd;;;;;;;;;;;;
+0034;;Hd;;;;;;;;;;;;
+0035;;Hd;;;;;;;;;;;;
+0036;;Hd;;;;;;;;;;;;
+0037;;Hd;;;;;;;;;;;;
+0038;;Hd;;;;;;;;;;;;
+0039;;Hd;;;;;;;;;;;;
+0041;;Hd;;;;;;;;;;;;
+0042;;Hd;;;;;;;;;;;;
+0043;;Hd;;;;;;;;;;;;
+0044;;Hd;;;;;;;;;;;;
+0045;;Hd;;;;;;;;;;;;
+0046;;Hd;;;;;;;;;;;;
+0061;;Hd;;;;;;;;;;;;
+0062;;Hd;;;;;;;;;;;;
+0063;;Hd;;;;;;;;;;;;
+0064;;Hd;;;;;;;;;;;;
+0065;;Hd;;;;;;;;;;;;
+0066;;Hd;;;;;;;;;;;;
+FF10;;Hd;;;;;;;;;;;;
+FF11;;Hd;;;;;;;;;;;;
+FF12;;Hd;;;;;;;;;;;;
+FF13;;Hd;;;;;;;;;;;;
+FF14;;Hd;;;;;;;;;;;;
+FF15;;Hd;;;;;;;;;;;;
+FF16;;Hd;;;;;;;;;;;;
+FF17;;Hd;;;;;;;;;;;;
+FF18;;Hd;;;;;;;;;;;;
+FF19;;Hd;;;;;;;;;;;;
+FF21;;Hd;;;;;;;;;;;;
+FF22;;Hd;;;;;;;;;;;;
+FF23;;Hd;;;;;;;;;;;;
+FF24;;Hd;;;;;;;;;;;;
+FF25;;Hd;;;;;;;;;;;;
+FF26;;Hd;;;;;;;;;;;;
+FF41;;Hd;;;;;;;;;;;;
+FF42;;Hd;;;;;;;;;;;;
+FF43;;Hd;;;;;;;;;;;;
+FF44;;Hd;;;;;;;;;;;;
+FF45;;Hd;;;;;;;;;;;;
+FF46;;Hd;;;;;;;;;;;;
+#
+# Quote marks.
+#
+0022;;Qm;;;;;;;;;;;;
+0027;;Qm;;;;;;;;;;;;
+00AB;;Qm;;;;;;;;;;;;
+00BB;;Qm;;;;;;;;;;;;
+2018;;Qm;;;;;;;;;;;;
+2019;;Qm;;;;;;;;;;;;
+201A;;Qm;;;;;;;;;;;;
+201B;;Qm;;;;;;;;;;;;
+201C;;Qm;;;;;;;;;;;;
+201D;;Qm;;;;;;;;;;;;
+201E;;Qm;;;;;;;;;;;;
+201F;;Qm;;;;;;;;;;;;
+2039;;Qm;;;;;;;;;;;;
+203A;;Qm;;;;;;;;;;;;
+300C;;Qm;;;;;;;;;;;;
+300D;;Qm;;;;;;;;;;;;
+300E;;Qm;;;;;;;;;;;;
+300F;;Qm;;;;;;;;;;;;
+301D;;Qm;;;;;;;;;;;;
+301E;;Qm;;;;;;;;;;;;
+301F;;Qm;;;;;;;;;;;;
+FE41;;Qm;;;;;;;;;;;;
+FE42;;Qm;;;;;;;;;;;;
+FE43;;Qm;;;;;;;;;;;;
+FE44;;Qm;;;;;;;;;;;;
+FF02;;Qm;;;;;;;;;;;;
+FF07;;Qm;;;;;;;;;;;;
+FF62;;Qm;;;;;;;;;;;;
+FF63;;Qm;;;;;;;;;;;;
diff --git a/intl/unicharutil/tools/README.txt b/intl/unicharutil/tools/README.txt
new file mode 100644
index 000000000..7295547f1
--- /dev/null
+++ b/intl/unicharutil/tools/README.txt
@@ -0,0 +1,3 @@
+Instructions for using these tools are on the Mozilla Wiki.
+
+https://wiki.mozilla.org/I18n:Updating_Unicode_version
diff --git a/intl/unicharutil/tools/UCDATAREADME.txt b/intl/unicharutil/tools/UCDATAREADME.txt
new file mode 100644
index 000000000..012098fc0
--- /dev/null
+++ b/intl/unicharutil/tools/UCDATAREADME.txt
@@ -0,0 +1,207 @@
+#
+# $Id: UCDATAREADME.txt,v 1.1 1999/01/08 00:19:20 ftang%netscape.com Exp $
+#
+
+ MUTT UCData Package 1.9
+ -----------------------
+
+This is a package that supports ctype-like operations for Unicode UCS-2 text
+(and surrogates), case mapping, and decomposition lookup. To use it, you will
+need to get the "UnicodeData-2.0.14.txt" (or later) file from the Unicode Web
+or FTP site.
+
+This package consists of two parts:
+
+ 1. A program called "ucgendat" which generates five data files from the
+ UnicodeData-2.*.txt file. The files are:
+
+ A. case.dat - the case mappings.
+ B. ctype.dat - the character property tables.
+ C. decomp.dat - the character decompositions.
+ D. cmbcl.dat - the non-zero combining classes.
+ E. num.dat - the codes representing numbers.
+
+ 2. The "ucdata.[ch]" files which implement the functions needed to
+ check to see if a character matches groups of properties, to map between
+ upper, lower, and title case, to look up the decomposition of a
+ character, look up the combining class of a character, and get the number
+ value of a character.
+
+A short reference to the functions available is in the "api.txt" file.
+
+Techie Details
+==============
+
+The "ucgendat" program parses files from the command line which are all in the
+Unicode Character Database (UCDB) format. An additional properties file,
+"MUTTUCData.txt", provides some extra properties for some characters.
+
+The program looks for the two character properties fields (2 and 4), the
+combining class field (3), the decomposition field (5), the numeric value
+field (8), and the case mapping fields (12, 13, and 14). The decompositions
+are recursively expanded before being written out.
+
+The decomposition table contains all the canonical decompositions. This means
+all decompositions that do not have tags such as "<compat>" or "<font>".
+
+The data is almost all stored as unsigned longs (32-bits assumed) and the
+routines that load the data take care of endian swaps when necessary. This
+also means that surrogates (>= 0x10000) can be placed in the data files the
+"ucgendat" program parses.
+
+The data is written as external files and broken into five parts so it can be
+selectively updated at runtime if necessary.
+
+The data files currently generated from the "ucgendat" program total about 56K
+in size all together.
+
+The format of the binary data files is documented in the "format.txt" file.
+
+Mark Leisher <mleisher@crl.nmsu.edu>
+13 December 1998
+
+CHANGES
+=======
+
+Version 1.9
+-----------
+1. Fixed a problem with an incorrect amount of storage being allocated for the
+ combining class nodes.
+
+2. Fixed an invalid initialization in the number code.
+
+3. Changed the Java template file formatting a bit.
+
+4. Added tables and function for getting decompositions in the Java class.
+
+Version 1.8
+-----------
+1. Fixed a problem with adding certain ranges.
+
+2. Added two more macros for testing for identifiers.
+
+3. Tested with the UnicodeData-2.1.5.txt file.
+
+Version 1.7
+-----------
+1. Fixed a problem with looking up decompositions in "ucgendat."
+
+Version 1.6
+-----------
+1. Added two new properties introduced with UnicodeData-2.1.4.txt.
+
+2. Changed the "ucgendat.c" program a little to automatically align the
+ property data on a 4-byte boundary when new properties are added.
+
+3. Changed the "ucgendat.c" programs to only generate canonical
+ decompositions.
+
+4. Added two new macros ucisinitialpunct() and ucisfinalpunct() to check for
+ initial and final punctuation characters.
+
+5. Minor additions and changes to the documentation.
+
+Version 1.5
+-----------
+1. Changed all file open calls to include binary mode with "b" for DOS/WIN
+ platforms.
+
+2. Wrapped the unistd.h include so it won't be included when compiled under
+ Win32.
+
+3. Fixed a bad range check for hex digits in ucgendat.c.
+
+4. Fixed a bad endian swap for combining classes.
+
+5. Added code to make a number table and associated lookup functions.
+ Functions added are ucnumber(), ucdigit(), and ucgetnumber(). The last
+ function is to maintain compatibility with John Cowan's "uctype" package.
+
+Version 1.4
+-----------
+1. Fixed a bug with adding a range.
+
+2. Fixed a bug with inserting a range in order.
+
+3. Fixed incorrectly specified ucisdefined() and ucisundefined() macros.
+
+4. Added the missing unload for the combining class data.
+
+5. Fixed a bad macro placement in ucisweak().
+
+Version 1.3
+-----------
+1. Bug with case mapping calculations fixed.
+
+2. Bug with empty character property entries fixed.
+
+3. Bug with incorrect type in the combining class lookup fixed.
+
+4. Some corrections done to api.txt.
+
+5. Bug in certain character property lookups fixed.
+
+6. Added a character property table that records the defined characters.
+
+7. Replaced ucisunknown() with ucisdefined() and ucisundefined().
+
+Version 1.2
+-----------
+1. Added code to ucgendat to generate a combining class table.
+
+2. Fixed an endian problem with the byte count of decompositions.
+
+3. Fixed some minor problems in the "format.txt" file.
+
+4. Removed some bogus "Ss" values from MUTTUCData.txt file.
+
+5. Added API function to get combining class.
+
+6. Changed the open mode to "rb" so binary data files will be opened correctly
+ on DOS/WIN as well as other platforms.
+
+7. Added the "api.txt" file.
+
+Version 1.1
+-----------
+1. Added ucisxdigit() which I overlooked.
+
+2. Added UC_LT to the ucisalpha() macro which I overlooked.
+
+3. Change uciscntrl() to include UC_CF.
+
+4. Added ucisocntrl() and ucfntcntrl() macros.
+
+5. Added a ucisblank() which I overlooked.
+
+6. Added missing properties to ucissymbol() and ucisnumber().
+
+7. Added ucisgraph() and ucisprint().
+
+8. Changed the "Mr" property to "Sy" to mark this subset of mirroring
+ characters as symmetric to avoid trampling the Unicode/ISO10646 sense of
+ mirroring.
+
+9. Added another property called "Ss" which includes control characters
+ traditionally seen as spaces in the isspace() macro.
+
+10. Added a bunch of macros to be API compatible with John Cowan's package.
+
+ACKNOWLEDGEMENTS
+================
+
+Thanks go to John Cowan <cowan@locke.ccil.org> for pointing out lots of
+missing things and giving me stuff, particularly a bunch of new macros.
+
+Thanks go to Bob Verbrugge <bob_verbrugge@nl.compuware.com> for pointing out
+various bugs.
+
+Thanks go to Christophe Pierret <cpierret@businessobjects.com> for pointing
+out that file modes need to have "b" for DOS/WIN machines, pointing out
+unistd.h is not a Win 32 header, and pointing out a problem with ucisalnum().
+
+Thanks go to Kent Johnson <kent@pondview.mv.com> for finding a bug that caused
+incomplete decompositions to be generated by the "ucgendat" program.
+
+Thanks go to Valeriy E. Ushakov <uwe@ptc.spbu.ru> for spotting an allocation
+error and an initialization error.
diff --git a/intl/unicharutil/tools/data/case.dat b/intl/unicharutil/tools/data/case.dat
new file mode 100644
index 000000000..3bf7f2c93
--- /dev/null
+++ b/intl/unicharutil/tools/data/case.dat
Binary files differ
diff --git a/intl/unicharutil/tools/data/cmbcl.dat b/intl/unicharutil/tools/data/cmbcl.dat
new file mode 100644
index 000000000..9a7513d8f
--- /dev/null
+++ b/intl/unicharutil/tools/data/cmbcl.dat
Binary files differ
diff --git a/intl/unicharutil/tools/data/ctype.dat b/intl/unicharutil/tools/data/ctype.dat
new file mode 100644
index 000000000..95e2c405b
--- /dev/null
+++ b/intl/unicharutil/tools/data/ctype.dat
Binary files differ
diff --git a/intl/unicharutil/tools/data/decomp.dat b/intl/unicharutil/tools/data/decomp.dat
new file mode 100644
index 000000000..29e3948e3
--- /dev/null
+++ b/intl/unicharutil/tools/data/decomp.dat
Binary files differ
diff --git a/intl/unicharutil/tools/data/num.dat b/intl/unicharutil/tools/data/num.dat
new file mode 100644
index 000000000..52498ece0
--- /dev/null
+++ b/intl/unicharutil/tools/data/num.dat
Binary files differ
diff --git a/intl/unicharutil/tools/format.txt b/intl/unicharutil/tools/format.txt
new file mode 100644
index 000000000..ef4210be1
--- /dev/null
+++ b/intl/unicharutil/tools/format.txt
@@ -0,0 +1,243 @@
+#
+# $Id: format.txt,v 1.1 1999/01/08 00:19:20 ftang%netscape.com Exp $
+#
+
+CHARACTER DATA
+==============
+
+This package generates some data files that contain character properties useful
+for text processing.
+
+CHARACTER PROPERTIES
+====================
+
+The first data file is called "ctype.dat" and contains a compressed form of
+the character properties found in the Unicode Character Database (UCDB).
+Additional properties can be specified in limited UCDB format in another file
+to avoid modifying the original UCDB.
+
+The following is a property name and code table to be used with the character
+data:
+
+NAME CODE DESCRIPTION
+---------------------
+Mn 0 Mark, Non-Spacing
+Mc 1 Mark, Spacing Combining
+Me 2 Mark, Enclosing
+Nd 3 Number, Decimal Digit
+Nl 4 Number, Letter
+No 5 Number, Other
+Zs 6 Separator, Space
+Zl 7 Separator, Line
+Zp 8 Separator, Paragraph
+Cc 9 Other, Control
+Cf 10 Other, Format
+Cs 11 Other, Surrogate
+Co 12 Other, Private Use
+Cn 13 Other, Not Assigned
+Lu 14 Letter, Uppercase
+Ll 15 Letter, Lowercase
+Lt 16 Letter, Titlecase
+Lm 17 Letter, Modifier
+Lo 18 Letter, Other
+Pc 19 Punctuation, Connector
+Pd 20 Punctuation, Dash
+Ps 21 Punctuation, Open
+Pe 22 Punctuation, Close
+Po 23 Punctuation, Other
+Sm 24 Symbol, Math
+Sc 25 Symbol, Currency
+Sk 26 Symbol, Modifier
+So 27 Symbol, Other
+L 28 Left-To-Right
+R 29 Right-To-Left
+EN 30 European Number
+ES 31 European Number Separator
+ET 32 European Number Terminator
+AN 33 Arabic Number
+CS 34 Common Number Separator
+B 35 Block Separator
+S 36 Segment Separator
+WS 37 Whitespace
+ON 38 Other Neutrals
+Pi 47 Punctuation, Initial
+Pf 48 Punctuation, Final
+#
+# Implementation specific properties.
+#
+Cm 39 Composite
+Nb 40 Non-Breaking
+Sy 41 Symmetric (characters which are part of open/close pairs)
+Hd 42 Hex Digit
+Qm 43 Quote Mark
+Mr 44 Mirroring
+Ss 45 Space, Other (controls viewed as spaces in ctype isspace())
+Cp 46 Defined character
+
+The actual binary data is formatted as follows:
+
+ Assumptions: unsigned short is at least 16-bits in size and unsigned long
+ is at least 32-bits in size.
+
+ unsigned short ByteOrderMark
+ unsigned short OffsetArraySize
+ unsigned long Bytes
+ unsigned short Offsets[OffsetArraySize + 1]
+ unsigned long Ranges[N], N = value of Offsets[OffsetArraySize]
+
+ The Bytes field provides the total byte count used for the Offsets[] and
+ Ranges[] arrays. The Offsets[] array is aligned on a 4-byte boundary and
+ there is always one extra node on the end to hold the final index of the
+ Ranges[] array. The Ranges[] array contains pairs of 4-byte values
+ representing a range of Unicode characters. The pairs are arranged in
+ increasing order by the first character code in the range.
+
+ Determining if a particular character is in the property list requires a
+ simple binary search to determine if a character is in any of the ranges
+ for the property.
+
+ If the ByteOrderMark is equal to 0xFFFE, then the data was generated on a
+ machine with a different endian order and the values must be byte-swapped.
+
+ To swap a 16-bit value:
+ c = (c >> 8) | ((c & 0xff) << 8)
+
+ To swap a 32-bit value:
+ c = ((c & 0xff) << 24) | (((c >> 8) & 0xff) << 16) |
+ (((c >> 16) & 0xff) << 8) | (c >> 24)
+
+CASE MAPPINGS
+=============
+
+The next data file is called "case.dat" and contains three case mapping tables
+in the following order: upper, lower, and title case. Each table is in
+increasing order by character code and each mapping contains 3 unsigned longs
+which represent the possible mappings.
+
+The format for the binary form of these tables is:
+
+ unsigned short ByteOrderMark
+ unsigned short NumMappingNodes, count of all mapping nodes
+ unsigned short CaseTableSizes[2], upper and lower mapping node counts
+ unsigned long CaseTables[NumMappingNodes]
+
+ The starting indexes of the case tables are calculated as following:
+
+ UpperIndex = 0;
+ LowerIndex = CaseTableSizes[0] * 3;
+ TitleIndex = LowerIndex + CaseTableSizes[1] * 3;
+
+ The order of the fields for the three tables are:
+
+ Upper case
+ ----------
+ unsigned long upper;
+ unsigned long lower;
+ unsigned long title;
+
+ Lower case
+ ----------
+ unsigned long lower;
+ unsigned long upper;
+ unsigned long title;
+
+ Title case
+ ----------
+ unsigned long title;
+ unsigned long upper;
+ unsigned long lower;
+
+ If the ByteOrderMark is equal to 0xFFFE, endian swapping is required in the
+ same way as described in the CHARACTER PROPERTIES section.
+
+ Because the tables are in increasing order by character code, locating a
+ mapping requires a simple binary search on one of the 3 codes that make up
+ each node.
+
+ It is important to note that there can only be 65536 mapping nodes which
+ divided into 3 portions allows 21845 nodes for each case mapping table. The
+ distribution of mappings may be more or less than 21845 per table, but only
+ 65536 are allowed.
+
+DECOMPOSITIONS
+==============
+
+The next data file is called "decomp.dat" and contains the decomposition data
+for all characters with decompositions containing more than one character and
+are *not* compatibility decompositions. Compatibility decompositions are
+signaled in the UCDB format by the use of the <compat> tag in the
+decomposition field. Each list of character codes represents a full
+decomposition of a composite character. The nodes are arranged in increasing
+order by character code.
+
+The format for the binary form of this table is:
+
+ unsigned short ByteOrderMark
+ unsigned short NumDecompNodes, count of all decomposition nodes
+ unsigned long Bytes
+ unsigned long DecompNodes[(NumDecompNodes * 2) + 1]
+ unsigned long Decomp[N], N = sum of all counts in DecompNodes[]
+
+ If the ByteOrderMark is equal to 0xFFFE, endian swapping is required in the
+ same way as described in the CHARACTER PROPERTIES section.
+
+ The DecompNodes[] array consists of pairs of unsigned longs, the first of
+ which is the character code and the second is the initial index of the list
+ of character codes representing the decomposition.
+
+ Locating the decomposition of a composite character requires a binary search
+ for a character code in the DecompNodes[] array and using its index to
+ locate the start of the decomposition. The length of the decomposition list
+ is the index in the following element in DecompNode[] minus the current
+ index.
+
+COMBINING CLASSES
+=================
+
+The fourth data file is called "cmbcl.dat" and contains the characters with
+non-zero combining classes.
+
+The format for the binary form of this table is:
+
+ unsigned short ByteOrderMark
+ unsigned short NumCCLNodes
+ unsigned long Bytes
+ unsigned long CCLNodes[NumCCLNodes * 3]
+
+ If the ByteOrderMark is equal to 0xFFFE, endian swapping is required in the
+ same way as described in the CHARACTER PROPERTIES section.
+
+ The CCLNodes[] array consists of groups of three unsigned longs. The first
+ and second are the beginning and ending of a range and the third is the
+ combining class of that range.
+
+ If a character is not found in this table, then the combining class is
+ assumed to be 0.
+
+ It is important to note that only 65536 distinct ranges plus combining class
+ can be specified because the NumCCLNodes is usually a 16-bit number.
+
+NUMBER TABLE
+============
+
+The final data file is called "num.dat" and contains the characters that have
+a numeric value associated with them.
+
+The format for the binary form of the table is:
+
+ unsigned short ByteOrderMark
+ unsigned short NumNumberNodes
+ unsigned long Bytes
+ unsigned long NumberNodes[NumNumberNodes]
+ unsigned short ValueNodes[(Bytes - (NumNumberNodes * sizeof(unsigned long)))
+ / sizeof(short)]
+
+ If the ByteOrderMark is equal to 0xFFFE, endian swapping is required in the
+ same way as described in the CHARACTER PROPERTIES section.
+
+ The NumberNodes array contains pairs of values, the first of which is the
+ character code and the second an index into the ValueNodes array. The
+ ValueNodes array contains pairs of integers which represent the numerator
+ and denominator of the numeric value of the character. If the character
+ happens to map to an integer, both the values in ValueNodes will be the
+ same.
diff --git a/intl/unicharutil/tools/genSpecialCasingData.pl b/intl/unicharutil/tools/genSpecialCasingData.pl
new file mode 100755
index 000000000..39b2d478e
--- /dev/null
+++ b/intl/unicharutil/tools/genSpecialCasingData.pl
@@ -0,0 +1,287 @@
+#!/usr/bin/env perl
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This tool is used to extract "special" (one-to-many) case mappings
+# into a form that can be used by nsTextRunTransformations.
+
+use strict;
+
+if ($#ARGV != 1) {
+ print <<__EOT;
+# Run this tool using a command line of the form
+#
+# perl genSpecialCasingData.pl UnicodeData.txt SpecialCasing.txt
+#
+# The nsSpecialCasingData.cpp file will be written to standard output.
+#
+# This tool will also write up-to-date versions of the test files
+# all-{upper,lower,title}.html
+# and corresponding -ref files in the current directory.
+#
+__EOT
+ exit 0;
+}
+
+my %allLower;
+my %allUpper;
+my %allTitle;
+my %compositions;
+my %gc;
+open FH, "< $ARGV[0]" or die "can't open $ARGV[0] (should be UnicodeData.txt)\n";
+while (<FH>) {
+ chomp;
+ my @fields = split /;/;
+ next if ($fields[1] =~ /</); # ignore ranges etc
+ my $usv = hex "0x$fields[0]";
+ $allUpper{$usv} = $fields[12] if $fields[12] ne '';
+ $allLower{$usv} = $fields[13] if $fields[13] ne '';
+ $allTitle{$usv} = $fields[14] if $fields[14] ne '';
+ $gc{$usv} = $fields[2];
+ # we only care about non-singleton canonical decomps
+ my $decomp = $fields[5];
+ next if $decomp eq '' or $decomp =~ /</ or not $decomp =~ / /;
+ $compositions{$decomp} = sprintf("%04X", $usv);
+}
+close FH;
+
+my %specialLower;
+my %specialUpper;
+my %specialTitle;
+my %charName;
+my @headerLines;
+open FH, "< $ARGV[1]" or die "can't open $ARGV[1] (should be SpecialCasing.txt)\n";
+while (<FH>) {
+ chomp;
+ m/#\s*(.+)$/;
+ my $comment = $1;
+ if ($comment =~ /^(SpecialCasing-|Date:)/) {
+ push @headerLines, $comment;
+ next;
+ }
+ s/#.*//;
+ s/;\s*$//;
+ next if $_ eq '';
+ my @fields = split /; */;
+ next unless (scalar @fields) == 4;
+ my $usv = hex "0x$fields[0]";
+ addIfSpecial(\%specialLower, $usv, $fields[1]);
+ addIfSpecial(\%specialTitle, $usv, $fields[2]);
+ addIfSpecial(\%specialUpper, $usv, $fields[3]);
+ $charName{$usv} = $comment;
+}
+close FH;
+
+print <<__END__;
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* Auto-generated from files in the Unicode Character Database
+ by genSpecialCasingData.pl - do not edit! */
+
+#include "nsSpecialCasingData.h"
+#include "mozilla/ArrayUtils.h" // for ArrayLength
+#include <stdlib.h> // for bsearch
+
+__END__
+map { print "/* $_ */\n" } @headerLines;
+
+print <<__END__;
+
+using mozilla::unicode::MultiCharMapping;
+
+__END__
+
+printMappings('Lower', \%specialLower);
+printMappings('Upper', \%specialUpper);
+printMappings('Title', \%specialTitle);
+
+print <<__END__;
+static int CompareMCM(const void* aKey, const void* aElement)
+{
+ const uint32_t ch = *static_cast<const uint32_t*>(aKey);
+ const MultiCharMapping* mcm = static_cast<const MultiCharMapping*>(aElement);
+ return int(ch) - int(mcm->mOriginalChar);
+}
+
+#define MAKE_SPECIAL_CASE_ACCESSOR(which) \\
+ const MultiCharMapping* \\
+ Special##which(uint32_t aChar) \\
+ { \\
+ const void* p = bsearch(&aChar, CaseSpecials_##which, \\
+ mozilla::ArrayLength(CaseSpecials_##which), \\
+ sizeof(MultiCharMapping), CompareMCM); \\
+ return static_cast<const MultiCharMapping*>(p); \\
+ }
+
+namespace mozilla {
+namespace unicode {
+
+MAKE_SPECIAL_CASE_ACCESSOR(Lower)
+MAKE_SPECIAL_CASE_ACCESSOR(Upper)
+MAKE_SPECIAL_CASE_ACCESSOR(Title)
+
+} // namespace unicode
+} // namespace mozilla
+__END__
+
+addSpecialsTo(\%allLower, \%specialLower);
+addSpecialsTo(\%allUpper, \%specialUpper);
+addSpecialsTo(\%allTitle, \%specialTitle);
+
+my $testFont = "../fonts/dejavu-sans/DejaVuSans.ttf";
+genTest('lower', \%allLower);
+genTest('upper', \%allUpper);
+genTitleTest();
+
+sub printMappings {
+ my ($whichMapping, $hash) = @_;
+ print "static const MultiCharMapping CaseSpecials_${whichMapping}[] = {\n";
+ foreach my $key (sort { $a <=> $b } keys %$hash) {
+ my @chars = split(/ /, $hash->{$key});
+ printf " { 0x%04x, {0x%04x, 0x%04x, 0x%04x} }, // %s\n", $key,
+ hex "0x0$chars[0]", hex "0x0$chars[1]", hex "0x0$chars[2]",
+ "$charName{$key}";
+ }
+ print "};\n\n";
+};
+
+sub addIfSpecial {
+ my ($hash, $usv, $mapping) = @_;
+ return unless $mapping =~ / /;
+ # only do compositions that start with the initial char
+ foreach (keys %compositions) {
+ $mapping =~ s/^$_/$compositions{$_}/;
+ }
+ $hash->{$usv} = $mapping;
+};
+
+sub addSpecialsTo {
+ my ($hash, $specials) = @_;
+ foreach my $key (keys %$specials) {
+ $hash->{$key} = $specials->{$key};
+ }
+};
+
+sub genTest {
+ my ($whichMapping, $hash) = @_;
+ open OUT, "> all-$whichMapping.html";
+ print OUT <<__END__;
+<!DOCTYPE html>
+<html>
+ <head>
+ <meta http-equiv="Content-type" content="text/html; charset=utf-8">
+ <style type="text/css">
+ \@font-face { font-family: foo; src: url($testFont); }
+ p { font-family: foo; text-transform: ${whichMapping}case; }
+ </style>
+ </head>
+ <body>
+ <p>
+__END__
+ foreach my $key (sort { $a <=> $b } keys %$hash) {
+ printf OUT "&#x%04X;", $key;
+ print OUT " <!-- $charName{$key} -->" if exists $charName{$key};
+ print OUT "\n";
+ }
+ print OUT <<__END__;
+ </p>
+ </body>
+</html>
+__END__
+ close OUT;
+
+ open OUT, "> all-$whichMapping-ref.html";
+ print OUT <<__END__;
+<!DOCTYPE html>
+<html>
+ <head>
+ <meta http-equiv="Content-type" content="text/html; charset=utf-8">
+ <style type="text/css">
+ \@font-face { font-family: foo; src: url($testFont); }
+ p { font-family: foo; }
+ </style>
+ </head>
+ <body>
+ <p>
+__END__
+ foreach my $key (sort { $a <=> $b } keys %$hash) {
+ print OUT join('', map { sprintf("&#x%s;", $_) } split(/ /, $hash->{$key}));
+ print OUT " <!-- $charName{$key} -->" if exists $charName{$key};
+ print OUT "\n";
+ }
+ print OUT <<__END__;
+ </p>
+ </body>
+</html>
+__END__
+ close OUT;
+};
+
+sub genTitleTest {
+ open OUT, "> all-title.html";
+ print OUT <<__END__;
+<!DOCTYPE html>
+<html>
+ <head>
+ <meta http-equiv="Content-type" content="text/html; charset=utf-8">
+ <style type="text/css">
+ \@font-face { font-family: foo; src: url($testFont); }
+ p { font-family: foo; text-transform: capitalize; }
+ </style>
+ </head>
+ <body>
+ <p>
+__END__
+ foreach my $key (sort { $a <=> $b } keys %allTitle) {
+ printf OUT "&#x%04X;x", $key;
+ print OUT " <!-- $charName{$key} -->" if exists $charName{$key};
+ print OUT "\n";
+ }
+ print OUT <<__END__;
+ </p>
+ </body>
+</html>
+__END__
+ close OUT;
+
+ open OUT, "> all-title-ref.html";
+ print OUT <<__END__;
+<!DOCTYPE html>
+<html>
+ <head>
+ <meta http-equiv="Content-type" content="text/html; charset=utf-8">
+ <style type="text/css">
+ \@font-face { font-family: foo; src: url($testFont); }
+ p { font-family: foo; }
+ </style>
+ </head>
+ <body>
+ <p>
+__END__
+ foreach my $key (sort { $a <=> $b } keys %allTitle) {
+ # capitalize is only applied to characters with GC=L* or N*...
+ if ($gc{$key} =~ /^[LN]/) {
+ # ...and those that are already uppercase are not transformed
+ if (exists $allUpper{$key}) {
+ print OUT join('', map { sprintf("&#x%s;", $_) } split(/ /, $allTitle{$key}));
+ } else {
+ printf OUT "&#x%04X;", $key;
+ }
+ print OUT "x";
+ } else {
+ printf OUT "&#x%04X;X", $key;
+ }
+ print OUT " <!-- $charName{$key} -->" if exists $charName{$key};
+ print OUT "\n";
+ }
+ print OUT <<__END__;
+ </p>
+ </body>
+</html>
+__END__
+ close OUT;
+};
diff --git a/intl/unicharutil/tools/genUnicodePropertyData.pl b/intl/unicharutil/tools/genUnicodePropertyData.pl
new file mode 100755
index 000000000..8b247e83c
--- /dev/null
+++ b/intl/unicharutil/tools/genUnicodePropertyData.pl
@@ -0,0 +1,943 @@
+#!/usr/bin/env perl
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This tool is used to prepare lookup tables of Unicode character properties
+# needed by gfx code to support text shaping operations. The properties are
+# read from the Unicode Character Database and compiled into multi-level arrays
+# for efficient lookup.
+#
+# To regenerate the tables in nsUnicodePropertyData.cpp:
+#
+# (1) Download the current Unicode data files from
+#
+# http://www.unicode.org/Public/UNIDATA/
+#
+# NB: not all the files are actually needed; currently, we require
+# - UnicodeData.txt
+# - Scripts.txt
+# - BidiMirroring.txt
+# - BidiBrackets.txt
+# - HangulSyllableType.txt
+# - LineBreak.txt
+# - EastAsianWidth.txt
+# - ReadMe.txt (to record version/date of the UCD)
+# - Unihan_Variants.txt (from Unihan.zip)
+# though this may change if we find a need for additional properties.
+#
+# The Unicode data files listed above should be together in one directory.
+#
+# We also require the file
+# http://www.unicode.org/Public/security/latest/xidmodifications.txt
+# This file should be in a sub-directory "security" immediately below the
+# directory containing the other Unicode data files.
+#
+# We also require the latest data file for UTR50, currently revision-13:
+# http://www.unicode.org/Public/vertical/revision-13/VerticalOrientation-13.txt
+# This file should be in a sub-directory "vertical" immediately below the
+# directory containing the other Unicode data files.
+#
+#
+# (2) Run this tool using a command line of the form
+#
+# perl genUnicodePropertyData.pl \
+# /path/to/harfbuzz/src \
+# /path/to/icu/common/unicode \
+# /path/to/UCD-directory
+#
+# This will generate (or overwrite!) the files
+#
+# nsUnicodePropertyData.cpp
+# nsUnicodeScriptCodes.h
+#
+# in the current directory.
+
+use strict;
+use List::Util qw(first);
+
+if ($#ARGV != 2) {
+ print <<__EOT;
+# Run this tool using a command line of the form
+#
+# perl genUnicodePropertyData.pl \\
+# /path/to/harfbuzz/src \\
+# /path/to/icu/common/unicode \\
+# /path/to/UCD-directory
+#
+# where harfbuzz/src is the directory containing harfbuzz .cc and .hh files,
+# icu/common/unicode is the directory containing ICU 'common' public headers,
+# and UCD-directory is a directory containing the current Unicode Character
+# Database files (UnicodeData.txt, etc), available from
+# http://www.unicode.org/Public/UNIDATA/, with additional resources as
+# detailed in the source comments.
+#
+# This will generate (or overwrite!) the files
+#
+# nsUnicodePropertyData.cpp
+# nsUnicodeScriptCodes.h
+#
+# in the current directory.
+__EOT
+ exit 0;
+}
+
+my $HARFBUZZ = $ARGV[0];
+my $ICU = $ARGV[1];
+my $UNICODE = $ARGV[2];
+
+# load HB_Category constants
+
+my $cc = -1;
+my %catCode;
+
+sub readHarfBuzzHeader
+{
+ my $file = shift;
+ open FH, "< $HARFBUZZ/$file" or die "can't open harfbuzz header $HARFBUZZ/$file\n";
+ while (<FH>) {
+ if (m/HB_UNICODE_GENERAL_CATEGORY_([A-Z_]+)/) {
+ $cc++;
+ $catCode{$1} = $cc;
+ }
+ }
+ close FH;
+}
+
+&readHarfBuzzHeader("hb-unicode.h");
+
+die "didn't find HarfBuzz category codes\n" if $cc == -1;
+
+my %scriptCode;
+my @scriptCodeToTag;
+my @scriptCodeToName;
+
+my $sc = -1;
+
+sub readIcuHeader
+{
+ my $file = shift;
+ open FH, "< $ICU/$file" or die "can't open ICU header $ICU/$file\n";
+ while (<FH>) {
+ # adjust for ICU vs UCD naming discrepancies
+ s/LANNA/TAI_THAM/;
+ s/MEITEI_MAYEK/MEETEI_MAYEK/;
+ s/ORKHON/OLD_TURKIC/;
+ s/MENDE/MENDE_KIKAKUI/;
+ s/SIGN_WRITING/SIGNWRITING/;
+ if (m|USCRIPT_([A-Z_]+)\s*=\s*([0-9]+),\s*/\*\s*([A-Z][a-z]{3})\s*\*/|) {
+ $sc = $2;
+ $scriptCode{$1} = $sc;
+ $scriptCodeToTag[$sc] = $3;
+ $scriptCodeToName[$sc] = $1;
+ }
+ }
+ close FH;
+}
+
+&readIcuHeader("uscript.h");
+
+die "didn't find ICU script codes\n" if $sc == -1;
+
+my %xidmodCode = (
+'Recommended' => 0,
+'Inclusion' => 1,
+'Uncommon_Use' => 2,
+'Technical' => 3,
+'Obsolete' => 4,
+'Aspirational' => 5,
+'Limited_Use' => 6,
+'Exclusion' => 7,
+'Not_XID' => 8,
+'Not_NFKC' => 9,
+'Default_Ignorable' => 10,
+'Deprecated' => 11,
+'not-chars' => 12
+);
+
+my %bidicategoryCode = (
+ "L" => 0, # Left-to-Right
+ "R" => 1, # Right-to-Left
+ "EN" => 2, # European Number
+ "ES" => 3, # European Number Separator
+ "ET" => 4, # European Number Terminator
+ "AN" => 5, # Arabic Number
+ "CS" => 6, # Common Number Separator
+ "B" => 7, # Paragraph Separator
+ "S" => 8, # Segment Separator
+ "WS" => 9, # Whitespace
+ "ON" => 10, # Other Neutrals
+ "LRE" => 11, # Left-to-Right Embedding
+ "LRO" => 12, # Left-to-Right Override
+ "AL" => 13, # Right-to-Left Arabic
+ "RLE" => 14, # Right-to-Left Embedding
+ "RLO" => 15, # Right-to-Left Override
+ "PDF" => 16, # Pop Directional Format
+ "NSM" => 17, # Non-Spacing Mark
+ "BN" => 18, # Boundary Neutral
+ "FSI" => 19, # First Strong Isolate
+ "LRI" => 20, # Left-to-Right Isolate
+ "RLI" => 21, # Right-to-left Isolate
+ "PDI" => 22 # Pop Direcitonal Isolate
+);
+
+my %verticalOrientationCode = (
+ 'U' => 0, # U - Upright, the same orientation as in the code charts
+ 'R' => 1, # R - Rotated 90 degrees clockwise compared to the code charts
+ 'Tu' => 2, # Tu - Transformed typographically, with fallback to Upright
+ 'Tr' => 3 # Tr - Transformed typographically, with fallback to Rotated
+);
+
+my %lineBreakCode = ( # ordering matches ICU's ULineBreak enum
+ "XX" => 0,
+ "AI" => 1,
+ "AL" => 2,
+ "B2" => 3,
+ "BA" => 4,
+ "BB" => 5,
+ "BK" => 6,
+ "CB" => 7,
+ "CL" => 8,
+ "CM" => 9,
+ "CR" => 10,
+ "EX" => 11,
+ "GL" => 12,
+ "HY" => 13,
+ "ID" => 14,
+ "IN" => 15,
+ "IS" => 16,
+ "LF" => 17,
+ "NS" => 18,
+ "NU" => 19,
+ "OP" => 20,
+ "PO" => 21,
+ "PR" => 22,
+ "QU" => 23,
+ "SA" => 24,
+ "SG" => 25,
+ "SP" => 26,
+ "SY" => 27,
+ "ZW" => 28,
+ "NL" => 29,
+ "WJ" => 30,
+ "H2" => 31,
+ "H3" => 32,
+ "JL" => 33,
+ "JT" => 34,
+ "JV" => 35,
+ "CP" => 36,
+ "CJ" => 37,
+ "HL" => 38,
+ "RI" => 39
+);
+
+my %eastAsianWidthCode = (
+ "N" => 0,
+ "A" => 1,
+ "H" => 2,
+ "W" => 3,
+ "F" => 4,
+ "Na" => 5
+);
+
+# initialize default properties
+my @script;
+my @category;
+my @combining;
+my @mirror;
+my @pairedBracketType;
+my @hangul;
+my @casemap;
+my @xidmod;
+my @numericvalue;
+my @hanVariant;
+my @bidicategory;
+my @fullWidth;
+my @fullWidthInverse;
+my @verticalOrientation;
+my @lineBreak;
+my @eastAsianWidthFWH;
+for (my $i = 0; $i < 0x110000; ++$i) {
+ $script[$i] = $scriptCode{"UNKNOWN"};
+ $category[$i] = $catCode{"UNASSIGNED"};
+ $combining[$i] = 0;
+ $pairedBracketType[$i] = 0;
+ $casemap[$i] = 0;
+ $xidmod[$i] = $xidmodCode{"not-chars"};
+ $numericvalue[$i] = -1;
+ $hanVariant[$i] = 0;
+ $bidicategory[$i] = $bidicategoryCode{"L"};
+ $fullWidth[$i] = 0;
+ $fullWidthInverse[$i] = 0;
+ $verticalOrientation[$i] = 1; # default for unlisted codepoints is 'R'
+ $lineBreak[$i] = $lineBreakCode{"XX"};
+ $eastAsianWidthFWH[$i] = 0;
+}
+
+# blocks where the default for bidi category is not L
+for my $i (0x0600..0x07BF, 0x08A0..0x08FF, 0xFB50..0xFDCF, 0xFDF0..0xFDFF, 0xFE70..0xFEFF, 0x1EE00..0x0001EEFF) {
+ $bidicategory[$i] = $bidicategoryCode{"AL"};
+}
+for my $i (0x0590..0x05FF, 0x07C0..0x089F, 0xFB1D..0xFB4F, 0x00010800..0x00010FFF, 0x0001E800..0x0001EDFF, 0x0001EF00..0x0001EFFF) {
+ $bidicategory[$i] = $bidicategoryCode{"R"};
+}
+for my $i (0x20A0..0x20CF) {
+ $bidicategory[$i] = $bidicategoryCode{"ET"};
+}
+
+my %ucd2hb = (
+'Cc' => 'CONTROL',
+'Cf' => 'FORMAT',
+'Cn' => 'UNASSIGNED',
+'Co' => 'PRIVATE_USE',
+'Cs' => 'SURROGATE',
+'Ll' => 'LOWERCASE_LETTER',
+'Lm' => 'MODIFIER_LETTER',
+'Lo' => 'OTHER_LETTER',
+'Lt' => 'TITLECASE_LETTER',
+'Lu' => 'UPPERCASE_LETTER',
+'Mc' => 'SPACING_MARK',
+'Me' => 'ENCLOSING_MARK',
+'Mn' => 'NON_SPACING_MARK',
+'Nd' => 'DECIMAL_NUMBER',
+'Nl' => 'LETTER_NUMBER',
+'No' => 'OTHER_NUMBER',
+'Pc' => 'CONNECT_PUNCTUATION',
+'Pd' => 'DASH_PUNCTUATION',
+'Pe' => 'CLOSE_PUNCTUATION',
+'Pf' => 'FINAL_PUNCTUATION',
+'Pi' => 'INITIAL_PUNCTUATION',
+'Po' => 'OTHER_PUNCTUATION',
+'Ps' => 'OPEN_PUNCTUATION',
+'Sc' => 'CURRENCY_SYMBOL',
+'Sk' => 'MODIFIER_SYMBOL',
+'Sm' => 'MATH_SYMBOL',
+'So' => 'OTHER_SYMBOL',
+'Zl' => 'LINE_SEPARATOR',
+'Zp' => 'PARAGRAPH_SEPARATOR',
+'Zs' => 'SPACE_SEPARATOR'
+);
+
+# read ReadMe.txt
+my @versionInfo;
+open FH, "< $UNICODE/ReadMe.txt" or die "can't open Unicode ReadMe.txt file\n";
+while (<FH>) {
+ chomp;
+ push @versionInfo, $_;
+}
+close FH;
+
+my $kTitleToUpper = 0x80000000;
+my $kUpperToLower = 0x40000000;
+my $kLowerToTitle = 0x20000000;
+my $kLowerToUpper = 0x10000000;
+my $kCaseMapCharMask = 0x001fffff;
+
+# read UnicodeData.txt
+open FH, "< $UNICODE/UnicodeData.txt" or die "can't open UCD file UnicodeData.txt\n";
+while (<FH>) {
+ chomp;
+ my @fields = split /;/;
+ if ($fields[1] =~ /First/) {
+ my $first = hex "0x$fields[0]";
+ $_ = <FH>;
+ @fields = split /;/;
+ if ($fields[1] =~ /Last/) {
+ my $last = hex "0x$fields[0]";
+ do {
+ $category[$first] = $catCode{$ucd2hb{$fields[2]}};
+ $combining[$first] = $fields[3];
+ $bidicategory[$first] = $bidicategoryCode{$fields[4]};
+ unless (length($fields[7]) == 0) {
+ $numericvalue[$first] = $fields[7];
+ }
+ if ($fields[1] =~ /CJK/) {
+ @hanVariant[$first] = 3;
+ }
+ $first++;
+ } while ($first <= $last);
+ } else {
+ die "didn't find Last code for range!\n";
+ }
+ } else {
+ my $usv = hex "0x$fields[0]";
+ $category[$usv] = $catCode{$ucd2hb{$fields[2]}};
+ $combining[$usv] = $fields[3];
+ my $upper = hex $fields[12];
+ my $lower = hex $fields[13];
+ my $title = hex $fields[14];
+ # we only store one mapping for each character,
+ # but also record what kind of mapping it is
+ if ($upper && $lower) {
+ $casemap[$usv] |= $kTitleToUpper;
+ $casemap[$usv] |= ($usv ^ $upper);
+ }
+ elsif ($lower) {
+ $casemap[$usv] |= $kUpperToLower;
+ $casemap[$usv] |= ($usv ^ $lower);
+ }
+ elsif ($title && ($title != $upper)) {
+ $casemap[$usv] |= $kLowerToTitle;
+ $casemap[$usv] |= ($usv ^ $title);
+ }
+ elsif ($upper) {
+ $casemap[$usv] |= $kLowerToUpper;
+ $casemap[$usv] |= ($usv ^ $upper);
+ }
+ $bidicategory[$usv] = $bidicategoryCode{$fields[4]};
+ unless (length($fields[7]) == 0) {
+ $numericvalue[$usv] = $fields[7];
+ }
+ if ($fields[1] =~ /CJK/) {
+ @hanVariant[$usv] = 3;
+ }
+ if ($fields[5] =~ /^<narrow>/) {
+ my $wideChar = hex(substr($fields[5], 9));
+ die "didn't expect supplementary-plane values here" if $usv > 0xffff || $wideChar > 0xffff;
+ $fullWidth[$usv] = $wideChar;
+ $fullWidthInverse[$wideChar] = $usv;
+ }
+ elsif ($fields[5] =~ /^<wide>/) {
+ my $narrowChar = hex(substr($fields[5], 7));
+ die "didn't expect supplementary-plane values here" if $usv > 0xffff || $narrowChar > 0xffff;
+ $fullWidth[$narrowChar] = $usv;
+ $fullWidthInverse[$usv] = $narrowChar;
+ }
+ }
+}
+close FH;
+
+# read Scripts.txt
+open FH, "< $UNICODE/Scripts.txt" or die "can't open UCD file Scripts.txt\n";
+push @versionInfo, "";
+while (<FH>) {
+ chomp;
+ push @versionInfo, $_;
+ last if /Date:/;
+}
+while (<FH>) {
+ if (m/([0-9A-F]{4,6})(?:\.\.([0-9A-F]{4,6}))*\s+;\s+([^ ]+)/) {
+ my $script = uc($3);
+ warn "unknown ICU script $script" unless exists $scriptCode{$script};
+ my $script = $scriptCode{$script};
+ my $start = hex "0x$1";
+ my $end = (defined $2) ? hex "0x$2" : $start;
+ for (my $i = $start; $i <= $end; ++$i) {
+ $script[$i] = $script;
+ }
+ }
+}
+close FH;
+
+# read BidiMirroring.txt
+my @offsets = ();
+push @offsets, 0;
+
+open FH, "< $UNICODE/BidiMirroring.txt" or die "can't open UCD file BidiMirroring.txt\n";
+push @versionInfo, "";
+while (<FH>) {
+ chomp;
+ push @versionInfo, $_;
+ last if /Date:/;
+}
+while (<FH>) {
+ s/#.*//;
+ if (m/([0-9A-F]{4,6});\s*([0-9A-F]{4,6})/) {
+ my $mirrorOffset = hex("0x$2") - hex("0x$1");
+ my $offsetIndex = first { $offsets[$_] eq $mirrorOffset } 0..$#offsets;
+ if ($offsetIndex == undef) {
+ die "too many offset codes\n" if scalar @offsets == 31;
+ push @offsets, $mirrorOffset;
+ $offsetIndex = $#offsets;
+ }
+ $mirror[hex "0x$1"] = $offsetIndex;
+ }
+}
+close FH;
+
+# read BidiBrackets.txt
+my %pairedBracketTypeCode = (
+ 'N' => 0,
+ 'O' => 1,
+ 'C' => 2
+);
+open FH, "< $UNICODE/BidiBrackets.txt" or die "can't open UCD file BidiBrackets.txt\n";
+push @versionInfo, "";
+while (<FH>) {
+ chomp;
+ push @versionInfo, $_;
+ last if /Date:/;
+}
+while (<FH>) {
+ s/#.*//;
+ if (m/([0-9A-F]{4,6});\s*([0-9A-F]{4,6});\s*(.)/) {
+ my $mirroredChar = $offsets[$mirror[hex "0x$1"]] + hex "0x$1";
+ die "bidi bracket does not match mirrored char\n" unless $mirroredChar == hex "0x$2";
+ my $pbt = uc($3);
+ warn "unknown Bidi Bracket type" unless exists $pairedBracketTypeCode{$pbt};
+ $pairedBracketType[hex "0x$1"] = $pairedBracketTypeCode{$pbt};
+ }
+}
+close FH;
+
+# read HangulSyllableType.txt
+my %hangulType = (
+ 'L' => 0x01,
+ 'V' => 0x02,
+ 'T' => 0x04,
+ 'LV' => 0x03,
+ 'LVT' => 0x07
+);
+open FH, "< $UNICODE/HangulSyllableType.txt" or die "can't open UCD file HangulSyllableType.txt\n";
+push @versionInfo, "";
+while (<FH>) {
+ chomp;
+ push @versionInfo, $_;
+ last if /Date:/;
+}
+while (<FH>) {
+ s/#.*//;
+ if (m/([0-9A-F]{4,6})(?:\.\.([0-9A-F]{4,6}))*\s*;\s*([^ ]+)/) {
+ my $hangul = uc($3);
+ warn "unknown Hangul syllable type" unless exists $hangulType{$hangul};
+ $hangul = $hangulType{$hangul};
+ my $start = hex "0x$1";
+ my $end = (defined $2) ? hex "0x$2" : $start;
+ for (my $i = $start; $i <= $end; ++$i) {
+ $hangul[$i] = $hangul;
+ }
+ }
+}
+close FH;
+
+# read LineBreak.txt
+open FH, "< $UNICODE/LineBreak.txt" or die "can't open UCD file LineBreak.txt\n";
+push @versionInfo, "";
+while (<FH>) {
+ chomp;
+ push @versionInfo, $_;
+ last if /Date:/;
+}
+while (<FH>) {
+ s/#.*//;
+ if (m/([0-9A-F]{4,6})(?:\.\.([0-9A-F]{4,6}))*\s*;\s*([^ ]+)/) {
+ my $lb = uc($3);
+ warn "unknown LineBreak class" unless exists $lineBreakCode{$lb};
+ $lb = $lineBreakCode{$lb};
+ my $start = hex "0x$1";
+ my $end = (defined $2) ? hex "0x$2" : $start;
+ for (my $i = $start; $i <= $end; ++$i) {
+ $lineBreak[$i] = $lb;
+ }
+ }
+}
+close FH;
+
+# read EastAsianWidth.txt
+open FH, "< $UNICODE/EastAsianWidth.txt" or die "can't open UCD file EastAsianWidth.txt\n";
+push @versionInfo, "";
+while (<FH>) {
+ chomp;
+ push @versionInfo, $_;
+ last if /Date:/;
+}
+while (<FH>) {
+ s/#.*//;
+ if (m/([0-9A-F]{4,6})(?:\.\.([0-9A-F]{4,6}))*\s*;\s*([^ ]+)/) {
+ my $start = hex "0x$1";
+ my $end = (defined $2) ? hex "0x$2" : $start;
+ my $eaw = $3;
+ warn "unknown EastAsianWidth class" unless exists $eastAsianWidthCode{$eaw};
+ my $isFWH = ($eaw =~ m/^[FWH]$/) ? 1 : 0;
+ for (my $i = $start; $i <= $end; ++$i) {
+ $eastAsianWidthFWH[$i] = $isFWH;
+ }
+ }
+}
+close FH;
+
+# read xidmodifications.txt
+open FH, "< $UNICODE/security/xidmodifications.txt" or die "can't open UCD file xidmodifications.txt\n";
+push @versionInfo, "";
+while (<FH>) {
+ chomp;
+ unless (/\xef\xbb\xbf/) {
+ push @versionInfo, $_;
+ }
+ last if /Generated:/;
+}
+while (<FH>) {
+ if (m/([0-9A-F]{4,6})(?:\.\.([0-9A-F]{4,6}))*\s+;\s+[^ ]+\s+;\s+([^ ]+)/) {
+ my $xidmod = $3;
+ warn "unknown Identifier Modification $xidmod" unless exists $xidmodCode{$xidmod};
+ $xidmod = $xidmodCode{$xidmod};
+ my $start = hex "0x$1";
+ my $end = (defined $2) ? hex "0x$2" : $start;
+ for (my $i = $start; $i <= $end; ++$i) {
+ $xidmod[$i] = $xidmod;
+ }
+ }
+}
+close FH;
+
+open FH, "< $UNICODE/Unihan_Variants.txt" or die "can't open UCD file Unihan_Variants.txt (from Unihan.zip)\n";
+push @versionInfo, "";
+while (<FH>) {
+ chomp;
+ push @versionInfo, $_;
+ last if /Date:/;
+}
+my $savedusv = 0;
+my $hasTC = 0;
+my $hasSC = 0;
+while (<FH>) {
+ chomp;
+ if (m/U\+([0-9A-F]{4,6})\s+k([^ ]+)Variant/) {
+ my $usv = hex "0x$1";
+ if ($usv != $savedusv) {
+ unless ($savedusv == 0) {
+ if ($hasTC && !$hasSC) {
+ $hanVariant[$savedusv] = 1;
+ } elsif (!$hasTC && $hasSC) {
+ $hanVariant[$savedusv] = 2;
+ }
+ }
+ $savedusv = $usv;
+ $hasTC = 0;
+ $hasSC = 0;
+ }
+ if ($2 eq "Traditional") {
+ $hasTC = 1;
+ }
+ if ($2 eq "Simplified") {
+ $hasSC = 1;
+ }
+ }
+}
+close FH;
+
+# read VerticalOrientation-13.txt
+open FH, "< $UNICODE/vertical/VerticalOrientation-13.txt" or die "can't open UTR50 data file VerticalOrientation-13.txt\n";
+push @versionInfo, "";
+while (<FH>) {
+ chomp;
+ push @versionInfo, $_;
+ last if /Date:/;
+}
+while (<FH>) {
+ chomp;
+ s/#.*//;
+ if (m/([0-9A-F]{4,6})(?:\.\.([0-9A-F]{4,6}))*\s*;\s*([^ ]+)/) {
+ my $vo = $3;
+ warn "unknown Vertical_Orientation code $vo"
+ unless exists $verticalOrientationCode{$vo};
+ $vo = $verticalOrientationCode{$vo};
+ my $start = hex "0x$1";
+ my $end = (defined $2) ? hex "0x$2" : $start;
+ for (my $i = $start; $i <= $end; ++$i) {
+ $verticalOrientation[$i] = $vo;
+ }
+ }
+}
+close FH;
+
+my $timestamp = gmtime();
+
+open DATA_TABLES, "> nsUnicodePropertyData.cpp" or die "unable to open nsUnicodePropertyData.cpp for output";
+
+my $licenseBlock = q[
+/* -*- Mode: C++; tab-width: 20; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * Derived from the Unicode Character Database by genUnicodePropertyData.pl
+ *
+ * For Unicode terms of use, see http://www.unicode.org/terms_of_use.html
+ */
+];
+
+my $versionInfo = join("\n", @versionInfo);
+
+print DATA_TABLES <<__END;
+$licenseBlock
+/*
+ * Created on $timestamp from UCD data files with version info:
+ *
+
+$versionInfo
+
+ *
+ * * * * * This file contains MACHINE-GENERATED DATA, do not edit! * * * * *
+ */
+
+#include <stdint.h>
+#include "harfbuzz/hb.h"
+
+__END
+
+open HEADER, "> nsUnicodeScriptCodes.h" or die "unable to open nsUnicodeScriptCodes.h for output";
+
+print HEADER <<__END;
+$licenseBlock
+/*
+ * Created on $timestamp from UCD data files with version info:
+ *
+
+$versionInfo
+
+ *
+ * * * * * This file contains MACHINE-GENERATED DATA, do not edit! * * * * *
+ */
+
+#ifndef NS_UNICODE_SCRIPT_CODES
+#define NS_UNICODE_SCRIPT_CODES
+
+__END
+
+print DATA_TABLES "#if !ENABLE_INTL_API\n";
+print DATA_TABLES "static const uint32_t sScriptCodeToTag[] = {\n";
+for (my $i = 0; $i < scalar @scriptCodeToTag; ++$i) {
+ printf DATA_TABLES " HB_TAG('%c','%c','%c','%c')", unpack('cccc', $scriptCodeToTag[$i]);
+ print DATA_TABLES $i < $#scriptCodeToTag ? ",\n" : "\n";
+}
+print DATA_TABLES "};\n";
+print DATA_TABLES "#endif\n\n";
+
+our $totalData = 0;
+
+print DATA_TABLES "#if !ENABLE_INTL_API\n";
+print DATA_TABLES "static const int16_t sMirrorOffsets[] = {\n";
+for (my $i = 0; $i < scalar @offsets; ++$i) {
+ printf DATA_TABLES " $offsets[$i]";
+ print DATA_TABLES $i < $#offsets ? ",\n" : "\n";
+}
+print DATA_TABLES "};\n";
+print DATA_TABLES "#endif\n\n";
+
+print HEADER "#pragma pack(1)\n\n";
+
+sub sprintCharProps1
+{
+ my $usv = shift;
+ return sprintf("{%d,%d,%d}, ", $mirror[$usv], $hangul[$usv], $combining[$usv]);
+}
+my $type = q/
+struct nsCharProps1 {
+ unsigned char mMirrorOffsetIndex:5;
+ unsigned char mHangulType:3;
+ unsigned char mCombiningClass:8;
+};
+/;
+&genTables("#if !ENABLE_INTL_API", "#endif",
+ "CharProp1", $type, "nsCharProps1", 11, 5, \&sprintCharProps1, 1, 2, 1);
+
+sub sprintCharProps2_short
+{
+ my $usv = shift;
+ return sprintf("{%d,%d},",
+ $verticalOrientation[$usv], $xidmod[$usv]);
+}
+$type = q|
+struct nsCharProps2 {
+ // Currently only 6 bits are defined here, so 2 more could be added without
+ // affecting the storage requirements for this struct.
+ unsigned char mVertOrient:2;
+ unsigned char mXidmod:4;
+};
+|;
+&genTables("#if ENABLE_INTL_API", "#endif",
+ "CharProp2", $type, "nsCharProps2", 9, 7, \&sprintCharProps2_short, 16, 1, 1);
+
+sub sprintCharProps2_full
+{
+ my $usv = shift;
+ return sprintf("{%d,%d,%d,%d,%d,%d,%d,%d,%d},",
+ $script[$usv], $pairedBracketType[$usv],
+ $eastAsianWidthFWH[$usv], $category[$usv],
+ $bidicategory[$usv], $xidmod[$usv], $numericvalue[$usv],
+ $verticalOrientation[$usv], $lineBreak[$usv]);
+}
+$type = q|
+struct nsCharProps2 {
+ unsigned char mScriptCode:8;
+ unsigned char mPairedBracketType:2;
+ unsigned char mEastAsianWidthFWH:1;
+ unsigned char mCategory:5;
+ unsigned char mBidiCategory:5;
+ unsigned char mXidmod:4;
+ signed char mNumericValue:5;
+ unsigned char mVertOrient:2;
+ unsigned char mLineBreak; // only 6 bits actually needed
+};
+|;
+&genTables("#if !ENABLE_INTL_API", "#endif",
+ "CharProp2", $type, "nsCharProps2", 12, 4, \&sprintCharProps2_full, 16, 5, 1);
+
+print HEADER "#pragma pack()\n\n";
+
+sub sprintHanVariants
+{
+ my $baseUsv = shift;
+ my $varShift = 0;
+ my $val = 0;
+ while ($varShift < 8) {
+ $val |= $hanVariant[$baseUsv++] << $varShift;
+ $varShift += 2;
+ }
+ return sprintf("0x%02x,", $val);
+}
+## Han Variant data currently unused but may be needed in future, see bug 857481
+## &genTables("", "", "HanVariant", "", "uint8_t", 9, 7, \&sprintHanVariants, 2, 1, 4);
+
+sub sprintFullWidth
+{
+ my $usv = shift;
+ return sprintf("0x%04x,", $fullWidth[$usv]);
+}
+&genTables("", "", "FullWidth", "", "uint16_t", 10, 6, \&sprintFullWidth, 0, 2, 1);
+
+sub sprintFullWidthInverse
+{
+ my $usv = shift;
+ return sprintf("0x%04x,", $fullWidthInverse[$usv]);
+}
+&genTables("", "", "FullWidthInverse", "", "uint16_t", 10, 6, \&sprintFullWidthInverse, 0, 2, 1);
+
+sub sprintCasemap
+{
+ my $usv = shift;
+ return sprintf("0x%08x,", $casemap[$usv]);
+}
+&genTables("#if !ENABLE_INTL_API", "#endif",
+ "CaseMap", "", "uint32_t", 11, 5, \&sprintCasemap, 1, 4, 1);
+
+print STDERR "Total data = $totalData\n";
+
+printf DATA_TABLES "const uint32_t kTitleToUpper = 0x%08x;\n", $kTitleToUpper;
+printf DATA_TABLES "const uint32_t kUpperToLower = 0x%08x;\n", $kUpperToLower;
+printf DATA_TABLES "const uint32_t kLowerToTitle = 0x%08x;\n", $kLowerToTitle;
+printf DATA_TABLES "const uint32_t kLowerToUpper = 0x%08x;\n", $kLowerToUpper;
+printf DATA_TABLES "const uint32_t kCaseMapCharMask = 0x%08x;\n\n", $kCaseMapCharMask;
+
+sub genTables
+{
+ my ($guardBegin, $guardEnd,
+ $prefix, $typedef, $type, $indexBits, $charBits, $func, $maxPlane, $bytesPerEntry, $charsPerEntry) = @_;
+
+ if ($typedef ne '') {
+ print HEADER "$guardBegin\n";
+ print HEADER "$typedef\n";
+ print HEADER "$guardEnd\n\n";
+ }
+
+ print DATA_TABLES "\n$guardBegin\n";
+ print DATA_TABLES "#define k${prefix}MaxPlane $maxPlane\n";
+ print DATA_TABLES "#define k${prefix}IndexBits $indexBits\n";
+ print DATA_TABLES "#define k${prefix}CharBits $charBits\n";
+
+ my $indexLen = 1 << $indexBits;
+ my $charsPerPage = 1 << $charBits;
+ my %charIndex = ();
+ my %pageMapIndex = ();
+ my @pageMap = ();
+ my @char = ();
+
+ my $planeMap = "\x00" x $maxPlane;
+ foreach my $plane (0 .. $maxPlane) {
+ my $pageMap = "\x00" x $indexLen * 2;
+ foreach my $page (0 .. $indexLen - 1) {
+ my $charValues = "";
+ for (my $ch = 0; $ch < $charsPerPage; $ch += $charsPerEntry) {
+ my $usv = $plane * 0x10000 + $page * $charsPerPage + $ch;
+ $charValues .= &$func($usv);
+ }
+ chop $charValues;
+
+ unless (exists $charIndex{$charValues}) {
+ $charIndex{$charValues} = scalar keys %charIndex;
+ $char[$charIndex{$charValues}] = $charValues;
+ }
+ substr($pageMap, $page * 2, 2) = pack('S', $charIndex{$charValues});
+ }
+
+ unless (exists $pageMapIndex{$pageMap}) {
+ $pageMapIndex{$pageMap} = scalar keys %pageMapIndex;
+ $pageMap[$pageMapIndex{$pageMap}] = $pageMap;
+ }
+ if ($plane > 0) {
+ substr($planeMap, $plane - 1, 1) = pack('C', $pageMapIndex{$pageMap});
+ }
+ }
+
+ if ($maxPlane) {
+ print DATA_TABLES "static const uint8_t s${prefix}Planes[$maxPlane] = {";
+ print DATA_TABLES join(',', map { sprintf("%d", $_) } unpack('C*', $planeMap));
+ print DATA_TABLES "};\n\n";
+ }
+
+ my $chCount = scalar @char;
+ my $pmBits = $chCount > 255 ? 16 : 8;
+ my $pmCount = scalar @pageMap;
+ if ($maxPlane == 0) {
+ die "there should only be one pageMap entry!" if $pmCount > 1;
+ print DATA_TABLES "static const uint${pmBits}_t s${prefix}Pages[$indexLen] = {\n";
+ } else {
+ print DATA_TABLES "static const uint${pmBits}_t s${prefix}Pages[$pmCount][$indexLen] = {\n";
+ }
+ for (my $i = 0; $i < scalar @pageMap; ++$i) {
+ print DATA_TABLES $maxPlane > 0 ? " {" : " ";
+ print DATA_TABLES join(',', map { sprintf("%d", $_) } unpack('S*', $pageMap[$i]));
+ print DATA_TABLES $maxPlane > 0 ? ($i < $#pageMap ? "},\n" : "}\n") : "\n";
+ }
+ print DATA_TABLES "};\n\n";
+
+ my $pageLen = $charsPerPage / $charsPerEntry;
+ print DATA_TABLES "static const $type s${prefix}Values[$chCount][$pageLen] = {\n";
+ for (my $i = 0; $i < scalar @char; ++$i) {
+ print DATA_TABLES " {";
+ print DATA_TABLES $char[$i];
+ print DATA_TABLES $i < $#char ? "},\n" : "}\n";
+ }
+ print DATA_TABLES "};\n";
+ print DATA_TABLES "$guardEnd\n";
+
+ my $dataSize = $pmCount * $indexLen * $pmBits/8 +
+ $chCount * $pageLen * $bytesPerEntry +
+ $maxPlane;
+ $totalData += $dataSize;
+
+ print STDERR "Data for $prefix = $dataSize\n";
+}
+
+print DATA_TABLES <<__END;
+/*
+ * * * * * This file contains MACHINE-GENERATED DATA, do not edit! * * * * *
+ */
+__END
+
+close DATA_TABLES;
+
+print HEADER "namespace mozilla {\n";
+print HEADER "namespace unicode {\n";
+print HEADER "enum class Script {\n";
+for (my $i = 0; $i < scalar @scriptCodeToName; ++$i) {
+ print HEADER " ", $scriptCodeToName[$i], " = ", $i, ",\n";
+}
+print HEADER "\n NUM_SCRIPT_CODES = ", scalar @scriptCodeToName, ",\n";
+print HEADER "\n INVALID = -1\n";
+print HEADER "};\n";
+print HEADER "} // namespace unicode\n";
+print HEADER "} // namespace mozilla\n\n";
+
+print HEADER <<__END;
+#endif
+/*
+ * * * * * This file contains MACHINE-GENERATED DATA, do not edit! * * * * *
+ */
+__END
+
+close HEADER;
+
diff --git a/intl/unicharutil/tools/moz.build b/intl/unicharutil/tools/moz.build
new file mode 100644
index 000000000..d2072d2aa
--- /dev/null
+++ b/intl/unicharutil/tools/moz.build
@@ -0,0 +1,12 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+Program('ucgendat')
+
+SOURCES += [
+ 'ucgendat.c',
+]
+
diff --git a/intl/unicharutil/tools/ucgendat.c b/intl/unicharutil/tools/ucgendat.c
new file mode 100644
index 000000000..c0134a0c0
--- /dev/null
+++ b/intl/unicharutil/tools/ucgendat.c
@@ -0,0 +1,1457 @@
+/*
+ * Copyright 1996, 1997, 1998 Computing Research Labs,
+ * New Mexico State University
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COMPUTING RESEARCH LAB OR NEW MEXICO STATE UNIVERSITY BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT
+ * OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
+ * THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef lint
+#ifdef __GNUC__
+static char rcsid[] __attribute__ ((unused)) = "$Id: ucgendat.c,v 1.1 1999/01/08 00:19:21 ftang%netscape.com Exp $";
+#else
+static char rcsid[] = "$Id: ucgendat.c,v 1.1 1999/01/08 00:19:21 ftang%netscape.com Exp $";
+#endif
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#ifndef WIN32
+#include <unistd.h>
+#endif
+
+#define ishdigit(cc) (((cc) >= '0' && (cc) <= '9') ||\
+ ((cc) >= 'A' && (cc) <= 'F') ||\
+ ((cc) >= 'a' && (cc) <= 'f'))
+
+/*
+ * A header written to the output file with the byte-order-mark and the number
+ * of property nodes.
+ */
+static unsigned short hdr[2] = {0xfeff, 0};
+
+#define NUMPROPS 49
+#define NEEDPROPS (NUMPROPS + (4 - (NUMPROPS & 3)))
+
+typedef struct {
+ char *name;
+ int len;
+} _prop_t;
+
+/*
+ * List of properties expected to be found in the Unicode Character Database
+ * including some implementation specific properties.
+ *
+ * The implementation specific properties are:
+ * Cm = Composed (can be decomposed)
+ * Nb = Non-breaking
+ * Sy = Symmetric (has left and right forms)
+ * Hd = Hex digit
+ * Qm = Quote marks
+ * Mr = Mirroring
+ * Ss = Space, other
+ * Cp = Defined character
+ */
+static _prop_t props[NUMPROPS] = {
+ {"Mn", 2}, {"Mc", 2}, {"Me", 2}, {"Nd", 2}, {"Nl", 2}, {"No", 2},
+ {"Zs", 2}, {"Zl", 2}, {"Zp", 2}, {"Cc", 2}, {"Cf", 2}, {"Cs", 2},
+ {"Co", 2}, {"Cn", 2}, {"Lu", 2}, {"Ll", 2}, {"Lt", 2}, {"Lm", 2},
+ {"Lo", 2}, {"Pc", 2}, {"Pd", 2}, {"Ps", 2}, {"Pe", 2}, {"Po", 2},
+ {"Sm", 2}, {"Sc", 2}, {"Sk", 2}, {"So", 2}, {"L", 1}, {"R", 1},
+ {"EN", 2}, {"ES", 2}, {"ET", 2}, {"AN", 2}, {"CS", 2}, {"B", 1},
+ {"S", 1}, {"WS", 2}, {"ON", 2},
+ {"Cm", 2}, {"Nb", 2}, {"Sy", 2}, {"Hd", 2}, {"Qm", 2}, {"Mr", 2},
+ {"Ss", 2}, {"Cp", 2}, {"Pi", 2}, {"Pf", 2}
+};
+
+typedef struct {
+ unsigned long *ranges;
+ unsigned short used;
+ unsigned short size;
+} _ranges_t;
+
+static _ranges_t proptbl[NUMPROPS];
+
+/*
+ * Make sure this array is sized to be on a 4-byte boundary at compile time.
+ */
+static unsigned short propcnt[NEEDPROPS];
+
+/*
+ * Array used to collect a decomposition before adding it to the decomposition
+ * table.
+ */
+static unsigned long dectmp[64];
+static unsigned long dectmp_size;
+
+typedef struct {
+ unsigned long code;
+ unsigned short size;
+ unsigned short used;
+ unsigned long *decomp;
+} _decomp_t;
+
+/*
+ * List of decomposition. Created and expanded in order as the characters are
+ * encountered.
+ */
+static _decomp_t *decomps;
+static unsigned long decomps_used;
+static unsigned long decomps_size;
+
+/*
+ * Types and lists for handling lists of case mappings.
+ */
+typedef struct {
+ unsigned long key;
+ unsigned long other1;
+ unsigned long other2;
+} _case_t;
+
+static _case_t *upper;
+static _case_t *lower;
+static _case_t *title;
+static unsigned long upper_used;
+static unsigned long upper_size;
+static unsigned long lower_used;
+static unsigned long lower_size;
+static unsigned long title_used;
+static unsigned long title_size;
+
+/*
+ * Array used to collect case mappings before adding them to a list.
+ */
+static unsigned long cases[3];
+
+/*
+ * An array to hold ranges for combining classes.
+ */
+static unsigned long *ccl;
+static unsigned long ccl_used;
+static unsigned long ccl_size;
+
+/*
+ * Structures for handling numbers.
+ */
+typedef struct {
+ unsigned long code;
+ unsigned long idx;
+} _codeidx_t;
+
+typedef struct {
+ short numerator;
+ short denominator;
+} _num_t;
+
+/*
+ * Arrays to hold the mapping of codes to numbers.
+ */
+static _codeidx_t *ncodes;
+static unsigned long ncodes_used;
+static unsigned long ncodes_size;
+
+static _num_t *nums;
+static unsigned long nums_used;
+static unsigned long nums_size;
+
+/*
+ * Array for holding numbers.
+ */
+static _num_t *nums;
+static unsigned long nums_used;
+static unsigned long nums_size;
+
+static void
+#ifdef __STDC__
+add_range(unsigned long start, unsigned long end, char *p1, char *p2)
+#else
+add_range(start, end, p1, p2)
+unsigned long start, end;
+char *p1, *p2;
+#endif
+{
+ int i, j, k, len;
+ _ranges_t *rlp;
+ char *name;
+
+ for (k = 0; k < 2; k++) {
+ if (k == 0) {
+ name = p1;
+ len = 2;
+ } else {
+ if (p2 == 0)
+ break;
+
+ name = p2;
+ len = 1;
+ }
+
+ for (i = 0; i < NUMPROPS; i++) {
+ if (props[i].len == len && memcmp(props[i].name, name, len) == 0)
+ break;
+ }
+
+ if (i == NUMPROPS)
+ continue;
+
+ rlp = &proptbl[i];
+
+ /*
+ * Resize the range list if necessary.
+ */
+ if (rlp->used == rlp->size) {
+ if (rlp->size == 0)
+ rlp->ranges = (unsigned long *)
+ malloc(sizeof(unsigned long) << 3);
+ else
+ rlp->ranges = (unsigned long *)
+ realloc((char *) rlp->ranges,
+ sizeof(unsigned long) * (rlp->size + 8));
+ rlp->size += 8;
+ }
+
+ /*
+ * If this is the first code for this property list, just add it
+ * and return.
+ */
+ if (rlp->used == 0) {
+ rlp->ranges[0] = start;
+ rlp->ranges[1] = end;
+ rlp->used += 2;
+ continue;
+ }
+
+ /*
+ * Optimize the case of adding the range to the end.
+ */
+ j = rlp->used - 1;
+ if (start > rlp->ranges[j]) {
+ j = rlp->used;
+ rlp->ranges[j++] = start;
+ rlp->ranges[j++] = end;
+ rlp->used = j;
+ continue;
+ }
+
+ /*
+ * Need to locate the insertion point.
+ */
+ for (i = 0;
+ i < rlp->used && start > rlp->ranges[i + 1] + 1; i += 2) ;
+
+ /*
+ * If the start value lies in the current range, then simply set the
+ * new end point of the range to the end value passed as a parameter.
+ */
+ if (rlp->ranges[i] <= start && start <= rlp->ranges[i + 1] + 1) {
+ rlp->ranges[i + 1] = end;
+ return;
+ }
+
+ /*
+ * Shift following values up by two.
+ */
+ for (j = rlp->used; j > i; j -= 2) {
+ rlp->ranges[j] = rlp->ranges[j - 2];
+ rlp->ranges[j + 1] = rlp->ranges[j - 1];
+ }
+
+ /*
+ * Add the new range at the insertion point.
+ */
+ rlp->ranges[i] = start;
+ rlp->ranges[i + 1] = end;
+ rlp->used += 2;
+ }
+}
+
+static void
+#ifdef __STDC__
+ordered_range_insert(unsigned long c, char *name, int len)
+#else
+ordered_range_insert(c, name, len)
+unsigned long c;
+char *name;
+int len;
+#endif
+{
+ int i, j;
+ unsigned long s, e;
+ _ranges_t *rlp;
+
+ if (len == 0)
+ return;
+
+ for (i = 0; i < NUMPROPS; i++) {
+ if (props[i].len == len && memcmp(props[i].name, name, len) == 0)
+ break;
+ }
+
+ if (i == NUMPROPS)
+ return;
+
+ /*
+ * Have a match, so insert the code in order.
+ */
+ rlp = &proptbl[i];
+
+ /*
+ * Resize the range list if necessary.
+ */
+ if (rlp->used == rlp->size) {
+ if (rlp->size == 0)
+ rlp->ranges = (unsigned long *)
+ malloc(sizeof(unsigned long) << 3);
+ else
+ rlp->ranges = (unsigned long *)
+ realloc((char *) rlp->ranges,
+ sizeof(unsigned long) * (rlp->size + 8));
+ rlp->size += 8;
+ }
+
+ /*
+ * If this is the first code for this property list, just add it
+ * and return.
+ */
+ if (rlp->used == 0) {
+ rlp->ranges[0] = rlp->ranges[1] = c;
+ rlp->used += 2;
+ return;
+ }
+
+ /*
+ * Optimize the cases of extending the last range and adding new ranges to
+ * the end.
+ */
+ j = rlp->used - 1;
+ e = rlp->ranges[j];
+ s = rlp->ranges[j - 1];
+
+ if (c == e + 1) {
+ /*
+ * Extend the last range.
+ */
+ rlp->ranges[j] = c;
+ return;
+ }
+
+ if (c > e + 1) {
+ /*
+ * Start another range on the end.
+ */
+ j = rlp->used;
+ rlp->ranges[j] = rlp->ranges[j + 1] = c;
+ rlp->used += 2;
+ return;
+ }
+
+ if (c >= s)
+ /*
+ * The code is a duplicate of a code in the last range, so just return.
+ */
+ return;
+
+ /*
+ * The code should be inserted somewhere before the last range in the
+ * list. Locate the insertion point.
+ */
+ for (i = 0;
+ i < rlp->used && c > rlp->ranges[i + 1] + 1; i += 2) ;
+
+ s = rlp->ranges[i];
+ e = rlp->ranges[i + 1];
+
+ if (c == e + 1)
+ /*
+ * Simply extend the current range.
+ */
+ rlp->ranges[i + 1] = c;
+ else if (c < s) {
+ /*
+ * Add a new entry before the current location. Shift all entries
+ * before the current one up by one to make room.
+ */
+ for (j = rlp->used; j > i; j -= 2) {
+ rlp->ranges[j] = rlp->ranges[j - 2];
+ rlp->ranges[j + 1] = rlp->ranges[j - 1];
+ }
+ rlp->ranges[i] = rlp->ranges[i + 1] = c;
+
+ rlp->used += 2;
+ }
+}
+
+static void
+#ifdef __STDC__
+add_decomp(unsigned long code)
+#else
+add_decomp(code)
+unsigned long code;
+#endif
+{
+ unsigned long i, j, size;
+
+ /*
+ * Add the code to the composite property.
+ */
+ ordered_range_insert(code, "Cm", 2);
+
+ /*
+ * Locate the insertion point for the code.
+ */
+ for (i = 0; i < decomps_used && code > decomps[i].code; i++) ;
+
+ /*
+ * Allocate space for a new decomposition.
+ */
+ if (decomps_used == decomps_size) {
+ if (decomps_size == 0)
+ decomps = (_decomp_t *) malloc(sizeof(_decomp_t) << 3);
+ else
+ decomps = (_decomp_t *)
+ realloc((char *) decomps,
+ sizeof(_decomp_t) * (decomps_size + 8));
+ (void) memset((char *) (decomps + decomps_size), 0,
+ sizeof(_decomp_t) << 3);
+ decomps_size += 8;
+ }
+
+ if (i < decomps_used && code != decomps[i].code) {
+ /*
+ * Shift the decomps up by one if the codes don't match.
+ */
+ for (j = decomps_used; j > i; j--)
+ (void) memcpy((char *) &decomps[j], (char *) &decomps[j - 1],
+ sizeof(_decomp_t));
+ }
+
+ /*
+ * Insert or replace a decomposition.
+ */
+ size = dectmp_size + (4 - (dectmp_size & 3));
+ if (decomps[i].size < size) {
+ if (decomps[i].size == 0)
+ decomps[i].decomp = (unsigned long *)
+ malloc(sizeof(unsigned long) * size);
+ else
+ decomps[i].decomp = (unsigned long *)
+ realloc((char *) decomps[i].decomp,
+ sizeof(unsigned long) * size);
+ decomps[i].size = size;
+ }
+
+ if (decomps[i].code != code)
+ decomps_used++;
+
+ decomps[i].code = code;
+ decomps[i].used = dectmp_size;
+ (void) memcpy((char *) decomps[i].decomp, (char *) dectmp,
+ sizeof(unsigned long) * dectmp_size);
+
+}
+
+static void
+#ifdef __STDC__
+add_title(unsigned long code)
+#else
+add_title(code)
+unsigned long code;
+#endif
+{
+ unsigned long i, j;
+
+ /*
+ * Always map the code to itself.
+ */
+ cases[2] = code;
+
+ if (title_used == title_size) {
+ if (title_size == 0)
+ title = (_case_t *) malloc(sizeof(_case_t) << 3);
+ else
+ title = (_case_t *) realloc((char *) title,
+ sizeof(_case_t) * (title_size + 8));
+ title_size += 8;
+ }
+
+ /*
+ * Locate the insertion point.
+ */
+ for (i = 0; i < title_used && code > title[i].key; i++) ;
+
+ if (i < title_used) {
+ /*
+ * Shift the array up by one.
+ */
+ for (j = title_used; j > i; j--)
+ (void) memcpy((char *) &title[j], (char *) &title[j - 1],
+ sizeof(_case_t));
+ }
+
+ title[i].key = cases[2]; /* Title */
+ title[i].other1 = cases[0]; /* Upper */
+ title[i].other2 = cases[1]; /* Lower */
+
+ title_used++;
+}
+
+static void
+#ifdef __STDC__
+add_upper(unsigned long code)
+#else
+add_upper(code)
+unsigned long code;
+#endif
+{
+ unsigned long i, j;
+
+ /*
+ * Always map the code to itself.
+ */
+ cases[0] = code;
+
+ /*
+ * If the title case character is not present, then make it the same as
+ * the upper case.
+ */
+ if (cases[2] == 0)
+ cases[2] = code;
+
+ if (upper_used == upper_size) {
+ if (upper_size == 0)
+ upper = (_case_t *) malloc(sizeof(_case_t) << 3);
+ else
+ upper = (_case_t *) realloc((char *) upper,
+ sizeof(_case_t) * (upper_size + 8));
+ upper_size += 8;
+ }
+
+ /*
+ * Locate the insertion point.
+ */
+ for (i = 0; i < upper_used && code > upper[i].key; i++) ;
+
+ if (i < upper_used) {
+ /*
+ * Shift the array up by one.
+ */
+ for (j = upper_used; j > i; j--)
+ (void) memcpy((char *) &upper[j], (char *) &upper[j - 1],
+ sizeof(_case_t));
+ }
+
+ upper[i].key = cases[0]; /* Upper */
+ upper[i].other1 = cases[1]; /* Lower */
+ upper[i].other2 = cases[2]; /* Title */
+
+ upper_used++;
+}
+
+static void
+#ifdef __STDC__
+add_lower(unsigned long code)
+#else
+add_lower(code)
+unsigned long code;
+#endif
+{
+ unsigned long i, j;
+
+ /*
+ * Always map the code to itself.
+ */
+ cases[1] = code;
+
+ /*
+ * If the title case character is empty, then make it the same as the
+ * upper case.
+ */
+ if (cases[2] == 0)
+ cases[2] = cases[0];
+
+ if (lower_used == lower_size) {
+ if (lower_size == 0)
+ lower = (_case_t *) malloc(sizeof(_case_t) << 3);
+ else
+ lower = (_case_t *) realloc((char *) lower,
+ sizeof(_case_t) * (lower_size + 8));
+ lower_size += 8;
+ }
+
+ /*
+ * Locate the insertion point.
+ */
+ for (i = 0; i < lower_used && code > lower[i].key; i++) ;
+
+ if (i < lower_used) {
+ /*
+ * Shift the array up by one.
+ */
+ for (j = lower_used; j > i; j--)
+ (void) memcpy((char *) &lower[j], (char *) &lower[j - 1],
+ sizeof(_case_t));
+ }
+
+ lower[i].key = cases[1]; /* Lower */
+ lower[i].other1 = cases[0]; /* Upper */
+ lower[i].other2 = cases[2]; /* Title */
+
+ lower_used++;
+}
+
+static void
+#ifdef __STDC__
+ordered_ccl_insert(unsigned long c, unsigned long ccl_code)
+#else
+ordered_ccl_insert(c, ccl_code)
+unsigned long c, ccl_code;
+#endif
+{
+ unsigned long i, j;
+
+ if (ccl_used == ccl_size) {
+ if (ccl_size == 0)
+ ccl = (unsigned long *) malloc(sizeof(unsigned long) * 24);
+ else
+ ccl = (unsigned long *)
+ realloc((char *) ccl, sizeof(unsigned long) * (ccl_size + 24));
+ ccl_size += 24;
+ }
+
+ /*
+ * Optimize adding the first item.
+ */
+ if (ccl_used == 0) {
+ ccl[0] = ccl[1] = c;
+ ccl[2] = ccl_code;
+ ccl_used += 3;
+ return;
+ }
+
+ /*
+ * Handle the special case of extending the range on the end. This
+ * requires that the combining class codes are the same.
+ */
+ if (ccl_code == ccl[ccl_used - 1] && c == ccl[ccl_used - 2] + 1) {
+ ccl[ccl_used - 2] = c;
+ return;
+ }
+
+ /*
+ * Handle the special case of adding another range on the end.
+ */
+ if (c > ccl[ccl_used - 2] + 1 ||
+ (c == ccl[ccl_used - 2] + 1 && ccl_code != ccl[ccl_used - 1])) {
+ ccl[ccl_used++] = c;
+ ccl[ccl_used++] = c;
+ ccl[ccl_used++] = ccl_code;
+ return;
+ }
+
+ /*
+ * Locate either the insertion point or range for the code.
+ */
+ for (i = 0; i < ccl_used && c > ccl[i + 1] + 1; i += 3) ;
+
+ if (ccl_code == ccl[i + 2] && c == ccl[i + 1] + 1) {
+ /*
+ * Extend an existing range.
+ */
+ ccl[i + 1] = c;
+ return;
+ } else if (c < ccl[i]) {
+ /*
+ * Start a new range before the current location.
+ */
+ for (j = ccl_used; j > i; j -= 3) {
+ ccl[j] = ccl[j - 3];
+ ccl[j - 1] = ccl[j - 4];
+ ccl[j - 2] = ccl[j - 5];
+ }
+ ccl[i] = ccl[i + 1] = c;
+ ccl[i + 2] = ccl_code;
+ }
+}
+
+/*
+ * Adds a number if it does not already exist and returns an index value
+ * multiplied by 2.
+ */
+static unsigned long
+#ifdef __STDC__
+make_number(short num, short denom)
+#else
+make_number(num, denom)
+short num, denom;
+#endif
+{
+ unsigned long n;
+
+ /*
+ * Determine if the number already exists.
+ */
+ for (n = 0; n < nums_used; n++) {
+ if (nums[n].numerator == num && nums[n].denominator == denom)
+ return n << 1;
+ }
+
+ if (nums_used == nums_size) {
+ if (nums_size == 0)
+ nums = (_num_t *) malloc(sizeof(_num_t) << 3);
+ else
+ nums = (_num_t *) realloc((char *) nums,
+ sizeof(_num_t) * (nums_size + 8));
+ nums_size += 8;
+ }
+
+ n = nums_used++;
+ nums[n].numerator = num;
+ nums[n].denominator = denom;
+
+ return n << 1;
+}
+
+static void
+#ifdef __STDC__
+add_number(unsigned long code, short num, short denom)
+#else
+add_number(code, num, denom)
+unsigned long code;
+short num, denom;
+#endif
+{
+ unsigned long i, j;
+
+ /*
+ * Insert the code in order.
+ */
+ for (i = 0; i < ncodes_used && code > ncodes[i].code; i++) ;
+
+ /*
+ * Handle the case of the codes matching and simply replace the number
+ * that was there before.
+ */
+ if (ncodes_used > 0 && code == ncodes[i].code) {
+ ncodes[i].idx = make_number(num, denom);
+ return;
+ }
+
+ /*
+ * Resize the array if necessary.
+ */
+ if (ncodes_used == ncodes_size) {
+ if (ncodes_size == 0)
+ ncodes = (_codeidx_t *) malloc(sizeof(_codeidx_t) << 3);
+ else
+ ncodes = (_codeidx_t *)
+ realloc((char *) ncodes, sizeof(_codeidx_t) * (ncodes_size + 8));
+
+ ncodes_size += 8;
+ }
+
+ /*
+ * Shift things around to insert the code if necessary.
+ */
+ if (i < ncodes_used) {
+ for (j = ncodes_used; j > i; j--) {
+ ncodes[j].code = ncodes[j - 1].code;
+ ncodes[j].idx = ncodes[j - 1].idx;
+ }
+ }
+ ncodes[i].code = code;
+ ncodes[i].idx = make_number(num, denom);
+
+ ncodes_used++;
+}
+
+/*
+ * This routine assumes that the line is a valid Unicode Character Database
+ * entry.
+ */
+static void
+#ifdef __STDC__
+read_cdata(FILE *in)
+#else
+read_cdata(in)
+FILE *in;
+#endif
+{
+ unsigned long i, lineno, skip, code, ccl_code;
+ short wnum, neg, number[2];
+ char line[512], *s, *e;
+
+ lineno = skip = 0;
+ while (fscanf(in, "%[^\n]\n", line) != EOF) {
+ lineno++;
+
+ /*
+ * Skip blank lines and lines that start with a '#'.
+ */
+ if (line[0] == 0 || line[0] == '#')
+ continue;
+
+ /*
+ * If lines need to be skipped, do it here.
+ */
+ if (skip) {
+ skip--;
+ continue;
+ }
+
+ /*
+ * Collect the code. The code can be up to 6 hex digits in length to
+ * allow surrogates to be specified.
+ */
+ for (s = line, i = code = 0; *s != ';' && i < 6; i++, s++) {
+ code <<= 4;
+ if (*s >= '0' && *s <= '9')
+ code += *s - '0';
+ else if (*s >= 'A' && *s <= 'F')
+ code += (*s - 'A') + 10;
+ else if (*s >= 'a' && *s <= 'f')
+ code += (*s - 'a') + 10;
+ }
+
+ /*
+ * Handle the following special cases:
+ * 1. 4E00-9FA5 CJK Ideographs.
+ * 2. AC00-D7A3 Hangul Syllables.
+ * 3. D800-DFFF Surrogates.
+ * 4. E000-F8FF Private Use Area.
+ * 5. F900-FA2D Han compatibility.
+ */
+ switch (code) {
+ case 0x4e00:
+ /*
+ * The Han ideographs.
+ */
+ add_range(0x4e00, 0x9fff, "Lo", "L");
+
+ /*
+ * Add the characters to the defined category.
+ */
+ add_range(0x4e00, 0x9fa5, "Cp", 0);
+
+ skip = 1;
+ break;
+ case 0xac00:
+ /*
+ * The Hangul syllables.
+ */
+ add_range(0xac00, 0xd7a3, "Lo", "L");
+
+ /*
+ * Add the characters to the defined category.
+ */
+ add_range(0xac00, 0xd7a3, "Cp", 0);
+
+ skip = 1;
+ break;
+ case 0xd800:
+ /*
+ * Make a range of all surrogates and assume some default
+ * properties.
+ */
+ add_range(0x010000, 0x10ffff, "Cs", "L");
+ skip = 5;
+ break;
+ case 0xe000:
+ /*
+ * The Private Use area. Add with a default set of properties.
+ */
+ add_range(0xe000, 0xf8ff, "Co", "L");
+ skip = 1;
+ break;
+ case 0xf900:
+ /*
+ * The CJK compatibility area.
+ */
+ add_range(0xf900, 0xfaff, "Lo", "L");
+
+ /*
+ * Add the characters to the defined category.
+ */
+ add_range(0xf900, 0xfaff, "Cp", 0);
+
+ skip = 1;
+ }
+
+ if (skip)
+ continue;
+
+ /*
+ * Add the code to the defined category.
+ */
+ ordered_range_insert(code, "Cp", 2);
+
+ /*
+ * Locate the first character property field.
+ */
+ for (i = 0; *s != 0 && i < 2; s++) {
+ if (*s == ';')
+ i++;
+ }
+ for (e = s; *e && *e != ';'; e++) ;
+
+ ordered_range_insert(code, s, e - s);
+
+ /*
+ * Locate the combining class code.
+ */
+ for (s = e; *s != 0 && i < 3; s++) {
+ if (*s == ';')
+ i++;
+ }
+
+ /*
+ * Convert the combining class code from decimal.
+ */
+ for (ccl_code = 0, e = s; *e && *e != ';'; e++)
+ ccl_code = (ccl_code * 10) + (*e - '0');
+
+ /*
+ * Add the code if it not 0.
+ */
+ if (ccl_code != 0)
+ ordered_ccl_insert(code, ccl_code);
+
+ /*
+ * Locate the second character property field.
+ */
+ for (s = e; *s != 0 && i < 4; s++) {
+ if (*s == ';')
+ i++;
+ }
+ for (e = s; *e && *e != ';'; e++) ;
+
+ ordered_range_insert(code, s, e - s);
+
+ /*
+ * Check for a decomposition.
+ */
+ s = ++e;
+ if (*s != ';' && *s != '<') {
+ /*
+ * Collect the codes of the decomposition.
+ */
+ for (dectmp_size = 0; *s != ';'; ) {
+ /*
+ * Skip all leading non-hex digits.
+ */
+ while (!ishdigit(*s))
+ s++;
+
+ for (dectmp[dectmp_size] = 0; ishdigit(*s); s++) {
+ dectmp[dectmp_size] <<= 4;
+ if (*s >= '0' && *s <= '9')
+ dectmp[dectmp_size] += *s - '0';
+ else if (*s >= 'A' && *s <= 'F')
+ dectmp[dectmp_size] += (*s - 'A') + 10;
+ else if (*s >= 'a' && *s <= 'f')
+ dectmp[dectmp_size] += (*s - 'a') + 10;
+ }
+ dectmp_size++;
+ }
+
+ /*
+ * If there is more than one code in the temporary decomposition
+ * array, then add the character with its decomposition.
+ */
+ if (dectmp_size > 1)
+ add_decomp(code);
+ }
+
+ /*
+ * Skip to the number field.
+ */
+ for (i = 0; i < 3 && *s; s++) {
+ if (*s == ';')
+ i++;
+ }
+
+ /*
+ * Scan the number in.
+ */
+ number[0] = number[1] = 0;
+ for (e = s, neg = wnum = 0; *e && *e != ';'; e++) {
+ if (*e == '-') {
+ neg = 1;
+ continue;
+ }
+
+ if (*e == '/') {
+ /*
+ * Move the the denominator of the fraction.
+ */
+ if (neg)
+ number[wnum] *= -1;
+ neg = 0;
+ e++;
+ wnum++;
+ }
+ number[wnum] = (number[wnum] * 10) + (*e - '0');
+ }
+
+ if (e > s) {
+ /*
+ * Adjust the denominator in case of integers and add the number.
+ */
+ if (wnum == 0)
+ number[1] = number[0];
+
+ add_number(code, number[0], number[1]);
+ }
+
+ /*
+ * Skip to the start of the possible case mappings.
+ */
+ for (s = e, i = 0; i < 4 && *s; s++) {
+ if (*s == ';')
+ i++;
+ }
+
+ /*
+ * Collect the case mappings.
+ */
+ cases[0] = cases[1] = cases[2] = 0;
+ for (i = 0; i < 3; i++) {
+ while (ishdigit(*s)) {
+ cases[i] <<= 4;
+ if (*s >= '0' && *s <= '9')
+ cases[i] += *s - '0';
+ else if (*s >= 'A' && *s <= 'F')
+ cases[i] += (*s - 'A') + 10;
+ else if (*s >= 'a' && *s <= 'f')
+ cases[i] += (*s - 'a') + 10;
+ s++;
+ }
+ if (*s == ';')
+ s++;
+ }
+ if (cases[0] && cases[1])
+ /*
+ * Add the upper and lower mappings for a title case character.
+ */
+ add_title(code);
+ else if (cases[1])
+ /*
+ * Add the lower and title case mappings for the upper case
+ * character.
+ */
+ add_upper(code);
+ else if (cases[0])
+ /*
+ * Add the upper and title case mappings for the lower case
+ * character.
+ */
+ add_lower(code);
+ }
+}
+
+static _decomp_t *
+#ifdef __STDC__
+find_decomp(unsigned long code)
+#else
+find_decomp(code)
+unsigned long code;
+#endif
+{
+ long l, r, m;
+
+ l = 0;
+ r = decomps_used - 1;
+ while (l <= r) {
+ m = (l + r) >> 1;
+ if (code > decomps[m].code)
+ l = m + 1;
+ else if (code < decomps[m].code)
+ r = m - 1;
+ else
+ return &decomps[m];
+ }
+ return 0;
+}
+
+static void
+#ifdef __STDC__
+decomp_it(_decomp_t *d)
+#else
+decomp_it(d)
+_decomp_t *d;
+#endif
+{
+ unsigned long i;
+ _decomp_t *dp;
+
+ for (i = 0; i < d->used; i++) {
+ if ((dp = find_decomp(d->decomp[i])) != 0)
+ decomp_it(dp);
+ else
+ dectmp[dectmp_size++] = d->decomp[i];
+ }
+}
+
+/*
+ * Expand all decompositions by recursively decomposing each character
+ * in the decomposition.
+ */
+static void
+#ifdef __STDC__
+expand_decomp(void)
+#else
+expand_decomp()
+#endif
+{
+ unsigned long i;
+
+ for (i = 0; i < decomps_used; i++) {
+ dectmp_size = 0;
+ decomp_it(&decomps[i]);
+ if (dectmp_size > 0)
+ add_decomp(decomps[i].code);
+ }
+}
+
+static void
+#ifdef __STDC__
+write_cdata(char *opath)
+#else
+write_cdata(opath)
+char *opath;
+#endif
+{
+ FILE *out;
+ unsigned long i, idx, bytes, nprops;
+ unsigned short casecnt[2];
+ char path[BUFSIZ];
+
+ /*****************************************************************
+ *
+ * Generate the ctype data.
+ *
+ *****************************************************************/
+
+ /*
+ * Open the ctype.dat file.
+ */
+ sprintf(path, "%s/ctype.dat", opath);
+ if ((out = fopen(path, "wb")) == 0)
+ return;
+
+ /*
+ * Collect the offsets for the properties. The offsets array is
+ * on a 4-byte boundary to keep things efficient for architectures
+ * that need such a thing.
+ */
+ for (i = idx = 0; i < NUMPROPS; i++) {
+ propcnt[i] = (proptbl[i].used != 0) ? idx : 0xffff;
+ idx += proptbl[i].used;
+ }
+
+ /*
+ * Add the sentinel index which is used by the binary search as the upper
+ * bound for a search.
+ */
+ propcnt[i] = idx;
+
+ /*
+ * Record the actual number of property lists. This may be different than
+ * the number of offsets actually written because of aligning on a 4-byte
+ * boundary.
+ */
+ hdr[1] = NUMPROPS;
+
+ /*
+ * Calculate the byte count needed and pad the property counts array to a
+ * 4-byte boundary.
+ */
+ if ((bytes = sizeof(unsigned short) * (NUMPROPS + 1)) & 3)
+ bytes += 4 - (bytes & 3);
+ nprops = bytes / sizeof(unsigned short);
+ bytes += sizeof(unsigned long) * idx;
+
+ /*
+ * Write the header.
+ */
+ fwrite((char *) hdr, sizeof(unsigned short), 2, out);
+
+ /*
+ * Write the byte count.
+ */
+ fwrite((char *) &bytes, sizeof(unsigned long), 1, out);
+
+ /*
+ * Write the property list counts.
+ */
+ fwrite((char *) propcnt, sizeof(unsigned short), nprops, out);
+
+ /*
+ * Write the property lists.
+ */
+ for (i = 0; i < NUMPROPS; i++) {
+ if (proptbl[i].used > 0)
+ fwrite((char *) proptbl[i].ranges, sizeof(unsigned long),
+ proptbl[i].used, out);
+ }
+
+ fclose(out);
+
+ /*****************************************************************
+ *
+ * Generate the case mapping data.
+ *
+ *****************************************************************/
+
+ /*
+ * Open the case.dat file.
+ */
+ sprintf(path, "%s/case.dat", opath);
+ if ((out = fopen(path, "wb")) == 0)
+ return;
+
+ /*
+ * Write the case mapping tables.
+ */
+ hdr[1] = upper_used + lower_used + title_used;
+ casecnt[0] = upper_used;
+ casecnt[1] = lower_used;
+
+ /*
+ * Write the header.
+ */
+ fwrite((char *) hdr, sizeof(unsigned short), 2, out);
+
+ /*
+ * Write the upper and lower case table sizes.
+ */
+ fwrite((char *) casecnt, sizeof(unsigned short), 2, out);
+
+ if (upper_used > 0)
+ /*
+ * Write the upper case table.
+ */
+ fwrite((char *) upper, sizeof(_case_t), upper_used, out);
+
+ if (lower_used > 0)
+ /*
+ * Write the lower case table.
+ */
+ fwrite((char *) lower, sizeof(_case_t), lower_used, out);
+
+ if (title_used > 0)
+ /*
+ * Write the title case table.
+ */
+ fwrite((char *) title, sizeof(_case_t), title_used, out);
+
+ fclose(out);
+
+ /*****************************************************************
+ *
+ * Generate the decomposition data.
+ *
+ *****************************************************************/
+
+ /*
+ * Fully expand all decompositions before generating the output file.
+ */
+ expand_decomp();
+
+ /*
+ * Open the decomp.dat file.
+ */
+ sprintf(path, "%s/decomp.dat", opath);
+ if ((out = fopen(path, "wb")) == 0)
+ return;
+
+ hdr[1] = decomps_used;
+
+ /*
+ * Write the header.
+ */
+ fwrite((char *) hdr, sizeof(unsigned short), 2, out);
+
+ /*
+ * Write a temporary byte count which will be calculated as the
+ * decompositions are written out.
+ */
+ bytes = 0;
+ fwrite((char *) &bytes, sizeof(unsigned long), 1, out);
+
+ if (decomps_used) {
+ /*
+ * Write the list of decomp nodes.
+ */
+ for (i = idx = 0; i < decomps_used; i++) {
+ fwrite((char *) &decomps[i].code, sizeof(unsigned long), 1, out);
+ fwrite((char *) &idx, sizeof(unsigned long), 1, out);
+ idx += decomps[i].used;
+ }
+
+ /*
+ * Write the sentinel index as the last decomp node.
+ */
+ fwrite((char *) &idx, sizeof(unsigned long), 1, out);
+
+ /*
+ * Write the decompositions themselves.
+ */
+ for (i = 0; i < decomps_used; i++)
+ fwrite((char *) decomps[i].decomp, sizeof(unsigned long),
+ decomps[i].used, out);
+
+ /*
+ * Seek back to the beginning and write the byte count.
+ */
+ bytes = (sizeof(unsigned long) * idx) +
+ (sizeof(unsigned long) * ((hdr[1] << 1) + 1));
+ fseek(out, sizeof(unsigned short) << 1, 0L);
+ fwrite((char *) &bytes, sizeof(unsigned long), 1, out);
+
+ fclose(out);
+ }
+
+ /*****************************************************************
+ *
+ * Generate the combining class data.
+ *
+ *****************************************************************/
+
+ /*
+ * Open the cmbcl.dat file.
+ */
+ sprintf(path, "%s/cmbcl.dat", opath);
+ if ((out = fopen(path, "wb")) == 0)
+ return;
+
+ /*
+ * Set the number of ranges used. Each range has a combining class which
+ * means each entry is a 3-tuple.
+ */
+ hdr[1] = ccl_used / 3;
+
+ /*
+ * Write the header.
+ */
+ fwrite((char *) hdr, sizeof(unsigned short), 2, out);
+
+ /*
+ * Write out the byte count to maintain header size.
+ */
+ bytes = ccl_used * sizeof(unsigned long);
+ fwrite((char *) &bytes, sizeof(unsigned long), 1, out);
+
+ if (ccl_used > 0)
+ /*
+ * Write the combining class ranges out.
+ */
+ fwrite((char *) ccl, sizeof(unsigned long), ccl_used, out);
+
+ fclose(out);
+
+ /*****************************************************************
+ *
+ * Generate the number data.
+ *
+ *****************************************************************/
+
+ /*
+ * Open the num.dat file.
+ */
+ sprintf(path, "%s/num.dat", opath);
+ if ((out = fopen(path, "wb")) == 0)
+ return;
+
+ /*
+ * The count part of the header will be the total number of codes that
+ * have numbers.
+ */
+ hdr[1] = (unsigned short) (ncodes_used << 1);
+ bytes = (ncodes_used * sizeof(_codeidx_t)) + (nums_used * sizeof(_num_t));
+
+ /*
+ * Write the header.
+ */
+ fwrite((char *) hdr, sizeof(unsigned short), 2, out);
+
+ /*
+ * Write out the byte count to maintain header size.
+ */
+ fwrite((char *) &bytes, sizeof(unsigned long), 1, out);
+
+ /*
+ * Now, if number mappings exist, write them out.
+ */
+ if (ncodes_used > 0) {
+ fwrite((char *) ncodes, sizeof(_codeidx_t), ncodes_used, out);
+ fwrite((char *) nums, sizeof(_num_t), nums_used, out);
+ }
+
+ fclose(out);
+}
+
+void
+#ifdef __STDC__
+main(int argc, char *argv[])
+#else
+main(argc, argv)
+int argc;
+char *argv[];
+#endif
+{
+ FILE *in;
+ char *prog, *opath;
+
+ if ((prog = strrchr(argv[0], '/')) != 0)
+ prog++;
+ else
+ prog = argv[0];
+
+ opath = 0;
+ in = stdin;
+
+ argc--;
+ argv++;
+
+ while (argc > 0) {
+ if (argv[0][0] == '-' && argv[0][1] == 'o') {
+ argc--;
+ argv++;
+ opath = argv[0];
+ } else {
+ if (in != stdin)
+ fclose(in);
+ if ((in = fopen(argv[0], "rb")) == 0)
+ fprintf(stderr, "%s: unable to open ctype file %s\n",
+ prog, argv[0]);
+ else {
+ read_cdata(in);
+ fclose(in);
+ in = 0;
+ }
+ }
+ argc--;
+ argv++;
+ }
+
+ if (opath == 0)
+ opath = ".";
+ write_cdata(opath);
+
+ exit(0);
+}
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-10 13:26:48 +0000