Add m-esr52 at 52.6.0

1 files changed, 943 insertions, 0 deletions

diff --git a/intl/unicharutil/tools/genUnicodePropertyData.pl b/intl/unicharutil/tools/genUnicodePropertyData.pl
new file mode 100755
index 000000000..8b247e83c
--- /dev/null
+++ b/intl/unicharutil/tools/genUnicodePropertyData.pl
@@ -0,0 +1,943 @@
+#!/usr/bin/env perl
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This tool is used to prepare lookup tables of Unicode character properties
+# needed by gfx code to support text shaping operations. The properties are
+# read from the Unicode Character Database and compiled into multi-level arrays
+# for efficient lookup.
+#
+# To regenerate the tables in nsUnicodePropertyData.cpp:
+#
+# (1) Download the current Unicode data files from
+#
+#         http://www.unicode.org/Public/UNIDATA/
+#
+#     NB: not all the files are actually needed; currently, we require
+#       - UnicodeData.txt
+#       - Scripts.txt
+#       - BidiMirroring.txt
+#       - BidiBrackets.txt
+#       - HangulSyllableType.txt
+#       - LineBreak.txt
+#       - EastAsianWidth.txt
+#       - ReadMe.txt (to record version/date of the UCD)
+#       - Unihan_Variants.txt (from Unihan.zip)
+#     though this may change if we find a need for additional properties.
+#
+#     The Unicode data files listed above should be together in one directory.
+#
+#     We also require the file
+#        http://www.unicode.org/Public/security/latest/xidmodifications.txt
+#     This file should be in a sub-directory "security" immediately below the
+#        directory containing the other Unicode data files.
+#
+#     We also require the latest data file for UTR50, currently revision-13:
+#        http://www.unicode.org/Public/vertical/revision-13/VerticalOrientation-13.txt
+#     This file should be in a sub-directory "vertical" immediately below the
+#        directory containing the other Unicode data files.
+#
+#
+# (2) Run this tool using a command line of the form
+#
+#         perl genUnicodePropertyData.pl      \
+#                 /path/to/harfbuzz/src       \
+#                 /path/to/icu/common/unicode \
+#                 /path/to/UCD-directory
+#
+#     This will generate (or overwrite!) the files
+#
+#         nsUnicodePropertyData.cpp
+#         nsUnicodeScriptCodes.h
+#
+#     in the current directory.
+
+use strict;
+use List::Util qw(first);
+
+if ($#ARGV != 2) {
+    print <<__EOT;
+# Run this tool using a command line of the form
+#
+#     perl genUnicodePropertyData.pl      \\
+#             /path/to/harfbuzz/src       \\
+#             /path/to/icu/common/unicode \\
+#             /path/to/UCD-directory
+#
+# where harfbuzz/src is the directory containing harfbuzz .cc and .hh files,
+# icu/common/unicode is the directory containing ICU 'common' public headers,
+# and UCD-directory is a directory containing the current Unicode Character
+# Database files (UnicodeData.txt, etc), available from
+# http://www.unicode.org/Public/UNIDATA/, with additional resources as
+# detailed in the source comments.
+#
+# This will generate (or overwrite!) the files
+#
+#     nsUnicodePropertyData.cpp
+#     nsUnicodeScriptCodes.h
+#
+# in the current directory.
+__EOT
+    exit 0;
+}
+
+my $HARFBUZZ = $ARGV[0];
+my $ICU = $ARGV[1];
+my $UNICODE = $ARGV[2];
+
+# load HB_Category constants
+
+my $cc = -1;
+my %catCode;
+
+sub readHarfBuzzHeader
+{
+    my $file = shift;
+    open FH, "< $HARFBUZZ/$file" or die "can't open harfbuzz header $HARFBUZZ/$file\n";
+    while (<FH>) {
+        if (m/HB_UNICODE_GENERAL_CATEGORY_([A-Z_]+)/) {
+            $cc++;
+            $catCode{$1} = $cc;
+        }
+    }
+    close FH;
+}
+
+&readHarfBuzzHeader("hb-unicode.h");
+
+die "didn't find HarfBuzz category codes\n" if $cc == -1;
+
+my %scriptCode;
+my @scriptCodeToTag;
+my @scriptCodeToName;
+
+my $sc = -1;
+
+sub readIcuHeader
+{
+    my $file = shift;
+    open FH, "< $ICU/$file" or die "can't open ICU header $ICU/$file\n";
+    while (<FH>) {
+        # adjust for ICU vs UCD naming discrepancies
+        s/LANNA/TAI_THAM/;
+        s/MEITEI_MAYEK/MEETEI_MAYEK/;
+        s/ORKHON/OLD_TURKIC/;
+        s/MENDE/MENDE_KIKAKUI/;
+        s/SIGN_WRITING/SIGNWRITING/;
+        if (m|USCRIPT_([A-Z_]+)\s*=\s*([0-9]+),\s*/\*\s*([A-Z][a-z]{3})\s*\*/|) {
+            $sc = $2;
+            $scriptCode{$1} = $sc;
+            $scriptCodeToTag[$sc] = $3;
+            $scriptCodeToName[$sc] = $1;
+        }
+    }
+    close FH;
+}
+
+&readIcuHeader("uscript.h");
+
+die "didn't find ICU script codes\n" if $sc == -1;
+
+my %xidmodCode = (
+'Recommended'       => 0,
+'Inclusion'         => 1,
+'Uncommon_Use'      => 2,
+'Technical'         => 3,
+'Obsolete'          => 4,
+'Aspirational'      => 5,
+'Limited_Use'       => 6,
+'Exclusion'         => 7,
+'Not_XID'           => 8,
+'Not_NFKC'          => 9,
+'Default_Ignorable' => 10,
+'Deprecated'        => 11,
+'not-chars'         => 12
+);
+
+my %bidicategoryCode = (
+  "L"   =>  0, # Left-to-Right
+  "R"   =>  1, # Right-to-Left
+  "EN"  =>  2, # European Number
+  "ES"  =>  3, # European Number Separator
+  "ET"  =>  4, # European Number Terminator
+  "AN"  =>  5, # Arabic Number
+  "CS"  =>  6, # Common Number Separator
+  "B"   =>  7, # Paragraph Separator
+  "S"   =>  8, # Segment Separator
+  "WS"  =>  9, # Whitespace
+  "ON"  => 10, # Other Neutrals
+  "LRE" => 11, # Left-to-Right Embedding
+  "LRO" => 12, # Left-to-Right Override
+  "AL"  => 13, # Right-to-Left Arabic
+  "RLE" => 14, # Right-to-Left Embedding
+  "RLO" => 15, # Right-to-Left Override
+  "PDF" => 16, # Pop Directional Format
+  "NSM" => 17, # Non-Spacing Mark
+  "BN"  => 18, # Boundary Neutral
+  "FSI" => 19, # First Strong Isolate
+  "LRI" => 20, # Left-to-Right Isolate
+  "RLI" => 21, # Right-to-left Isolate
+  "PDI" => 22  # Pop Direcitonal Isolate
+);
+
+my %verticalOrientationCode = (
+  'U' => 0,  #   U - Upright, the same orientation as in the code charts
+  'R' => 1,  #   R - Rotated 90 degrees clockwise compared to the code charts
+  'Tu' => 2, #   Tu - Transformed typographically, with fallback to Upright
+  'Tr' => 3  #   Tr - Transformed typographically, with fallback to Rotated
+);
+
+my %lineBreakCode = ( # ordering matches ICU's ULineBreak enum
+  "XX" => 0,
+  "AI" => 1,
+  "AL" => 2,
+  "B2" => 3,
+  "BA" => 4,
+  "BB" => 5,
+  "BK" => 6,
+  "CB" => 7,
+  "CL" => 8,
+  "CM" => 9,
+  "CR" => 10,
+  "EX" => 11,
+  "GL" => 12,
+  "HY" => 13,
+  "ID" => 14,
+  "IN" => 15,
+  "IS" => 16,
+  "LF" => 17,
+  "NS" => 18,
+  "NU" => 19,
+  "OP" => 20,
+  "PO" => 21,
+  "PR" => 22,
+  "QU" => 23,
+  "SA" => 24,
+  "SG" => 25,
+  "SP" => 26,
+  "SY" => 27,
+  "ZW" => 28,
+  "NL" => 29,
+  "WJ" => 30,
+  "H2" => 31,
+  "H3" => 32,
+  "JL" => 33,
+  "JT" => 34,
+  "JV" => 35,
+  "CP" => 36,
+  "CJ" => 37,
+  "HL" => 38,
+  "RI" => 39
+);
+
+my %eastAsianWidthCode = (
+  "N" => 0,
+  "A" => 1,
+  "H" => 2,
+  "W" => 3,
+  "F" => 4,
+  "Na" => 5
+);
+
+# initialize default properties
+my @script;
+my @category;
+my @combining;
+my @mirror;
+my @pairedBracketType;
+my @hangul;
+my @casemap;
+my @xidmod;
+my @numericvalue;
+my @hanVariant;
+my @bidicategory;
+my @fullWidth;
+my @fullWidthInverse;
+my @verticalOrientation;
+my @lineBreak;
+my @eastAsianWidthFWH;
+for (my $i = 0; $i < 0x110000; ++$i) {
+    $script[$i] = $scriptCode{"UNKNOWN"};
+    $category[$i] = $catCode{"UNASSIGNED"};
+    $combining[$i] = 0;
+    $pairedBracketType[$i] = 0;
+    $casemap[$i] = 0;
+    $xidmod[$i] = $xidmodCode{"not-chars"};
+    $numericvalue[$i] = -1;
+    $hanVariant[$i] = 0;
+    $bidicategory[$i] = $bidicategoryCode{"L"};
+    $fullWidth[$i] = 0;
+    $fullWidthInverse[$i] = 0;
+    $verticalOrientation[$i] = 1; # default for unlisted codepoints is 'R'
+    $lineBreak[$i] = $lineBreakCode{"XX"};
+    $eastAsianWidthFWH[$i] = 0;
+}
+
+# blocks where the default for bidi category is not L
+for my $i (0x0600..0x07BF, 0x08A0..0x08FF, 0xFB50..0xFDCF, 0xFDF0..0xFDFF, 0xFE70..0xFEFF, 0x1EE00..0x0001EEFF) {
+  $bidicategory[$i] = $bidicategoryCode{"AL"};
+}
+for my $i (0x0590..0x05FF, 0x07C0..0x089F, 0xFB1D..0xFB4F, 0x00010800..0x00010FFF, 0x0001E800..0x0001EDFF, 0x0001EF00..0x0001EFFF) {
+  $bidicategory[$i] = $bidicategoryCode{"R"};
+}
+for my $i (0x20A0..0x20CF) {
+  $bidicategory[$i] = $bidicategoryCode{"ET"};
+}
+
+my %ucd2hb = (
+'Cc' => 'CONTROL',
+'Cf' => 'FORMAT',
+'Cn' => 'UNASSIGNED',
+'Co' => 'PRIVATE_USE',
+'Cs' => 'SURROGATE',
+'Ll' => 'LOWERCASE_LETTER',
+'Lm' => 'MODIFIER_LETTER',
+'Lo' => 'OTHER_LETTER',
+'Lt' => 'TITLECASE_LETTER',
+'Lu' => 'UPPERCASE_LETTER',
+'Mc' => 'SPACING_MARK',
+'Me' => 'ENCLOSING_MARK',
+'Mn' => 'NON_SPACING_MARK',
+'Nd' => 'DECIMAL_NUMBER',
+'Nl' => 'LETTER_NUMBER',
+'No' => 'OTHER_NUMBER',
+'Pc' => 'CONNECT_PUNCTUATION',
+'Pd' => 'DASH_PUNCTUATION',
+'Pe' => 'CLOSE_PUNCTUATION',
+'Pf' => 'FINAL_PUNCTUATION',
+'Pi' => 'INITIAL_PUNCTUATION',
+'Po' => 'OTHER_PUNCTUATION',
+'Ps' => 'OPEN_PUNCTUATION',
+'Sc' => 'CURRENCY_SYMBOL',
+'Sk' => 'MODIFIER_SYMBOL',
+'Sm' => 'MATH_SYMBOL',
+'So' => 'OTHER_SYMBOL',
+'Zl' => 'LINE_SEPARATOR',
+'Zp' => 'PARAGRAPH_SEPARATOR',
+'Zs' => 'SPACE_SEPARATOR'
+);
+
+# read ReadMe.txt
+my @versionInfo;
+open FH, "< $UNICODE/ReadMe.txt" or die "can't open Unicode ReadMe.txt file\n";
+while (<FH>) {
+    chomp;
+    push @versionInfo, $_;
+}
+close FH;
+
+my $kTitleToUpper = 0x80000000;
+my $kUpperToLower = 0x40000000;
+my $kLowerToTitle = 0x20000000;
+my $kLowerToUpper = 0x10000000;
+my $kCaseMapCharMask = 0x001fffff;
+
+# read UnicodeData.txt
+open FH, "< $UNICODE/UnicodeData.txt" or die "can't open UCD file UnicodeData.txt\n";
+while (<FH>) {
+    chomp;
+    my @fields = split /;/;
+    if ($fields[1] =~ /First/) {
+        my $first = hex "0x$fields[0]";
+        $_ = <FH>;
+        @fields = split /;/;
+        if ($fields[1] =~ /Last/) {
+            my $last = hex "0x$fields[0]";
+            do {
+                $category[$first] = $catCode{$ucd2hb{$fields[2]}};
+                $combining[$first] = $fields[3];
+                $bidicategory[$first] = $bidicategoryCode{$fields[4]};
+                unless (length($fields[7]) == 0) {
+                  $numericvalue[$first] = $fields[7];
+                }
+                if ($fields[1] =~ /CJK/) {
+                  @hanVariant[$first] = 3;
+                }
+                $first++;
+            } while ($first <= $last);
+        } else {
+            die "didn't find Last code for range!\n";
+        }
+    } else {
+        my $usv = hex "0x$fields[0]";
+        $category[$usv] = $catCode{$ucd2hb{$fields[2]}};
+        $combining[$usv] = $fields[3];
+        my $upper = hex $fields[12];
+        my $lower = hex $fields[13];
+        my $title = hex $fields[14];
+        # we only store one mapping for each character,
+        # but also record what kind of mapping it is
+        if ($upper && $lower) {
+            $casemap[$usv] |= $kTitleToUpper;
+            $casemap[$usv] |= ($usv ^ $upper);
+        }
+        elsif ($lower) {
+            $casemap[$usv] |= $kUpperToLower;
+            $casemap[$usv] |= ($usv ^ $lower);
+        }
+        elsif ($title && ($title != $upper)) {
+            $casemap[$usv] |= $kLowerToTitle;
+            $casemap[$usv] |= ($usv ^ $title);
+        }
+        elsif ($upper) {
+            $casemap[$usv] |= $kLowerToUpper;
+            $casemap[$usv] |= ($usv ^ $upper);
+        }
+        $bidicategory[$usv] = $bidicategoryCode{$fields[4]};
+        unless (length($fields[7]) == 0) {
+          $numericvalue[$usv] = $fields[7];
+        }
+        if ($fields[1] =~ /CJK/) {
+          @hanVariant[$usv] = 3;
+        }
+        if ($fields[5] =~ /^<narrow>/) {
+          my $wideChar = hex(substr($fields[5], 9));
+          die "didn't expect supplementary-plane values here" if $usv > 0xffff || $wideChar > 0xffff;
+          $fullWidth[$usv] = $wideChar;
+          $fullWidthInverse[$wideChar] = $usv;
+        }
+        elsif ($fields[5] =~ /^<wide>/) {
+          my $narrowChar = hex(substr($fields[5], 7));
+          die "didn't expect supplementary-plane values here" if $usv > 0xffff || $narrowChar > 0xffff;
+          $fullWidth[$narrowChar] = $usv;
+          $fullWidthInverse[$usv] = $narrowChar;
+        }
+    }
+}
+close FH;
+
+# read Scripts.txt
+open FH, "< $UNICODE/Scripts.txt" or die "can't open UCD file Scripts.txt\n";
+push @versionInfo, "";
+while (<FH>) {
+    chomp;
+    push @versionInfo, $_;
+    last if /Date:/;
+}
+while (<FH>) {
+    if (m/([0-9A-F]{4,6})(?:\.\.([0-9A-F]{4,6}))*\s+;\s+([^ ]+)/) {
+        my $script = uc($3);
+        warn "unknown ICU script $script" unless exists $scriptCode{$script};
+        my $script = $scriptCode{$script};
+        my $start = hex "0x$1";
+        my $end = (defined $2) ? hex "0x$2" : $start;
+        for (my $i = $start; $i <= $end; ++$i) {
+            $script[$i] = $script;
+        }
+    }
+}
+close FH;
+
+# read BidiMirroring.txt
+my @offsets = ();
+push @offsets, 0;
+
+open FH, "< $UNICODE/BidiMirroring.txt" or die "can't open UCD file BidiMirroring.txt\n";
+push @versionInfo, "";
+while (<FH>) {
+    chomp;
+    push @versionInfo, $_;
+    last if /Date:/;
+}
+while (<FH>) {
+    s/#.*//;
+    if (m/([0-9A-F]{4,6});\s*([0-9A-F]{4,6})/) {
+        my $mirrorOffset = hex("0x$2") - hex("0x$1");
+        my $offsetIndex = first { $offsets[$_] eq $mirrorOffset } 0..$#offsets;
+        if ($offsetIndex == undef) {
+            die "too many offset codes\n" if scalar @offsets == 31;
+            push @offsets, $mirrorOffset;
+            $offsetIndex = $#offsets;
+        }
+        $mirror[hex "0x$1"] = $offsetIndex;
+    }
+}
+close FH;
+
+# read BidiBrackets.txt
+my %pairedBracketTypeCode = (
+  'N' => 0,
+  'O' => 1,
+  'C' => 2
+);
+open FH, "< $UNICODE/BidiBrackets.txt" or die "can't open UCD file BidiBrackets.txt\n";
+push @versionInfo, "";
+while (<FH>) {
+    chomp;
+    push @versionInfo, $_;
+    last if /Date:/;
+}
+while (<FH>) {
+    s/#.*//;
+    if (m/([0-9A-F]{4,6});\s*([0-9A-F]{4,6});\s*(.)/) {
+        my $mirroredChar = $offsets[$mirror[hex "0x$1"]] + hex "0x$1";
+        die "bidi bracket does not match mirrored char\n" unless $mirroredChar == hex "0x$2";
+        my $pbt = uc($3);
+        warn "unknown Bidi Bracket type" unless exists $pairedBracketTypeCode{$pbt};
+        $pairedBracketType[hex "0x$1"] = $pairedBracketTypeCode{$pbt};
+    }
+}
+close FH;
+
+# read HangulSyllableType.txt
+my %hangulType = (
+  'L'   => 0x01,
+  'V'   => 0x02,
+  'T'   => 0x04,
+  'LV'  => 0x03,
+  'LVT' => 0x07
+);
+open FH, "< $UNICODE/HangulSyllableType.txt" or die "can't open UCD file HangulSyllableType.txt\n";
+push @versionInfo, "";
+while (<FH>) {
+    chomp;
+    push @versionInfo, $_;
+    last if /Date:/;
+}
+while (<FH>) {
+    s/#.*//;
+    if (m/([0-9A-F]{4,6})(?:\.\.([0-9A-F]{4,6}))*\s*;\s*([^ ]+)/) {
+        my $hangul = uc($3);
+        warn "unknown Hangul syllable type" unless exists $hangulType{$hangul};
+        $hangul = $hangulType{$hangul};
+        my $start = hex "0x$1";
+        my $end = (defined $2) ? hex "0x$2" : $start;
+        for (my $i = $start; $i <= $end; ++$i) {
+            $hangul[$i] = $hangul;
+        }
+    }
+}
+close FH;
+
+# read LineBreak.txt
+open FH, "< $UNICODE/LineBreak.txt" or die "can't open UCD file LineBreak.txt\n";
+push @versionInfo, "";
+while (<FH>) {
+    chomp;
+    push @versionInfo, $_;
+    last if /Date:/;
+}
+while (<FH>) {
+    s/#.*//;
+    if (m/([0-9A-F]{4,6})(?:\.\.([0-9A-F]{4,6}))*\s*;\s*([^ ]+)/) {
+        my $lb = uc($3);
+        warn "unknown LineBreak class" unless exists $lineBreakCode{$lb};
+        $lb = $lineBreakCode{$lb};
+        my $start = hex "0x$1";
+        my $end = (defined $2) ? hex "0x$2" : $start;
+        for (my $i = $start; $i <= $end; ++$i) {
+            $lineBreak[$i] = $lb;
+        }
+    }
+}
+close FH;
+
+# read EastAsianWidth.txt
+open FH, "< $UNICODE/EastAsianWidth.txt" or die "can't open UCD file EastAsianWidth.txt\n";
+push @versionInfo, "";
+while (<FH>) {
+    chomp;
+    push @versionInfo, $_;
+    last if /Date:/;
+}
+while (<FH>) {
+    s/#.*//;
+    if (m/([0-9A-F]{4,6})(?:\.\.([0-9A-F]{4,6}))*\s*;\s*([^ ]+)/) {
+        my $start = hex "0x$1";
+        my $end = (defined $2) ? hex "0x$2" : $start;
+        my $eaw = $3;
+        warn "unknown EastAsianWidth class" unless exists $eastAsianWidthCode{$eaw};
+        my $isFWH = ($eaw =~ m/^[FWH]$/) ? 1 : 0;
+        for (my $i = $start; $i <= $end; ++$i) {
+            $eastAsianWidthFWH[$i] = $isFWH;
+        }
+    }
+}
+close FH;
+
+# read xidmodifications.txt
+open FH, "< $UNICODE/security/xidmodifications.txt" or die "can't open UCD file xidmodifications.txt\n";
+push @versionInfo, "";
+while (<FH>) {
+  chomp;
+  unless (/\xef\xbb\xbf/) {
+    push @versionInfo, $_;
+  }
+  last if /Generated:/;
+}
+while (<FH>) {
+  if (m/([0-9A-F]{4,6})(?:\.\.([0-9A-F]{4,6}))*\s+;\s+[^ ]+\s+;\s+([^ ]+)/) {
+    my $xidmod = $3;
+    warn "unknown Identifier Modification $xidmod" unless exists $xidmodCode{$xidmod};
+    $xidmod = $xidmodCode{$xidmod};
+    my $start = hex "0x$1";
+    my $end = (defined $2) ? hex "0x$2" : $start;
+    for (my $i = $start; $i <= $end; ++$i) {
+      $xidmod[$i] = $xidmod;
+    }
+  }
+}
+close FH;
+
+open FH, "< $UNICODE/Unihan_Variants.txt" or die "can't open UCD file Unihan_Variants.txt (from Unihan.zip)\n";
+push @versionInfo, "";
+while (<FH>) {
+  chomp;
+  push @versionInfo, $_;
+  last if /Date:/;
+}
+my $savedusv = 0;
+my $hasTC = 0;
+my $hasSC = 0;
+while (<FH>) {
+  chomp;
+  if (m/U\+([0-9A-F]{4,6})\s+k([^ ]+)Variant/) {
+    my $usv = hex "0x$1";
+    if ($usv != $savedusv) {
+      unless ($savedusv == 0) {
+        if ($hasTC && !$hasSC) {
+          $hanVariant[$savedusv] = 1;
+        } elsif (!$hasTC && $hasSC) {
+          $hanVariant[$savedusv] = 2;
+        }
+      }
+      $savedusv = $usv;
+      $hasTC = 0;
+      $hasSC = 0;
+    }
+    if ($2 eq "Traditional") {
+      $hasTC = 1;
+    }
+    if ($2 eq "Simplified") {
+      $hasSC = 1;
+    }
+  } 
+}
+close FH;
+
+# read VerticalOrientation-13.txt
+open FH, "< $UNICODE/vertical/VerticalOrientation-13.txt" or die "can't open UTR50 data file VerticalOrientation-13.txt\n";
+push @versionInfo, "";
+while (<FH>) {
+    chomp;
+    push @versionInfo, $_;
+    last if /Date:/;
+}
+while (<FH>) {
+    chomp;
+    s/#.*//;
+    if (m/([0-9A-F]{4,6})(?:\.\.([0-9A-F]{4,6}))*\s*;\s*([^ ]+)/) {
+        my $vo = $3;
+        warn "unknown Vertical_Orientation code $vo"
+            unless exists $verticalOrientationCode{$vo};
+        $vo = $verticalOrientationCode{$vo};
+        my $start = hex "0x$1";
+        my $end = (defined $2) ? hex "0x$2" : $start;
+        for (my $i = $start; $i <= $end; ++$i) {
+            $verticalOrientation[$i] = $vo;
+        }
+    }
+}
+close FH;
+
+my $timestamp = gmtime();
+
+open DATA_TABLES, "> nsUnicodePropertyData.cpp" or die "unable to open nsUnicodePropertyData.cpp for output";
+
+my $licenseBlock = q[
+/* -*- Mode: C++; tab-width: 20; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * Derived from the Unicode Character Database by genUnicodePropertyData.pl
+ *
+ * For Unicode terms of use, see http://www.unicode.org/terms_of_use.html
+ */
+];
+
+my $versionInfo = join("\n", @versionInfo);
+
+print DATA_TABLES <<__END;
+$licenseBlock
+/*
+ * Created on $timestamp from UCD data files with version info:
+ *
+
+$versionInfo
+
+ *
+ * * * * * This file contains MACHINE-GENERATED DATA, do not edit! * * * * *
+ */
+
+#include <stdint.h>
+#include "harfbuzz/hb.h"
+
+__END
+
+open HEADER, "> nsUnicodeScriptCodes.h" or die "unable to open nsUnicodeScriptCodes.h for output";
+
+print HEADER <<__END;
+$licenseBlock
+/*
+ * Created on $timestamp from UCD data files with version info:
+ *
+
+$versionInfo
+
+ *
+ * * * * * This file contains MACHINE-GENERATED DATA, do not edit! * * * * *
+ */
+
+#ifndef NS_UNICODE_SCRIPT_CODES
+#define NS_UNICODE_SCRIPT_CODES
+
+__END
+
+print DATA_TABLES "#if !ENABLE_INTL_API\n";
+print DATA_TABLES "static const uint32_t sScriptCodeToTag[] = {\n";
+for (my $i = 0; $i < scalar @scriptCodeToTag; ++$i) {
+  printf DATA_TABLES "  HB_TAG('%c','%c','%c','%c')", unpack('cccc', $scriptCodeToTag[$i]);
+  print DATA_TABLES $i < $#scriptCodeToTag ? ",\n" : "\n";
+}
+print DATA_TABLES "};\n";
+print DATA_TABLES "#endif\n\n";
+
+our $totalData = 0;
+
+print DATA_TABLES "#if !ENABLE_INTL_API\n";
+print DATA_TABLES "static const int16_t sMirrorOffsets[] = {\n";
+for (my $i = 0; $i < scalar @offsets; ++$i) {
+    printf DATA_TABLES "  $offsets[$i]";
+    print DATA_TABLES $i < $#offsets ? ",\n" : "\n";
+}
+print DATA_TABLES "};\n";
+print DATA_TABLES "#endif\n\n";
+
+print HEADER "#pragma pack(1)\n\n";
+
+sub sprintCharProps1
+{
+  my $usv = shift;
+  return sprintf("{%d,%d,%d}, ", $mirror[$usv], $hangul[$usv], $combining[$usv]);
+}
+my $type = q/
+struct nsCharProps1 {
+  unsigned char mMirrorOffsetIndex:5;
+  unsigned char mHangulType:3;
+  unsigned char mCombiningClass:8;
+};
+/;
+&genTables("#if !ENABLE_INTL_API", "#endif",
+           "CharProp1", $type, "nsCharProps1", 11, 5, \&sprintCharProps1, 1, 2, 1);
+
+sub sprintCharProps2_short
+{
+  my $usv = shift;
+  return sprintf("{%d,%d},",
+                 $verticalOrientation[$usv], $xidmod[$usv]);
+}
+$type = q|
+struct nsCharProps2 {
+  // Currently only 6 bits are defined here, so 2 more could be added without
+  // affecting the storage requirements for this struct.
+  unsigned char mVertOrient:2;
+  unsigned char mXidmod:4;
+};
+|;
+&genTables("#if ENABLE_INTL_API", "#endif",
+           "CharProp2", $type, "nsCharProps2", 9, 7, \&sprintCharProps2_short, 16, 1, 1);
+
+sub sprintCharProps2_full
+{
+  my $usv = shift;
+  return sprintf("{%d,%d,%d,%d,%d,%d,%d,%d,%d},",
+                 $script[$usv], $pairedBracketType[$usv],
+                 $eastAsianWidthFWH[$usv], $category[$usv],
+                 $bidicategory[$usv], $xidmod[$usv], $numericvalue[$usv],
+                 $verticalOrientation[$usv], $lineBreak[$usv]);
+}
+$type = q|
+struct nsCharProps2 {
+  unsigned char mScriptCode:8;
+  unsigned char mPairedBracketType:2;
+  unsigned char mEastAsianWidthFWH:1;
+  unsigned char mCategory:5;
+  unsigned char mBidiCategory:5;
+  unsigned char mXidmod:4;
+  signed char   mNumericValue:5;
+  unsigned char mVertOrient:2;
+  unsigned char mLineBreak; // only 6 bits actually needed
+};
+|;
+&genTables("#if !ENABLE_INTL_API", "#endif",
+           "CharProp2", $type, "nsCharProps2", 12, 4, \&sprintCharProps2_full, 16, 5, 1);
+
+print HEADER "#pragma pack()\n\n";
+
+sub sprintHanVariants
+{
+  my $baseUsv = shift;
+  my $varShift = 0;
+  my $val = 0;
+  while ($varShift < 8) {
+    $val |= $hanVariant[$baseUsv++] << $varShift;
+    $varShift += 2;
+  }
+  return sprintf("0x%02x,", $val);
+}
+## Han Variant data currently unused but may be needed in future, see bug 857481
+## &genTables("", "", "HanVariant", "", "uint8_t", 9, 7, \&sprintHanVariants, 2, 1, 4);
+
+sub sprintFullWidth
+{
+  my $usv = shift;
+  return sprintf("0x%04x,", $fullWidth[$usv]);
+}
+&genTables("", "", "FullWidth", "", "uint16_t", 10, 6, \&sprintFullWidth, 0, 2, 1);
+
+sub sprintFullWidthInverse
+{
+  my $usv = shift;
+  return sprintf("0x%04x,", $fullWidthInverse[$usv]);
+}
+&genTables("", "", "FullWidthInverse", "", "uint16_t", 10, 6, \&sprintFullWidthInverse, 0, 2, 1);
+
+sub sprintCasemap
+{
+  my $usv = shift;
+  return sprintf("0x%08x,", $casemap[$usv]);
+}
+&genTables("#if !ENABLE_INTL_API", "#endif",
+           "CaseMap", "", "uint32_t", 11, 5, \&sprintCasemap, 1, 4, 1);
+
+print STDERR "Total data = $totalData\n";
+
+printf DATA_TABLES "const uint32_t kTitleToUpper = 0x%08x;\n", $kTitleToUpper;
+printf DATA_TABLES "const uint32_t kUpperToLower = 0x%08x;\n", $kUpperToLower;
+printf DATA_TABLES "const uint32_t kLowerToTitle = 0x%08x;\n", $kLowerToTitle;
+printf DATA_TABLES "const uint32_t kLowerToUpper = 0x%08x;\n", $kLowerToUpper;
+printf DATA_TABLES "const uint32_t kCaseMapCharMask = 0x%08x;\n\n", $kCaseMapCharMask;
+
+sub genTables
+{
+  my ($guardBegin, $guardEnd,
+      $prefix, $typedef, $type, $indexBits, $charBits, $func, $maxPlane, $bytesPerEntry, $charsPerEntry) = @_;
+
+  if ($typedef ne '') {
+    print HEADER "$guardBegin\n";
+    print HEADER "$typedef\n";
+    print HEADER "$guardEnd\n\n";
+  }
+
+  print DATA_TABLES "\n$guardBegin\n";
+  print DATA_TABLES "#define k${prefix}MaxPlane  $maxPlane\n";
+  print DATA_TABLES "#define k${prefix}IndexBits $indexBits\n";
+  print DATA_TABLES "#define k${prefix}CharBits  $charBits\n";
+
+  my $indexLen = 1 << $indexBits;
+  my $charsPerPage = 1 << $charBits;
+  my %charIndex = ();
+  my %pageMapIndex = ();
+  my @pageMap = ();
+  my @char = ();
+  
+  my $planeMap = "\x00" x $maxPlane;
+  foreach my $plane (0 .. $maxPlane) {
+    my $pageMap = "\x00" x $indexLen * 2;
+    foreach my $page (0 .. $indexLen - 1) {
+        my $charValues = "";
+        for (my $ch = 0; $ch < $charsPerPage; $ch += $charsPerEntry) {
+            my $usv = $plane * 0x10000 + $page * $charsPerPage + $ch;
+            $charValues .= &$func($usv);
+        }
+        chop $charValues;
+
+        unless (exists $charIndex{$charValues}) {
+            $charIndex{$charValues} = scalar keys %charIndex;
+            $char[$charIndex{$charValues}] = $charValues;
+        }
+        substr($pageMap, $page * 2, 2) = pack('S', $charIndex{$charValues});
+    }
+    
+    unless (exists $pageMapIndex{$pageMap}) {
+        $pageMapIndex{$pageMap} = scalar keys %pageMapIndex;
+        $pageMap[$pageMapIndex{$pageMap}] = $pageMap;
+    }
+    if ($plane > 0) {
+        substr($planeMap, $plane - 1, 1) = pack('C', $pageMapIndex{$pageMap});
+    }
+  }
+
+  if ($maxPlane) {
+    print DATA_TABLES "static const uint8_t s${prefix}Planes[$maxPlane] = {";
+    print DATA_TABLES join(',', map { sprintf("%d", $_) } unpack('C*', $planeMap));
+    print DATA_TABLES "};\n\n";
+  }
+
+  my $chCount = scalar @char;
+  my $pmBits = $chCount > 255 ? 16 : 8;
+  my $pmCount = scalar @pageMap;
+  if ($maxPlane == 0) {
+    die "there should only be one pageMap entry!" if $pmCount > 1;
+    print DATA_TABLES "static const uint${pmBits}_t s${prefix}Pages[$indexLen] = {\n";
+  } else {
+    print DATA_TABLES "static const uint${pmBits}_t s${prefix}Pages[$pmCount][$indexLen] = {\n";
+  }
+  for (my $i = 0; $i < scalar @pageMap; ++$i) {
+    print DATA_TABLES $maxPlane > 0 ? "  {" : "  ";
+    print DATA_TABLES join(',', map { sprintf("%d", $_) } unpack('S*', $pageMap[$i]));
+    print DATA_TABLES $maxPlane > 0 ? ($i < $#pageMap ? "},\n" : "}\n") : "\n";
+  }
+  print DATA_TABLES "};\n\n";
+
+  my $pageLen = $charsPerPage / $charsPerEntry;
+  print DATA_TABLES "static const $type s${prefix}Values[$chCount][$pageLen] = {\n";
+  for (my $i = 0; $i < scalar @char; ++$i) {
+    print DATA_TABLES "  {";
+    print DATA_TABLES $char[$i];
+    print DATA_TABLES $i < $#char ? "},\n" : "}\n";
+  }
+  print DATA_TABLES "};\n";
+  print DATA_TABLES "$guardEnd\n";
+
+  my $dataSize = $pmCount * $indexLen * $pmBits/8 +
+                 $chCount * $pageLen * $bytesPerEntry + 
+                 $maxPlane;
+  $totalData += $dataSize;
+
+  print STDERR "Data for $prefix = $dataSize\n";
+}
+
+print DATA_TABLES <<__END;
+/*
+ * * * * * This file contains MACHINE-GENERATED DATA, do not edit! * * * * *
+ */
+__END
+
+close DATA_TABLES;
+
+print HEADER "namespace mozilla {\n";
+print HEADER "namespace unicode {\n";
+print HEADER "enum class Script {\n";
+for (my $i = 0; $i < scalar @scriptCodeToName; ++$i) {
+  print HEADER "  ", $scriptCodeToName[$i], " = ", $i, ",\n";
+}
+print HEADER "\n  NUM_SCRIPT_CODES = ", scalar @scriptCodeToName, ",\n";
+print HEADER "\n  INVALID = -1\n";
+print HEADER "};\n";
+print HEADER "} // namespace unicode\n";
+print HEADER "} // namespace mozilla\n\n";
+
+print HEADER <<__END;
+#endif
+/*
+ * * * * * This file contains MACHINE-GENERATED DATA, do not edit! * * * * *
+ */
+__END
+
+close HEADER;
+