Add m-esr52 at 52.6.0

1 files changed, 228 insertions, 0 deletions

diff --git a/intl/uconv/tools/gengb18030tables.pl b/intl/uconv/tools/gengb18030tables.pl
new file mode 100644
index 000000000..3de1b2191
--- /dev/null
+++ b/intl/uconv/tools/gengb18030tables.pl
@@ -0,0 +1,228 @@
+#!/usr/local/bin/perl
+# -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+%gb18030tounicode = {};
+%unicodetogb18030 = {};
+%unicodetocp936 = {};
+%cp936tounicode = {};
+%tounicodecommon = {};
+%gb18030tounicodeuniq = {};
+%gb180304btounicode = {};
+%cp936tounicodeuniq = {};
+
+%map = {};
+$rowwidth = ((0xff - 0x80)+(0x7f - 0x40));
+sub cp936tonum()
+{
+   my($cp936) = (@_);
+   my($first,$second,$jnum);
+   $first = hex(substr($cp936,2,2));
+   $second = hex(substr($cp936,4,2));
+   $jnum = ($first - 0x81 ) * $rowwidth;
+   if($second >= 0x80)
+   {
+       $jnum += $second - 0x80 + (0x7f-0x40);
+   }
+   else
+   {
+       $jnum += $second - 0x40;
+   }
+   return $jnum;
+}
+sub addeudc()
+{
+  my($l,$h,$hl,$us);
+
+  $u = 0xE000;
+  $us = sprintf "%04X", $u;
+  # For AAA1-AFFE
+  for($h=0xAA; $h <=0xAF;$h++)
+  {
+    for($l=0xA1; $l <=0xFE;$l++,$u++)
+    {
+        $us = sprintf "%04X", $u;
+        $hl = sprintf "%02X%02X", $h, $l;
+        $unicodetocp936{$us} = $hl;
+    }
+  }
+
+  # For F8A1-FEFE
+  $us = sprintf "%04X", $u;
+  for($h=0xF8; $h <=0xFE;$h++)
+  {
+    for($l=0xA1; $l <=0xFE;$l++,$u++)
+    {
+        $us = sprintf "%04X", $u;
+        $hl = sprintf "%02X%02X", $h, $l;
+        $unicodetocp936{$us} = $hl;
+    }
+  }
+
+  # For A140-A7A0
+  $us = sprintf "%04X", $u;
+  for($h=0xA1; $h <=0xA7;$h++)
+  {
+    for($l=0x40; $l <=0x7E;$l++,$u++)
+    {
+        $us = sprintf "%04X", $u;
+        $hl = sprintf "%02X%02X", $h, $l;
+        $unicodetocp936{$us} = $hl;
+    }
+    # We need to skip 7F
+    for($l=0x80; $l <=0xA0;$l++,$u++)
+    {
+        $us = sprintf "%04X", $u;
+        $hl = sprintf "%02X%02X", $h, $l;
+        $unicodetocp936{$us} = $hl;
+    }
+  }
+}
+
+sub readcp936()
+{
+  open(CP936, "<CP936.txt") || die "Cannot open CP936 file";
+  while(<CP936>)
+  {
+    if(! /^#/) {
+      chop();
+      ($gb, $u) = split(/\t/, $_);
+      if($u =~ /^0x/) {
+        $u1 = substr($u, 2, 4);
+        $gb1 = substr($gb, 2, 4);
+        $cp936tounicode{$gb1} = $u1;
+        if($unicodetocp936{$u1} == "") {
+          $unicodetocp936{$u1} = $gb1;
+        } else {
+          "WARNING: Unicode " . $u1 . " already map to CP936 " . 
+            $unicodetocp936{$u1} . " when we try to map to " . $gb1 . "\n";
+        }
+
+      }
+    }
+  }
+}
+sub readgb18030()
+{
+  open(GB18030, "<GB18030") || die "Cannot open GB18030 file";
+  while(<GB18030>)
+  {
+    if(/^[0-9A-F]/) {
+      chop();
+      ($u, $gb) = split(/\s/, $_);
+      $gb18030tounicode{$gb} = $u;
+        if( $unicodetogb18030{$u} == "" ) {
+          $unicodetogb18030{$u} = $gb;
+        } else {
+          "WARNING: Unicode " . $u1 . " already map to CP936 " . 
+            $unicodetocp936{$u1} . " when we try to map to " . $gb1 . "\n";
+        }
+    }
+  }
+}
+sub splittable()
+{
+  my($i, $u);
+  for($i = 0; $i < 0x10000; $i++) {
+     $u = sprintf "%04X", $i;
+     if($unicodetogb18030{$u} eq $unicodetocp936{$u}) {
+        if($unicodetogb18030{$u} ne "") {
+          $tounicodecommon{$unicodetogb18030{$u}} = $u;
+        } else {
+#          print $u . "|" . $unicodetogb18030{$u} . "|" . $unicodetocp936{$u} . "\n";
+        }
+     } else {
+        if($unicodetogb18030{$u} ne "" ) {
+           if($unicodetogb18030{$u}.length > 4) {
+             $gb180304btounicode{$unicodetogb18030{$u}} = $u;
+           } else {
+             $gb18030tounicodeuniq{$unicodetogb18030{$u}} = $u;
+           }
+        } 
+        if($unicodetocp936{$u} ne "" ) {
+           $cp936tounicodeuniq{$unicodetocp936{$u}} = $u;
+        }
+     }
+  }
+}
+sub gb4bytestoidx()
+{
+  my($gb) = @_;
+  my($b1,$b2, $b3, $b4,$idx);
+  $b1 = hex(substr($gb, 0, 2)) - 0x81;
+  $b2 = hex(substr($gb, 2, 2)) - 0x30;
+  $b3 = hex(substr($gb, 4, 2)) - 0x81;
+  $b4 = hex(substr($gb, 6, 2)) - 0x30;
+  $idx = sprintf "%04X" , ((($b1 * 10) + $b2 ) * 126 + $b3) * 10 + $b4;
+  return $idx;
+}
+sub printcommontable()
+{
+  open ( GBKCOMMON, ">gbkcommon.txt" ) || die "cannot open gbkcommon.txt";
+  foreach $gb (sort(keys %tounicodecommon)) {
+      print GBKCOMMON "0x" . $gb . "\t0x" . $tounicodecommon{$gb} . "\n";
+  }
+  close GBKCOMMON;
+}
+sub printcp936table()
+{
+  open ( CP936UNIQ, ">cp936uniq.txt" ) || die "cannot open cp936uniq.txt";
+  foreach $gb (sort(keys %cp936tounicodeuniq)) {
+      print CP936UNIQ "0x" . $gb . "\t0x" . $cp936tounicodeuniq{$gb} . "\n";
+  }
+  close CP936UNIQ;
+}
+sub printgb180304btable()
+{
+  open ( GB180304B, ">gb180304b.txt" ) || die "cannot open gb180304b.txt";
+  foreach $gb (sort(keys %gb180304btounicode)) {
+      if($gb180304btounicode{$gb} ne "FFFF" ) {
+        print GB180304B "0x" . &gb4bytestoidx($gb) . "\t0x" . $gb180304btounicode{$gb} . "\t# 0x" . $gb . "\n";
+      }
+  }
+  close GB180304B;
+}
+sub printgb18030table()
+{
+  open ( GB18030UNIQ, ">gb18030uniq.txt" ) || die "cannot open gb18030uniq.txt";
+  foreach $gb (sort(keys %gb18030tounicodeuniq)) {
+      print GB18030UNIQ "0x" . $gb . "\t0x" . $gb18030tounicodeuniq{$gb} . "\n";
+  }
+  close GB18030UNIQ;
+}
+
+sub genufut()
+{
+ print ( "umaptable -uf < gb18030uniq.txt > gb18030uniq2b.uf\n");
+ system( "umaptable -uf < gb18030uniq.txt > gb18030uniq2b.uf");
+
+ print ( "umaptable -ut < gb18030uniq.txt > gb18030uniq2b.ut\n");
+ system( "umaptable -ut < gb18030uniq.txt > gb18030uniq2b.ut");
+
+ print ( "umaptable -uf < cp936uniq.txt > gbkuniq.uf\n") ;
+ system( "umaptable -uf < cp936uniq.txt > gbkuniq.uf") ;
+
+ print ( "umaptable -ut < cp936uniq.txt > gbkuniq.ut\n") ;
+ system( "umaptable -ut < cp936uniq.txt > gbkuniq.ut") ;
+
+ print ( "umaptable -uf < gb180304b.txt > gb180304bytes.uf\n")  ;
+ system( "umaptable -uf < gb180304b.txt > gb180304bytes.uf")  ;
+
+ print ( "umaptable -ut < gb180304b.txt > gb180304bytes.ut\n")  ;
+ system( "umaptable -ut < gb180304b.txt > gb180304bytes.ut")  ;
+
+ print ( "perl cp936tocdx.pl > cp936map.h\n");
+ system( "perl cp936tocdx.pl > cp936map.h");
+}
+
+&readgb18030();
+&readcp936();
+&addeudc();
+&splittable();
+&printcommontable();
+&printgb180304btable();
+&printgb18030table();
+&printcp936table();
+&genufut();