1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
|
#!/usr/bin/env python
#
# Based on src/etc/unicode.py from Rust 1.2.0.
#
# Copyright 2011-2013 The Rust Project Developers.
# Copyright 2015 The Servo Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution and at
# http://rust-lang.org/COPYRIGHT.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.
import fileinput, re, os, sys, operator
preamble = '''// NOTE:
// The following code was generated by "tools/generate.py". do not edit directly
#![allow(missing_docs, non_upper_case_globals, non_snake_case)]
'''
# these are the surrogate codepoints, which are not valid rust characters
surrogate_codepoints = (0xd800, 0xdfff)
def fetch(f):
if not os.path.exists(os.path.basename(f)):
os.system("curl -O http://www.unicode.org/Public/UNIDATA/%s"
% f)
if not os.path.exists(os.path.basename(f)):
sys.stderr.write("cannot load %s" % f)
exit(1)
def is_surrogate(n):
return surrogate_codepoints[0] <= n <= surrogate_codepoints[1]
def load_unicode_data(f):
fetch(f)
udict = {};
range_start = -1;
for line in fileinput.input(f):
data = line.split(';');
if len(data) != 15:
continue
cp = int(data[0], 16);
if is_surrogate(cp):
continue
if range_start >= 0:
for i in xrange(range_start, cp):
udict[i] = data;
range_start = -1;
if data[1].endswith(", First>"):
range_start = cp;
continue;
udict[cp] = data;
# Mapping of code point to Bidi_Class property:
bidi_class = {}
for code in udict:
[code_org, name, gencat, combine, bidi,
decomp, deci, digit, num, mirror,
old, iso, upcase, lowcase, titlecase ] = udict[code];
if bidi not in bidi_class:
bidi_class[bidi] = []
bidi_class[bidi].append(code)
# Default Bidi_Class for unassigned codepoints.
# http://www.unicode.org/Public/UNIDATA/extracted/DerivedBidiClass.txt
default_ranges = [
(0x0600, 0x07BF, "AL"), (0x08A0, 0x08FF, "AL"),
(0xFB50, 0xFDCF, "AL"), (0xFDF0, 0xFDFF, "AL"),
(0xFE70, 0xFEFF, "AL"), (0x1EE00, 0x0001EEFF, "AL"),
(0x0590, 0x05FF, "R"), (0x07C0, 0x089F, "R"),
(0xFB1D, 0xFB4F, "R"), (0x00010800, 0x00010FFF, "R"),
(0x0001E800, 0x0001EDFF, "R"), (0x0001EF00, 0x0001EFFF, "R"),
(0x20A0, 0x20CF, "ET")]
for (start, end, default) in default_ranges:
for code in range(start, end+1):
if not code in udict:
bidi_class[default].append(code)
bidi_class = group_cats(bidi_class)
return bidi_class
def group_cats(cats):
cats_out = []
for cat in cats:
cats_out.extend([(x, y, cat) for (x, y) in group_cat(cats[cat])])
cats_out.sort(key=lambda w: w[0])
return (sorted(cats.keys()), cats_out)
def group_cat(cat):
cat_out = []
letters = sorted(set(cat))
cur_start = letters.pop(0)
cur_end = cur_start
for letter in letters:
assert letter > cur_end, \
"cur_end: %s, letter: %s" % (hex(cur_end), hex(letter))
if letter == cur_end + 1:
cur_end = letter
else:
cat_out.append((cur_start, cur_end))
cur_start = cur_end = letter
cat_out.append((cur_start, cur_end))
return cat_out
def format_table_content(f, content, indent):
line = " "*indent
first = True
for chunk in content.split(","):
if len(line) + len(chunk) < 98:
if first:
line += chunk
else:
line += ", " + chunk
first = False
else:
f.write(line + ",\n")
line = " "*indent + chunk
f.write(line)
def escape_char(c):
return "'\\u{%x}'" % c
def emit_table(f, name, t_data, t_type = "&'static [(char, char)]", is_pub=True,
pfun=lambda x: "(%s,%s)" % (escape_char(x[0]), escape_char(x[1]))):
pub_string = ""
if is_pub:
pub_string = "pub "
f.write(" %sconst %s: %s = &[\n" % (pub_string, name, t_type))
data = ""
first = True
for dat in t_data:
if not first:
data += ","
first = False
data += pfun(dat)
format_table_content(f, data, 8)
f.write("\n ];\n\n")
def emit_bidi_module(f, bidi_class, cats):
f.write("""pub use self::BidiClass::*;
#[allow(non_camel_case_types)]
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
/// Represents the Unicode character property **Bidi_Class**, also known as
/// the *bidirectional character type*.
///
/// Use the `bidi_class` function to look up the BidiClass of a code point.
///
/// http://www.unicode.org/reports/tr9/#Bidirectional_Character_Types
pub enum BidiClass {
""")
for cat in cats:
f.write(" " + cat + ",\n")
f.write(""" }
fn bsearch_range_value_table(c: char, r: &'static [(char, char, BidiClass)]) -> BidiClass {
use ::std::cmp::Ordering::{Equal, Less, Greater};
match r.binary_search_by(|&(lo, hi, _)| {
if lo <= c && c <= hi { Equal }
else if hi < c { Less }
else { Greater }
}) {
Ok(idx) => {
let (_, _, cat) = r[idx];
cat
}
// UCD/extracted/DerivedBidiClass.txt: "All code points not explicitly listed
// for Bidi_Class have the value Left_To_Right (L)."
Err(_) => L
}
}
/// Find the BidiClass of a single char.
pub fn bidi_class(c: char) -> BidiClass {
bsearch_range_value_table(c, bidi_class_table)
}
""")
emit_table(f, "bidi_class_table", bidi_class, "&'static [(char, char, BidiClass)]",
pfun=lambda x: "(%s,%s,%s)" % (escape_char(x[0]), escape_char(x[1]), x[2]),
is_pub=False)
if __name__ == "__main__":
os.chdir("../src/") # changing download path to /unicode-bidi/src/
r = "tables.rs"
# downloading the test case files
fetch("BidiTest.txt")
fetch("BidiCharacterTest.txt")
if os.path.exists(r):
os.remove(r)
with open(r, "w") as rf:
# write the file's preamble
rf.write(preamble)
# download and parse all the data
fetch("ReadMe.txt")
with open("ReadMe.txt") as readme:
pattern = "for Version (\d+)\.(\d+)\.(\d+) of the Unicode"
unicode_version = re.search(pattern, readme.read()).groups()
rf.write("""
/// The version of [Unicode](http://www.unicode.org/)
/// that the `bidi_class` function is based on.
pub const UNICODE_VERSION: (u64, u64, u64) = (%s, %s, %s);
""" % unicode_version)
(bidi_cats, bidi_class) = load_unicode_data("UnicodeData.txt")
emit_bidi_module(rf, bidi_class, bidi_cats)
|
