1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
|
// Copyright 2013 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// File: lang_script.h
// ================
//
// Author: dsites@google.com (Dick Sites)
//
// This file declares language and script numbers and names for CLD2,
// plus routines that access side tables based on these
//
#ifndef I18N_ENCODINGS_CLD2_LANG_SCRIPT_H__
#define I18N_ENCODINGS_CLD2_LANG_SCRIPT_H__
#include "generated_language.h"
#include "generated_ulscript.h"
#include "integral_types.h"
// NOTE: The script numbers and language numbers here are not guaranteed to be
// stable. If you want to record a result for posterity, save the
// ULScriptCode(ULScript ulscript) result as character strings.
//
// The Unicode scripts recognized by CLD2 are numbered almost arbitrarily,
// specified in an enum. Each script has human-readable script name and a
// 4-letter ISO 15924 script code. Each has a C name (largely for use by
// programs that generate declarations in cld2_generated_scripts.h). Each
// also has a recognition type
// r_type: 0 script-only, 1 nilgrams, 2 quadgrams, 3 CJK
//
// The declarations for a particular version of Unicode are machine-generated in
// generated_scripts.h
//
// This file includes that one and declares the access routines. The type
// involved is called "ULScript" to signify Unicode Letters-Marks Scripts,
// which are not quite Unicode Scripts. In particular, the CJK scripts are
// merged into a single number because CLD2 recognizes the CJK languages from
// four scripts intermixed: Hani (both Hans and Hant), Hangul, Hiragana, and
// Katakana.
// Each script has one of these four recognition types.
// RTypeNone: There is no language associated with this script. In extended
// language recognition calls, return a fake language number that maps to
// xx-Cham, with literally "xx" for the language code,and with the script
// code instead of "Cham". In non-extended calls, return UNKNOWN_LANGUAGE.
// RTypeOne: The script maps 1:1 to a single language. No letters are examined
// during recognition and no lookups done.
// RTypeMany: The usual quadgram + delta-octagram + distinctive-words scoring
// is done to determine the languages involved.
// RTypeCJK: The CJK unigram + delta-bigram scoring is done to determine the
// languages involved.
//
// Note that the choice of recognition type is a function of script, not
// language. In particular, some languges are recognized in multiple scripts
// and those have different recognition types (Mongolian mn-Latn vs. mn-Mong
// for example).
namespace CLD2 {
//----------------------------------------------------------------------------//
// Functions of ULScript //
//----------------------------------------------------------------------------//
// If the input is out of range or otherwise unrecognized, it is treated
// as ULScript_Common (which never participates in language recognition)
const char* ULScriptName(ULScript ulscript);
const char* ULScriptCode(ULScript ulscript);
const char* ULScriptDeclaredName(ULScript ulscript);
ULScriptRType ULScriptRecognitionType(ULScript ulscript);
// Name can be either full name or ISO code, or can be ISO code embedded in
// a language-script combination such as "en-Latn-GB"
ULScript GetULScriptFromName(const char* src);
// Map script into Latin, Cyrillic, Arabic, Other
int LScript4(ULScript ulscript);
//----------------------------------------------------------------------------//
// Functions of Language //
//----------------------------------------------------------------------------//
// The languages recognized by CLD2 are numbered almost arbitrarily,
// specified in an enum. Each language has human-readable language name and a
// 2- or 3-letter ISO 639 language code. Each has a C name (largely for use by
// programs that generate declarations in cld2_generated_languagess.h).
// Each has a list of up to four scripts in which it is currently recognized.
//
// The declarations for a particular set of recognized languages are
// machine-generated in
// generated_languages.h
//
// The Language enum is intended to match the internal Google Language enum
// in i18n/languages/proto/languages.proto up to NUM_LANGUAGES, with additional
// languages assigned above that. Over time, some languages may be renumbered
// if they are moved into the Language enum.
//
// The Language enum includes the fake language numbers for RTypeNone above.
//
// If the input is out of range or otherwise unrecognized, it is treated
// as UNKNOWN_LANGUAGE
//
// LanguageCode
// ------------
// Given the Language, return the language code, e.g. "ko"
// This is determined by
// the following (in order of preference):
// - ISO-639-1 two-letter language code
// (all except those mentioned below)
// - ISO-639-2 three-letter bibliographic language code
// (Tibetan, Dhivehi, Cherokee, Syriac)
// - Google-specific language code
// (ChineseT ("zh-TW"), Teragram Unknown, Unknown,
// Portuguese-Portugal, Portuguese-Brazil, Limbu)
// - Fake RTypeNone names.
const char* LanguageName(Language lang);
const char* LanguageCode(Language lang);
const char* LanguageShortCode(Language lang);
const char* LanguageDeclaredName(Language lang);
// n is in 0..3. Trailing entries are filled with
// ULScript_Common (which never participates in language recognition)
ULScript LanguageRecognizedScript(Language lang, int n);
// Name can be either full name or ISO code, or can be ISO code embedded in
// a language-script combination such as "en-Latn-GB"
Language GetLanguageFromName(const char* src);
// Returns which set of statistically-close languages lang is in. 0 means none.
int LanguageCloseSet(Language lang);
//----------------------------------------------------------------------------//
// Functions of ULScript and Language //
//----------------------------------------------------------------------------//
// Most common language in each script
Language DefaultLanguage(ULScript ulscript);
// For RTypeMany recognition,
// the CLD2 lookup tables are kept small by encoding a language into one byte.
// To avoid limiting CLD2 to at most 256 languages, a larger range of external
// Language numbers is mapped to a smaller range of per-script numbers. At
// the moment (January 2013) the Latin script has about 90 languages to be
// recognized, while all the other scripts total about 50 more languages. In
// addition, the RTypeNone scripts map to about 100 fake languages.
// So we map all Latin-script languages to one range of 1..255 per-script
// numbers and map all the other RTypeMany languages to an overlapping range
// 1..255 of per-script numbers.
uint8 PerScriptNumber(ULScript ulscript, Language lang);
Language FromPerScriptNumber(ULScript ulscript, uint8 perscript_number);
// While the speed-sensitive processing deals with per-script language numbers,
// there is a need for low-performance dealing with original language numbers
// and unknown scripts, mostly for processing language hints.
// These routines let one derive a script class from a bare language.
// For languages written in multiple scripts, both of these can return true.
bool IsLatnLanguage(Language lang);
bool IsOthrLanguage(Language lang);
//----------------------------------------------------------------------------//
// Other //
//----------------------------------------------------------------------------//
// Utility routine to search alphabetical tables
int BinarySearch(const char* key, int lo, int hi, const CharIntPair* cipair);
} // namespace CLD2
#endif // I18N_ENCODINGS_CLD2_LANG_SCRIPT_H__
|
