1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
|
// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef CLD2_INTERNAL_CLD2_DYNAMIC_DATA_H_
#define CLD2_INTERNAL_CLD2_DYNAMIC_DATA_H_
#include "integral_types.h"
#include "cld2tablesummary.h"
#include "utf8statetable.h"
#include "scoreonescriptspan.h"
/*
There are two primary parts to a CLD2 dynamic data file:
1. A header, wherein trivial data, block lengths and block offsets are kept
2. A data block, wherein the large binary blocks are kept
By reading the header, an application can determine the offsets and lengths of
all the data blocks for all tables. Offsets in the header are expressed
relative to the first byte of the file, inclusive of the header itself; thus,
any offset whose value is less than the length of the header is invalid.
Any offset whose value is zero indicates a field that is null in the
underlying CLD2 data; a real example of this is the fast_state field of the
UTF8PropObj, which may be null.
The size of the header can be precalculated by calling calculateHeaderSize(),
which will indicate the exact size of the header for a data file that contains
a given number of CLD2TableSummary objects.
Notes on endianness:
The data format is only suitable for little-endian machines. For big-endian
systems, a tedious transformation would need to be made first to reverse the
byte order of significant portions of the binary - not just the lengths, but
also some of the underlying table data.
Note on 32/64 bit:
The data format is agnostic to 32/64 bit pointers. All the offsets within the
data blob itself are 32-bit values relative to the start of the file, and the
file should certainly never be gigabytes in size!
When the file is ultimately read by the loading code and mmap()'d, new
pointers are generated at whatever size the system uses, initialized to the
start of the mmap, and incremented by the 32-bit offset. This should be safe
regardless of 32- or 64-bit architectures.
--------------------------------------------------------------------
FIELD
--------------------------------------------------------------------
DATA_FILE_MARKER (no null terminator)
total file size (sanity check, uint32)
--------------------------------------------------------------------
UTF8PropObj: const uint32 state0
UTF8PropObj: const uint32 state0_size
UTF8PropObj: const uint32 total_size
UTF8PropObj: const int max_expand
UTF8PropObj: const int entry_shift (coerced to 32 bits)
UTF8PropObj: const int bytes_per_entry (coerced to 32 bits)
UTF8PropObj: const uint32 losub
UTF8PropObj: const uint32 hiadd
offset of UTF8PropObj: const uint8* state_table
length of UTF8PropObj: const uint8* state_table
offset of UTF8PropObj: const RemapEntry* remap_base (4-byte struct)
length of UTF8PropObj: const RemapEntry* remap_base (4-byte struct)
offset of UTF8PropObj: const uint8* remap_string
length of UTF8PropObj: const uint8* remap_string
offset of UTF8PropObj: const uint8* fast_state
length of UTF8PropObj: const uint8* fast_state
--------------------------------------------------------------------
start of const short kAvgDeltaOctaScore[]
length of const short kAvgDeltaOctaScore[]
--------------------------------------------------------------------
number of CLD2TableSummary objects encoded (n)
[Table 1]: CLD2TableSummary: uint32 kCLDTableSizeOne
[Table 1]: CLD2TableSummary: uint32 kCLDTableSize
[Table 1]: CLD2TableSummary: uint32 kCLDTableKeyMask
[Table 1]: CLD2TableSummary: uint32 kCLDTableBuildDate
[Table 1]: offset of CLD2TableSummary: const IndirectProbBucket4* kCLDTable
[Table 1]: length of CLD2TableSummary: const IndirectProbBucket4* kCLDTable
[Table 1]: offset of CLD2TableSummary: const uint32* kCLDTableInd
[Table 1]: length of CLD2TableSummary: const uint32* kCLDTableInd
[Table 1]: offset of CLD2TableSummary: const char* kRecognizedLangScripts
[Table 1]: length of CLD2TableSummary: const char* kRecognizedLangScripts + 1
.
.
.
[Table n]: CLD2TableSummary: uint32 kCLDTableSizeOne
[Table n]: CLD2TableSummary: uint32 kCLDTableSize
[Table n]: CLD2TableSummary: uint32 kCLDTableKeyMask
[Table n]: CLD2TableSummary: uint32 kCLDTableBuildDate
[Table n]: offset of CLD2TableSummary: const IndirectProbBucket4* kCLDTable
[Table n]: length of CLD2TableSummary: const IndirectProbBucket4* kCLDTable
[Table n]: offset of CLD2TableSummary: const uint32* kCLDTableInd
[Table n]: length of CLD2TableSummary: const uint32* kCLDTableInd
[Table n]: offset of CLD2TableSummary: const char* kRecognizedLangScripts
[Table n]: length of CLD2TableSummary: const char* kRecognizedLangScripts + 1
--------------------------------------------------------------------
Immediately after the header fields comes the data block. The data block has
the following content, in this order (note that padding is applied in order to
keep lookups word-aligned):
UTF8PropObj: const uint8* state_table
UTF8PropObj: const RemapEntry* remap_base (4-byte struct)
UTF8PropObj: const uint8* remap_string
UTF8PropObj: const uint8* fast_state
const short kAvgDeltaOctaScore[]
[Table 1]: CLD2TableSummary: const IndirectProbBucket4* kCLDTable
[Table 1]: CLD2TableSummary: const uint32* kCLDTableInd
[Table 1]: CLD2TableSummary: const char* kRecognizedLangScripts (with null terminator)
.
.
.
[Table n]: CLD2TableSummary: const IndirectProbBucket4* kCLDTable
[Table n]: CLD2TableSummary: const uint32* kCLDTableInd
[Table n]: CLD2TableSummary: const char* kRecognizedLangScripts (with null terminator)
It is STRONGLY recommended that the chunks within the data block be kept
128-bit aligned for efficiency reasons, although the code will work without
such alignment: the main lookup tables have randomly-accessed groups of four
4-byte entries, and these must be 16-byte aligned to avoid the performance
cost of multiple cache misses per group.
*/
namespace CLD2DynamicData {
static const char* DATA_FILE_MARKER = "cld2_data_file00";
static const int DATA_FILE_MARKER_LENGTH = 16; // Keep aligned to 128 bits
// Nicer version of memcmp that shows the offset at which bytes differ
bool mem_compare(const void* data1, const void* data2, const int length);
// Enable or disable debugging; 0 to disable, 1 to enable
void setDebug(int debug);
// Lower-level structure for individual tables. There are n table headers in
// a given file header.
typedef struct {
CLD2::uint32 kCLDTableSizeOne;
CLD2::uint32 kCLDTableSize;
CLD2::uint32 kCLDTableKeyMask;
CLD2::uint32 kCLDTableBuildDate;
CLD2::uint32 startOf_kCLDTable;
CLD2::uint32 lengthOf_kCLDTable;
CLD2::uint32 startOf_kCLDTableInd;
CLD2::uint32 lengthOf_kCLDTableInd;
CLD2::uint32 startOf_kRecognizedLangScripts;
CLD2::uint32 lengthOf_kRecognizedLangScripts;
} TableHeader;
// Top-level structure for a CLD2 Data File Header.
// Contains all the primitive fields for the header as well as an array of
// headers for the individual tables.
typedef struct {
// Marker fields help recognize and verify the data file
char sanityString[DATA_FILE_MARKER_LENGTH];
CLD2::uint32 totalFileSizeBytes;
// UTF8 primitives
CLD2::uint32 utf8PropObj_state0;
CLD2::uint32 utf8PropObj_state0_size;
CLD2::uint32 utf8PropObj_total_size;
CLD2::uint32 utf8PropObj_max_expand;
CLD2::uint32 utf8PropObj_entry_shift;
CLD2::uint32 utf8PropObj_bytes_per_entry;
CLD2::uint32 utf8PropObj_losub;
CLD2::uint32 utf8PropObj_hiadd;
CLD2::uint32 startOf_utf8PropObj_state_table;
CLD2::uint32 lengthOf_utf8PropObj_state_table;
CLD2::uint32 startOf_utf8PropObj_remap_base;
CLD2::uint32 lengthOf_utf8PropObj_remap_base;
CLD2::uint32 startOf_utf8PropObj_remap_string;
CLD2::uint32 lengthOf_utf8PropObj_remap_string;
CLD2::uint32 startOf_utf8PropObj_fast_state;
CLD2::uint32 lengthOf_utf8PropObj_fast_state;
// Average delta-octa-score bits
CLD2::uint32 startOf_kAvgDeltaOctaScore;
CLD2::uint32 lengthOf_kAvgDeltaOctaScore;
// Table bits
CLD2::uint32 numTablesEncoded;
TableHeader* tableHeaders;
} FileHeader;
// Calculate the exact size of a header that encodes the specified number of
// tables. This can be used to reserve space within the data file,
// calculate offsets, and so on.
CLD2::uint32 calculateHeaderSize(CLD2::uint32 numTables);
// Dump a given header to stdout as a human-readable string.
void dumpHeader(FileHeader* header);
// Verify that a given pair of scoring tables match precisely
// If there is a problem, returns an error message; otherwise, the empty string.
bool verify(const CLD2::ScoringTables* realData, const CLD2::ScoringTables* loadedData);
// Return true iff the program is running in little-endian mode.
bool isLittleEndian();
// Return true iff the core size assumptions are ok on this platform.
bool coreAssumptionsOk();
} // End namespace CLD2DynamicData
#endif // CLD2_INTERNAL_CLD2_DYNAMIC_DATA_H_
|
