summaryrefslogtreecommitdiffstats
path: root/application/basilisk/components/translation/cld2/internal/utf8statetable.h
diff options
context:
space:
mode:
Diffstat (limited to 'application/basilisk/components/translation/cld2/internal/utf8statetable.h')
-rw-r--r--application/basilisk/components/translation/cld2/internal/utf8statetable.h283
1 files changed, 283 insertions, 0 deletions
diff --git a/application/basilisk/components/translation/cld2/internal/utf8statetable.h b/application/basilisk/components/translation/cld2/internal/utf8statetable.h
new file mode 100644
index 000000000..55c00f45e
--- /dev/null
+++ b/application/basilisk/components/translation/cld2/internal/utf8statetable.h
@@ -0,0 +1,283 @@
+// Copyright 2013 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//
+// State Table follower for scanning UTF-8 strings without converting to
+// 32- or 16-bit Unicode values.
+//
+// Author: dsites@google.com (Dick Sites)
+//
+
+#ifndef UTIL_UTF8_UTF8STATETABLE_H_
+#define UTIL_UTF8_UTF8STATETABLE_H_
+
+#include <string>
+#include "integral_types.h" // for uint8, uint32, uint16
+#include "stringpiece.h"
+
+
+namespace CLD2 {
+
+class OffsetMap;
+
+
+// These four-byte entries compactly encode how many bytes 0..255 to delete
+// in making a string replacement, how many bytes to add 0..255, and the offset
+// 0..64k-1 of the replacement string in remap_string.
+struct RemapEntry {
+ uint8 delete_bytes;
+ uint8 add_bytes;
+ uint16 bytes_offset;
+};
+
+// Exit type codes for state tables. All but the first get stuffed into
+// signed one-byte entries. The first is only generated by executable code.
+// To distinguish from next-state entries, these must be contiguous and
+// all <= kExitNone
+typedef enum {
+ kExitDstSpaceFull = 239,
+ kExitIllegalStructure, // 240
+ kExitOK, // 241
+ kExitReject, // ...
+ kExitReplace1,
+ kExitReplace2,
+ kExitReplace3,
+ kExitReplace21,
+ kExitReplace31,
+ kExitReplace32,
+ kExitReplaceOffset1,
+ kExitReplaceOffset2,
+ kExitReplace1S0,
+ kExitSpecial,
+ kExitDoAgain,
+ kExitRejectAlt,
+ kExitNone // 255
+} ExitReason;
+
+typedef enum {
+ kExitDstSpaceFull_2 = 32767, // 0x7fff
+ kExitIllegalStructure_2, // 32768 0x8000
+ kExitOK_2, // 32769 0x8001
+ kExitReject_2, // ...
+ kExitReplace1_2,
+ kExitReplace2_2,
+ kExitReplace3_2,
+ kExitReplace21_2,
+ kExitReplace31_2,
+ kExitReplace32_2,
+ kExitReplaceOffset1_2,
+ kExitReplaceOffset2_2,
+ kExitReplace1S0_2,
+ kExitSpecial_2,
+ kExitDoAgain_2,
+ kExitRejectAlt_2,
+ kExitNone_2 // 32783 0x800f
+} ExitReason_2;
+
+
+// This struct represents one entire state table. The three initialized byte
+// areas are state_table, remap_base, and remap_string. state0 and state0_size
+// give the byte offset and length within state_table of the initial state --
+// table lookups are expected to start and end in this state, but for
+// truncated UTF-8 strings, may end in a different state. These allow a quick
+// test for that condition. entry_shift is 8 for tables subscripted by a full
+// byte value and 6 for space-optimized tables subscripted by only six
+// significant bits in UTF-8 continuation bytes.
+typedef struct {
+ const uint32 state0;
+ const uint32 state0_size;
+ const uint32 total_size;
+ const int max_expand;
+ const int entry_shift;
+ const int bytes_per_entry;
+ const uint32 losub;
+ const uint32 hiadd;
+ const uint8* state_table;
+ const RemapEntry* remap_base;
+ const uint8* remap_string;
+ const uint8* fast_state;
+} UTF8StateMachineObj;
+
+// Near-duplicate declaration for tables with two-byte entries
+typedef struct {
+ const uint32 state0;
+ const uint32 state0_size;
+ const uint32 total_size;
+ const int max_expand;
+ const int entry_shift;
+ const int bytes_per_entry;
+ const uint32 losub;
+ const uint32 hiadd;
+ const unsigned short* state_table;
+ const RemapEntry* remap_base;
+ const uint8* remap_string;
+ const uint8* fast_state;
+} UTF8StateMachineObj_2;
+
+
+typedef UTF8StateMachineObj UTF8PropObj;
+typedef UTF8StateMachineObj UTF8ScanObj;
+typedef UTF8StateMachineObj UTF8ReplaceObj;
+typedef UTF8StateMachineObj_2 UTF8PropObj_2;
+typedef UTF8StateMachineObj_2 UTF8ReplaceObj_2;
+// NOT IMPLEMENTED typedef UTF8StateMachineObj_2 UTF8ScanObj_2;
+
+
+// Look up property of one UTF-8 character and advance over it
+// Return 0 if input length is zero
+// Return 0 and advance one byte if input is ill-formed
+uint8 UTF8GenericProperty(const UTF8PropObj* st,
+ const uint8** src,
+ int* srclen);
+
+// Look up property of one UTF-8 character (assumed to be valid).
+// (This is a faster version of UTF8GenericProperty.)
+bool UTF8HasGenericProperty(const UTF8PropObj& st, const char* src);
+
+
+// BigOneByte versions are needed for tables > 240 states, but most
+// won't need the TwoByte versions.
+
+// Look up property of one UTF-8 character and advance over it
+// Return 0 if input length is zero
+// Return 0 and advance one byte if input is ill-formed
+uint8 UTF8GenericPropertyBigOneByte(const UTF8PropObj* st,
+ const uint8** src,
+ int* srclen);
+
+
+// TwoByte versions are needed for tables > 240 states that don't fit onto
+// BigOneByte -- rare ultimate fallback
+
+// Look up property of one UTF-8 character (assumed to be valid).
+// (This is a faster version of UTF8GenericProperty.)
+bool UTF8HasGenericPropertyBigOneByte(const UTF8PropObj& st, const char* src);
+
+// Look up property of one UTF-8 character and advance over it
+// Return 0 if input length is zero
+// Return 0 and advance one byte if input is ill-formed
+uint8 UTF8GenericPropertyTwoByte(const UTF8PropObj_2* st,
+ const uint8** src,
+ int* srclen);
+
+// Look up property of one UTF-8 character (assumed to be valid).
+// (This is a faster version of UTF8GenericProperty.)
+bool UTF8HasGenericPropertyTwoByte(const UTF8PropObj_2& st, const char* src);
+
+// Scan a UTF-8 stringpiece based on a state table.
+// Always scan complete UTF-8 characters
+// Set number of bytes scanned. Return reason for exiting
+int UTF8GenericScan(const UTF8ScanObj* st,
+ const StringPiece& str,
+ int* bytes_consumed);
+
+
+
+// Scan a UTF-8 stringpiece based on state table, copying to output stringpiece
+// and doing text replacements.
+// Always scan complete UTF-8 characters
+// Set number of bytes consumed from input, number filled to output.
+// Return reason for exiting
+// Also writes an optional OffsetMap. Pass NULL to skip writing one.
+int UTF8GenericReplace(const UTF8ReplaceObj* st,
+ const StringPiece& istr,
+ StringPiece& ostr,
+ bool is_plain_text,
+ int* bytes_consumed,
+ int* bytes_filled,
+ int* chars_changed,
+ OffsetMap* offsetmap);
+
+// Older version without offsetmap
+int UTF8GenericReplace(const UTF8ReplaceObj* st,
+ const StringPiece& istr,
+ StringPiece& ostr,
+ bool is_plain_text,
+ int* bytes_consumed,
+ int* bytes_filled,
+ int* chars_changed);
+
+// Older version without is_plain_text or offsetmap
+int UTF8GenericReplace(const UTF8ReplaceObj* st,
+ const StringPiece& istr,
+ StringPiece& ostr,
+ int* bytes_consumed,
+ int* bytes_filled,
+ int* chars_changed);
+
+
+// TwoByte version is needed for tables > about 256 states, such
+// as the table for full Unicode 4.1 canonical + compatibility mapping
+
+// Scan a UTF-8 stringpiece based on state table with two-byte entries,
+// copying to output stringpiece
+// and doing text replacements.
+// Always scan complete UTF-8 characters
+// Set number of bytes consumed from input, number filled to output.
+// Return reason for exiting
+// Also writes an optional OffsetMap. Pass NULL to skip writing one.
+int UTF8GenericReplaceTwoByte(const UTF8ReplaceObj_2* st,
+ const StringPiece& istr,
+ StringPiece& ostr,
+ bool is_plain_text,
+ int* bytes_consumed,
+ int* bytes_filled,
+ int* chars_changed,
+ OffsetMap* offsetmap);
+
+// Older version without offsetmap
+int UTF8GenericReplaceTwoByte(const UTF8ReplaceObj_2* st,
+ const StringPiece& istr,
+ StringPiece& ostr,
+ bool is_plain_text,
+ int* bytes_consumed,
+ int* bytes_filled,
+ int* chars_changed);
+
+// Older version without is_plain_text or offsetmap
+int UTF8GenericReplaceTwoByte(const UTF8ReplaceObj_2* st,
+ const StringPiece& istr,
+ StringPiece& ostr,
+ int* bytes_consumed,
+ int* bytes_filled,
+ int* chars_changed);
+
+
+static const unsigned char kUTF8LenTbl[256] = {
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
+
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
+ 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
+ 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+ 3,3,3,3,3,3,3,3, 3,3,3,3,3,3,3,3, 4,4,4,4,4,4,4,4, 4,4,4,4,4,4,4,4
+};
+
+inline int UTF8OneCharLen(const char* in) {
+ return kUTF8LenTbl[*reinterpret_cast<const uint8*>(in)];
+}
+
+// Adjust a stringpiece to encompass complete UTF-8 characters.
+// The data pointer will be increased by 0..3 bytes to get to a character
+// boundary, and the length will then be decreased by 0..3 bytes
+// to encompass the last complete character.
+// This is useful especially when a UTF-8 string must be put into a fixed-
+// maximum-size buffer cleanly, such as a MySQL buffer.
+void UTF8TrimToChars(StringPiece* istr);
+
+} // End namespace CLD2
+
+#endif // UTIL_UTF8_UTF8STATETABLE_H_
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-08 13:41:51 +0000