diff options
Diffstat (limited to 'application/basilisk/components/translation/cld2/internal/getonescriptspan.cc')
| -rw-r--r-- | application/basilisk/components/translation/cld2/internal/getonescriptspan.cc | 1086 |
1 files changed, 0 insertions, 1086 deletions
diff --git a/application/basilisk/components/translation/cld2/internal/getonescriptspan.cc b/application/basilisk/components/translation/cld2/internal/getonescriptspan.cc deleted file mode 100644 index 6bdd4871b..000000000 --- a/application/basilisk/components/translation/cld2/internal/getonescriptspan.cc +++ /dev/null @@ -1,1086 +0,0 @@ -// Copyright 2013 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// -// Author: dsites@google.com (Dick Sites) -// - - -#include "getonescriptspan.h" -#include <string.h> - -#include "fixunicodevalue.h" -#include "lang_script.h" -#include "port.h" -#include "utf8statetable.h" - -#include "utf8prop_lettermarkscriptnum.h" -#include "utf8repl_lettermarklower.h" -#include "utf8scannot_lettermarkspecial.h" - - -namespace CLD2 { - -// Alphabetical order for binary search, from -// generated_entities.cc -extern const int kNameToEntitySize; -extern const CharIntPair kNameToEntity[]; - -static const int kMaxUpToWordBoundary = 50; // span < this make longer, - // else make shorter -static const int kMaxAdvanceToWordBoundary = 10; // +/- this many bytes - // to round to word boundary, - // direction above - -static const char kSpecialSymbol[256] = { // true for < > & - 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,1,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,1,0,1,0, - 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, - - 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, -}; - - - -#define LT 0 // < -#define GT 1 // > -#define EX 2 // ! -#define HY 3 // - -#define QU 4 // " -#define AP 5 // ' -#define SL 6 // / -#define S_ 7 -#define C_ 8 -#define R_ 9 -#define I_ 10 -#define P_ 11 -#define T_ 12 -#define Y_ 13 -#define L_ 14 -#define E_ 15 -#define CR 16 // <cr> or <lf> -#define NL 17 // non-letter: ASCII whitespace, digit, punctuation -#define PL 18 // possible letter, incl. & -#define xx 19 // <unused> - -// Map byte to one of ~20 interesting categories for cheap tag parsing -static const uint8 kCharToSub[256] = { - NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,CR,NL, NL,CR,NL,NL, - NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL, - NL,EX,QU,NL, NL,NL,PL,AP, NL,NL,NL,NL, NL,HY,NL,SL, - NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL, LT,NL,GT,NL, - - PL,PL,PL,C_, PL,E_,PL,PL, PL,I_,PL,PL, L_,PL,PL,PL, - P_,PL,R_,S_, T_,PL,PL,PL, PL,Y_,PL,NL, NL,NL,NL,NL, - PL,PL,PL,C_, PL,E_,PL,PL, PL,I_,PL,PL, L_,PL,PL,PL, - P_,PL,R_,S_, T_,PL,PL,PL, PL,Y_,PL,NL, NL,NL,NL,NL, - - NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL, - NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL, - NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL, - NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL, - - PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL, - PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL, - PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL, - PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL, -}; - -#undef LT -#undef GT -#undef EX -#undef HY -#undef QU -#undef AP -#undef SL -#undef S_ -#undef C_ -#undef R_ -#undef I_ -#undef P_ -#undef T_ -#undef Y_ -#undef L_ -#undef E_ -#undef CR -#undef NL -#undef PL -#undef xx - - -#define OK 0 -#define X_ 1 - - -static const int kMaxExitStateLettersMarksOnly = 1; -static const int kMaxExitStateAllText = 2; - - -// State machine to do cheap parse of non-letter strings incl. tags -// advances <tag> -// | | -// advances <tag> ... </tag> for <script> <style> -// | | -// advances <!-- ... <tag> ... --> -// | | -// advances <tag -// || (0) -// advances <tag <tag2> -// || (0) -// -// We start in state [0] at a non-letter and make at least one transition -// When scanning for just letters, arriving back at state [0] or [1] exits -// the state machine. -// When scanning for any non-tag text, arriving at state [2] also exits -static const uint8 kTagParseTbl_0[] = { -// < > ! - " ' / S C R I P T Y L E CR NL PL xx - 3, 2, 2, 2, 2, 2, 2,OK, OK,OK,OK,OK, OK,OK,OK,OK, 2, 2,OK,X_, // [0] OK exit state - X_,X_,X_,X_, X_,X_,X_,X_, X_,X_,X_,X_, X_,X_,X_,X_, X_,X_,X_,X_, // [1] error exit state - 3, 2, 2, 2, 2, 2, 2,OK, OK,OK,OK,OK, OK,OK,OK,OK, 2, 2,OK,X_, // [2] NL* [exit state] - X_, 2, 4, 9, 10,11, 9,13, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,X_, // [3] < - X_, 2, 9, 5, 10,11, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,X_, // [4] <! - X_, 2, 9, 6, 10,11, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,X_, // [5] <!- - 6, 6, 6, 7, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,X_, // [6] <!--.* - 6, 6, 6, 8, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,X_, // [7] <!--.*- - 6, 2, 6, 8, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,X_, // [8] <!--.*-- - X_, 2, 9, 9, 10,11, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,X_, // [9] <.* - 10,10,10,10, 9,10,10,10, 10,10,10,10, 10,10,10,10, 12,10,10,X_, // [10] <.*" - 11,11,11,11, 11, 9,11,11, 11,11,11,11, 11,11,11,11, 12,11,11,X_, // [11] <.*' - X_, 2,12,12, 12,12,12,12, 12,12,12,12, 12,12,12,12, 12,12,12,X_, // [12] <.* no " ' - -// < > ! - " ' / S C R I P T Y L E CR NL PL xx - X_, 2, 9, 9, 10,11, 9, 9, 14, 9, 9, 9, 28, 9, 9, 9, 9, 9, 9,X_, // [13] <S - X_, 2, 9, 9, 10,11, 9, 9, 9,15, 9, 9, 9, 9, 9, 9, 9, 9, 9,X_, // [14] <SC - X_, 2, 9, 9, 10,11, 9, 9, 9, 9,16, 9, 9, 9, 9, 9, 9, 9, 9,X_, // [15] <SCR - X_, 2, 9, 9, 10,11, 9, 9, 9, 9, 9,17, 9, 9, 9, 9, 9, 9, 9,X_, // [16] <SCRI - X_, 2, 9, 9, 10,11, 9, 9, 9, 9, 9, 9, 18, 9, 9, 9, 9, 9, 9,X_, // [17] <SCRIP - X_,19, 9, 9, 10,11, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 19,19, 9,X_, // [18] <SCRIPT - 20,19,19,19, 19,19,19,19, 19,19,19,19, 19,19,19,19, 19,19,19,X_, // [19] <SCRIPT .* - 19,19,19,19, 19,19,21,19, 19,19,19,19, 19,19,19,19, 19,19,19,X_, // [20] <SCRIPT .*< - 19,19,19,19, 19,19,19,22, 19,19,19,19, 19,19,19,19, 21,21,19,X_, // [21] <SCRIPT .*</ allow SP CR LF - 19,19,19,19, 19,19,19,19, 23,19,19,19, 19,19,19,19, 19,19,19,X_, // [22] <SCRIPT .*</S - 19,19,19,19, 19,19,19,19, 19,24,19,19, 19,19,19,19, 19,19,19,X_, // [23] <SCRIPT .*</SC - 19,19,19,19, 19,19,19,19, 19,19,25,19, 19,19,19,19, 19,19,19,X_, // [24] <SCRIPT .*</SCR - 19,19,19,19, 19,19,19,19, 19,19,19,26, 19,19,19,19, 19,19,19,X_, // [25] <SCRIPT .*</SCRI - 19,19,19,19, 19,19,19,19, 19,19,19,19, 27,19,19,19, 19,19,19,X_, // [26] <SCRIPT .*</SCRIP - 19, 2,19,19, 19,19,19,19, 19,19,19,19, 19,19,19,19, 19,19,19,X_, // [27] <SCRIPT .*</SCRIPT - -// < > ! - " ' / S C R I P T Y L E CR NL PL xx - X_, 2, 9, 9, 10,11, 9, 9, 9, 9, 9, 9, 9,29, 9, 9, 9, 9, 9,X_, // [28] <ST - X_, 2, 9, 9, 10,11, 9, 9, 9, 9, 9, 9, 9, 9,30, 9, 9, 9, 9,X_, // [29] <STY - X_, 2, 9, 9, 10,11, 9, 9, 9, 9, 9, 9, 9, 9, 9,31, 9, 9, 9,X_, // [30] <STYL - X_,32, 9, 9, 10,11, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 32,32, 9,X_, // [31] <STYLE - 33,32,32,32, 32,32,32,32, 32,32,32,32, 32,32,32,32, 32,32,32,X_, // [32] <STYLE .* - 32,32,32,32, 32,32,34,32, 32,32,32,32, 32,32,32,32, 32,32,32,X_, // [33] <STYLE .*< - 32,32,32,32, 32,32,32,35, 32,32,32,32, 32,32,32,32, 34,34,32,X_, // [34] <STYLE .*</ allow SP CR LF - 32,32,32,32, 32,32,32,32, 32,32,32,32, 36,32,32,32, 32,32,32,X_, // [35] <STYLE .*</S - 32,32,32,32, 32,32,32,32, 32,32,32,32, 32,37,32,32, 32,32,32,X_, // [36] <STYLE .*</ST - 32,32,32,32, 32,32,32,32, 32,32,32,32, 32,32,38,32, 32,32,32,X_, // [37] <STYLE .*</STY - 32,32,32,32, 32,32,32,32, 32,32,32,32, 32,32,32,39, 32,32,32,X_, // [38] <STYLE .*</STYL - 32, 2,32,32, 32,32,32,32, 32,32,32,32, 32,32,32,32, 32,32,32,X_, // [39] <STYLE .*</STYLE -}; - -#undef OK -#undef X_ - -enum -{ - UTFmax = 4, // maximum bytes per rune - Runesync = 0x80, // cannot represent part of a UTF sequence (<) - Runeself = 0x80, // rune and UTF sequences are the same (<) - Runeerror = 0xFFFD, // decoding error in UTF - Runemax = 0x10FFFF, // maximum rune value -}; - -// Debugging. Not thread safe. -static char gDisplayPiece[32]; -const uint8 gCharlen[16] = {1,1,1,1, 1,1,1,1, 1,1,1,1, 2,2,3,4}; -char* DisplayPiece(const char* next_byte_, int byte_length_) { - // Copy up to 8 UTF-8 chars to buffer - int k = 0; // byte count - int n = 0; // character count - for (int i = 0; i < byte_length_; ++i) { - char c = next_byte_[i]; - if ((c & 0xc0) != 0x80) { - // Beginning of a UTF-8 character - int charlen = gCharlen[static_cast<uint8>(c) >> 4]; - if (i + charlen > byte_length_) {break;} // Not enough room for full char - if (k >= (32 - 7)) {break;} // Not necessarily enough room - if (n >= 8) {break;} // Enough characters already - ++n; - } - if (c == '<') { - memcpy(&gDisplayPiece[k], "<", 4); k += 4; - } else if (c == '>') { - memcpy(&gDisplayPiece[k], ">", 4); k += 4; - } else if (c == '&') { - memcpy(&gDisplayPiece[k], "&", 5); k += 5; - } else if (c == '\'') { - memcpy(&gDisplayPiece[k], "'", 6); k += 6; - } else if (c == '"') { - memcpy(&gDisplayPiece[k], """, 6); k += 6; - } else { - gDisplayPiece[k++] = c; - } - } - gDisplayPiece[k++] = '\0'; - return gDisplayPiece; -} - - - -// runetochar copies (encodes) one rune, pointed to by r, to at most -// UTFmax bytes starting at s and returns the number of bytes generated. -int runetochar(char *str, const char32 *rune) { - // Convert to unsigned for range check. - unsigned long c; - - // 1 char 00-7F - c = *rune; - if(c <= 0x7F) { - str[0] = c; - return 1; - } - - // 2 char 0080-07FF - if(c <= 0x07FF) { - str[0] = 0xC0 | (c >> 1*6); - str[1] = 0x80 | (c & 0x3F); - return 2; - } - - // Range check - if (c > Runemax) { - c = Runeerror; - } - - // 3 char 0800-FFFF - if (c <= 0xFFFF) { - str[0] = 0xE0 | (c >> 2*6); - str[1] = 0x80 | ((c >> 1*6) & 0x3F); - str[2] = 0x80 | (c & 0x3F); - return 3; - } - - // 4 char 10000-1FFFFF - str[0] = 0xF0 | (c >> 3*6); - str[1] = 0x80 | ((c >> 2*6) & 0x3F); - str[2] = 0x80 | ((c >> 1*6) & 0x3F); - str[3] = 0x80 | (c & 0x3F); - return 4; -} - - - -// Useful for converting an entity to an ascii value. -// RETURNS unicode value, or -1 if entity isn't valid. Don't include & or ; -int LookupEntity(const char* entity_name, int entity_len) { - // Make a C string - if (entity_len >= 16) {return -1;} // All real entities are shorter - char temp[16]; - memcpy(temp, entity_name, entity_len); - temp[entity_len] = '\0'; - int match = BinarySearch(temp, 0, kNameToEntitySize, kNameToEntity); - if (match >= 0) {return kNameToEntity[match].i;} - return -1; -} - -bool ascii_isdigit(char c) { - return ('0' <= c) && (c <= '9'); -} -bool ascii_isxdigit(char c) { - if (('0' <= c) && (c <= '9')) {return true;} - if (('a' <= c) && (c <= 'f')) {return true;} - if (('A' <= c) && (c <= 'F')) {return true;} - return false; -} -bool ascii_isalnum(char c) { - if (('0' <= c) && (c <= '9')) {return true;} - if (('a' <= c) && (c <= 'z')) {return true;} - if (('A' <= c) && (c <= 'Z')) {return true;} - return false; -} -int hex_digit_to_int(char c) { - if (('0' <= c) && (c <= '9')) {return c - '0';} - if (('a' <= c) && (c <= 'f')) {return c - 'a' + 10;} - if (('A' <= c) && (c <= 'F')) {return c - 'A' + 10;} - return 0; -} - -static int32 strto32_base10(const char* nptr, const char* limit, - const char **endptr) { - *endptr = nptr; - while (nptr < limit && *nptr == '0') { - ++nptr; - } - if (nptr == limit || !ascii_isdigit(*nptr)) - return -1; - const char* end_digits_run = nptr; - while (end_digits_run < limit && ascii_isdigit(*end_digits_run)) { - ++end_digits_run; - } - *endptr = end_digits_run; - const int num_digits = end_digits_run - nptr; - // kint32max == 2147483647. - if (num_digits < 9 || - (num_digits == 10 && memcmp(nptr, "2147483647", 10) <= 0)) { - int value = 0; - for (; nptr < end_digits_run; ++nptr) { - value *= 10; - value += *nptr - '0'; - } - // Overflow past the last valid unicode codepoint - // (0x10ffff) is converted to U+FFFD by FixUnicodeValue(). - return FixUnicodeValue(value); - } else { - // Overflow: can't fit in an int32; - // returns the replacement character 0xFFFD. - return 0xFFFD; - } -} - -static int32 strto32_base16(const char* nptr, const char* limit, - const char **endptr) { - *endptr = nptr; - while (nptr < limit && *nptr == '0') { - ++nptr; - } - if (nptr == limit || !ascii_isxdigit(*nptr)) { - return -1; - } - const char* end_xdigits_run = nptr; - while (end_xdigits_run < limit && ascii_isxdigit(*end_xdigits_run)) { - ++end_xdigits_run; - } - *endptr = end_xdigits_run; - const int num_xdigits = end_xdigits_run - nptr; - // kint32max == 0x7FFFFFFF. - if (num_xdigits < 8 || (num_xdigits == 8 && nptr[0] < '8')) { - int value = 0; - for (; nptr < end_xdigits_run; ++nptr) { - value <<= 4; - value += hex_digit_to_int(*nptr); - } - // Overflow past the last valid unicode codepoint - // (0x10ffff) is converted to U+FFFD by FixUnicodeValue(). - return FixUnicodeValue(value); - } else { - // Overflow: can't fit in an int32; - // returns the replacement character 0xFFFD. - return 0xFFFD; - } -} - -// Unescape the current character pointed to by src. SETS the number -// of chars read for the conversion (in UTF8). If src isn't a valid entity, -// just consume the & and RETURN -1. If src doesn't point to & -- which it -// should -- set src_consumed to 0 and RETURN -1. -int ReadEntity(const char* src, int srcn, int* src_consumed) { - const char* const srcend = src + srcn; - - if (srcn == 0 || *src != '&') { // input should start with an ampersand - *src_consumed = 0; - return -1; - } - *src_consumed = 1; // we'll get the & at least - - // The standards are a bit unclear on when an entity ends. Certainly a ";" - // ends one, but spaces probably do too. We follow the lead of both IE and - // Netscape, which as far as we can tell end numeric entities (1st case below) - // at any non-digit, and end character entities (2nd case) at any non-alnum. - const char* entstart, *entend; // where the entity starts and ends - entstart = src + 1; // read past the & - int entval; // UCS2 value of the entity - if ( *entstart == '#' ) { // -- 1st case: numeric entity - if ( entstart + 2 >= srcend ) { - return -1; // no way a legitimate number could fit - } else if ( entstart[1] == 'x' || entstart[1] == 'X' ) { // hex numeric - entval = strto32_base16(entstart + 2, srcend, &entend); - } else { // decimal numeric entity - entval = strto32_base10(entstart+1, srcend, &entend); - } - if (entval == -1 || entend > srcend) { - return -1; // not entirely correct, but close enough - } - } else { // -- 2nd case: character entity - for (entend = entstart; - entend < srcend && ascii_isalnum(*entend); - ++entend ) { - // entity consists of alphanumeric chars - } - entval = LookupEntity(entstart, entend - entstart); - if (entval < 0) { - return -1; // not a legal entity name - } - // Now we do a strange-seeming IE6-compatibility check: if entval is - // >= 256, it *must* be followed by a semicolon or it's not considered - // an entity. The problem is lots of the newfangled entity names, like - // "lang", also occur in URL CGI arguments: "/search?q=test&lang=en". - // When these links are written in HTML, it would be really bad if the - // "&lang" were treated as an entity, which is what the spec says - // *should* happen (even when the HTML is inside an "A HREF" tag!) - // IE ignores the spec for these new, high-value entities, so we do too. - if ( entval >= 256 && !(entend < srcend && *entend == ';') ) { - return -1; // make non-;-terminated entity illegal - } - } - - // Finally, figure out how much src was consumed - if ( entend < srcend && *entend == ';' ) { - entend++; // standard says ; terminator is special - } - *src_consumed = entend - src; - return entval; -} - - -// Src points to '&' -// Writes entity value to dst. Returns take(src), put(dst) byte counts -void EntityToBuffer(const char* src, int len, char* dst, - int* tlen, int* plen) { - char32 entval = ReadEntity(src, len, tlen); - - // ReadEntity does this already: entval = FixUnicodeValue(entval); - - // Convert UTF-32 to UTF-8 - if (entval > 0) { - *plen = runetochar(dst, &entval); - } else { - // Illegal entity; ignore the '&' - *tlen = 1; - *plen = 0; - } -} - -// Returns true if character is < > or &, none of which are letters -bool inline IsSpecial(char c) { - if ((c & 0xe0) == 0x20) { - return kSpecialSymbol[static_cast<uint8>(c)]; - } - return false; -} - -// Quick Skip to next letter or < > & or to end of string (eos) -// Always return is_letter for eos -int ScanToLetterOrSpecial(const char* src, int len) { - int bytes_consumed; - StringPiece str(src, len); - UTF8GenericScan(&utf8scannot_lettermarkspecial_obj, str, &bytes_consumed); - return bytes_consumed; -} - - - - -// src points to non-letter, such as tag-opening '<' -// Return length from here to next possible letter -// On another < before >, return 1 -// advances <tag> -// | | -// advances <tag> ... </tag> for <script> <style> -// | | -// advances <!-- ... <tag> ... --> -// | | -// advances <tag -// | | end of string -// advances <tag <tag2> -// || -int ScanToPossibleLetter(const char* isrc, int len, int max_exit_state) { - const uint8* src = reinterpret_cast<const uint8*>(isrc); - const uint8* srclimit = src + len; - const uint8* tagParseTbl = kTagParseTbl_0; - int e = 0; - while (src < srclimit) { - e = tagParseTbl[kCharToSub[*src++]]; - if (e <= max_exit_state) { - // We overshot by one byte - --src; - break; - } - tagParseTbl = &kTagParseTbl_0[e * 20]; - } - - if (src >= srclimit) { - // We fell off the end of the text. - // It looks like the most common case for this is a truncated file, not - // mismatched angle brackets. So we pretend that the last char was '>' - return len; - } - - // OK to be in state 0 or state 2 at exit - if ((e != 0) && (e != 2)) { - // Error, '<' followed by '<' - // We want to back up to first <, then advance by one byte past it - int offset = src - reinterpret_cast<const uint8*>(isrc); - - // Backscan to first '<' and return enough length to just get past it - --offset; // back up over the second '<', which caused us to stop - while ((0 < offset) && (isrc[offset] != '<')) { - // Find the first '<', which is unmatched - --offset; - } - // skip to just beyond first '<' - return offset + 1; - } - - return src - reinterpret_cast<const uint8*>(isrc); -} - - -ScriptScanner::ScriptScanner(const char* buffer, - int buffer_length, - bool is_plain_text) - : start_byte_(buffer), - next_byte_(buffer), - next_byte_limit_(buffer + buffer_length), - byte_length_(buffer_length), - is_plain_text_(is_plain_text), - letters_marks_only_(true), - one_script_only_(true), - exit_state_(kMaxExitStateLettersMarksOnly) { - script_buffer_ = new char[kMaxScriptBuffer]; - script_buffer_lower_ = new char[kMaxScriptLowerBuffer]; - map2original_.Clear(); // map from script_buffer_ to buffer - map2uplow_.Clear(); // map from script_buffer_lower_ to script_buffer_ -} - -// Extended version to allow spans of any non-tag text and spans of mixed script -ScriptScanner::ScriptScanner(const char* buffer, - int buffer_length, - bool is_plain_text, - bool any_text, - bool any_script) - : start_byte_(buffer), - next_byte_(buffer), - next_byte_limit_(buffer + buffer_length), - byte_length_(buffer_length), - is_plain_text_(is_plain_text), - letters_marks_only_(!any_text), - one_script_only_(!any_script), - exit_state_(any_text ? kMaxExitStateAllText : kMaxExitStateLettersMarksOnly) { - script_buffer_ = new char[kMaxScriptBuffer]; - script_buffer_lower_ = new char[kMaxScriptLowerBuffer]; - map2original_.Clear(); // map from script_buffer_ to buffer - map2uplow_.Clear(); // map from script_buffer_lower_ to script_buffer_ -} - - -ScriptScanner::~ScriptScanner() { - delete[] script_buffer_; - delete[] script_buffer_lower_; -} - - - - -// Get to the first real non-tag letter or entity that is a letter -// Sets script of that letter -// Return len if no more letters -int ScriptScanner::SkipToFrontOfSpan(const char* src, int len, int* script) { - int sc = UNKNOWN_ULSCRIPT; - int skip = 0; - int tlen, plen; - - // Do run of non-letters (tag | &NL | NL)* - tlen = 0; - while (skip < len) { - // Do fast scan to next interesting byte - // int oldskip = skip; - skip += ScanToLetterOrSpecial(src + skip, len - skip); - - // Check for no more letters/specials - if (skip >= len) { - // All done - *script = sc; - return len; - } - - // We are at a letter, nonletter, tag, or entity - if (IsSpecial(src[skip]) && !is_plain_text_) { - if (src[skip] == '<') { - // Begining of tag; skip to end and go around again - tlen = ScanToPossibleLetter(src + skip, len - skip, - exit_state_); - sc = 0; - } else if (src[skip] == '>') { - // Unexpected end of tag; skip it and go around again - tlen = 1; // Over the > - sc = 0; - } else if (src[skip] == '&') { - // Expand entity, no advance - char temp[4]; - EntityToBuffer(src + skip, len - skip, - temp, &tlen, &plen); - sc = GetUTF8LetterScriptNum(temp); - } - } else { - // Update 1..4 bytes - tlen = UTF8OneCharLen(src + skip); - sc = GetUTF8LetterScriptNum(src + skip); - } - if (sc != 0) {break;} // Letter found - skip += tlen; // Else advance - } - - *script = sc; - return skip; -} - - -// These are for ASCII-only tag names -// Compare one letter uplow to c, ignoring case of uplowp -inline bool EqCase(char uplow, char c) { - return (uplow | 0x20) == c; -} - -// These are for ASCII-only tag names -// Return true for space / < > etc. all less than 0x40 -inline bool NeqLetter(char c) { - return c < 0x40; -} - -// These are for ASCII-only tag names -// Return true for space \n false for \r -inline bool WS(char c) { - return (c == ' ') || (c == '\n'); -} - -// Canonical CR or LF -static const char LF = '\n'; - - -// The naive loop scans from next_byte_ to script_buffer_ until full. -// But this can leave an awkward hard-to-identify short fragment at the -// end of the input. We would prefer to make the next-to-last fragment -// shorter and the last fragment longer. - -// Copy next run of non-tag characters to buffer [NUL terminated] -// This just replaces tags with space or \n and removes entities. -// Tags <br> <p> and <tr> are replaced with \n. Non-letter sequences -// including \r or \n are replaced by \n. All other tags and skipped text -// are replaced with ASCII space. -// -// Buffer ALWAYS has leading space and trailing space space space NUL -bool ScriptScanner::GetOneTextSpan(LangSpan* span) { - span->text = script_buffer_; - span->text_bytes = 0; - span->offset = next_byte_ - start_byte_; - span->ulscript = UNKNOWN_ULSCRIPT; - span->lang = UNKNOWN_LANGUAGE; - span->truncated = false; - - int put_soft_limit = kMaxScriptBytes - kWithinScriptTail; - if ((kMaxScriptBytes <= byte_length_) && - (byte_length_ < (2 * kMaxScriptBytes))) { - // Try to split the last two fragments in half - put_soft_limit = byte_length_ / 2; - } - - script_buffer_[0] = ' '; // Always a space at front of output - script_buffer_[1] = '\0'; - int take = 0; - int put = 1; // Start after the initial space - int tlen, plen; - - if (byte_length_ <= 0) { - return false; // No more text to be found - } - - // Go over alternating spans of text and tags, - // copying letters to buffer with single spaces for each run of non-letters - bool last_byte_was_space = false; - while (take < byte_length_) { - char c = next_byte_[take]; - if (c == '\r') {c = LF;} // Canonical CR or LF - if (c == '\n') {c = LF;} // Canonical CR or LF - - if (IsSpecial(c) && !is_plain_text_) { - if (c == '<') { - // Replace tag with space - c = ' '; // for almost-full test below - // or if <p> <br> <tr>, replace with \n - if (take < (byte_length_ - 3)) { - if (EqCase(next_byte_[take + 1], 'p') && - NeqLetter(next_byte_[take + 2])) { - c = LF; - } - if (EqCase(next_byte_[take + 1], 'b') && - EqCase(next_byte_[take + 2], 'r') && - NeqLetter(next_byte_[take + 3])) { - c = LF; - } - if (EqCase(next_byte_[take + 1], 't') && - EqCase(next_byte_[take + 2], 'r') && - NeqLetter(next_byte_[take + 3])) { - c = LF; - } - } - // Begining of tag; skip to end and go around again - tlen = 1 + ScanToPossibleLetter(next_byte_ + take, byte_length_ - take, - exit_state_); - // Copy one byte, compressing spaces - if (!last_byte_was_space || !WS(c)) { - script_buffer_[put++] = c; // Advance dest - last_byte_was_space = WS(c); - } - } else if (c == '>') { - // Unexpected end of tag; copy it and go around again - tlen = 1; // Over the > - script_buffer_[put++] = c; // Advance dest - } else if (c == '&') { - // Expand entity, no advance - EntityToBuffer(next_byte_ + take, byte_length_ - take, - script_buffer_ + put, &tlen, &plen); - put += plen; // Advance dest - } - take += tlen; // Advance source - } else { - // Copy one byte, compressing spaces - if (!last_byte_was_space || !WS(c)) { - script_buffer_[put++] = c; // Advance dest - last_byte_was_space = WS(c); - } - ++take; // Advance source - } - - if (WS(c) && - (put >= put_soft_limit)) { - // Buffer is almost full - span->truncated = true; - break; - } - if (put >= kMaxScriptBytes) { - // Buffer is completely full - span->truncated = true; - break; - } - } - - // Almost done. Back up to a character boundary if needed - while ((0 < take) && ((next_byte_[take] & 0xc0) == 0x80)) { - // Back up over continuation byte - --take; - --put; - } - - // Update input position - next_byte_ += take; - byte_length_ -= take; - - // Put four more spaces/NUL. Worst case is abcd _ _ _ \0 - // kMaxScriptBytes | | put - script_buffer_[put + 0] = ' '; - script_buffer_[put + 1] = ' '; - script_buffer_[put + 2] = ' '; - script_buffer_[put + 3] = '\0'; - - span->text_bytes = put; // Does not include the last four chars above - return true; -} - - -// Copy next run of same-script non-tag letters to buffer [NUL terminated] -// Buffer ALWAYS has leading space and trailing space space space NUL -bool ScriptScanner::GetOneScriptSpan(LangSpan* span) { - if (!letters_marks_only_) { - // Return non-tag text, including punctuation and digits - return GetOneTextSpan(span); - } - - span->text = script_buffer_; - span->text_bytes = 0; - span->offset = next_byte_ - start_byte_; - span->ulscript = UNKNOWN_ULSCRIPT; - span->lang = UNKNOWN_LANGUAGE; - span->truncated = false; - - // struct timeval script_start, script_mid, script_end; - - int put_soft_limit = kMaxScriptBytes - kWithinScriptTail; - if ((kMaxScriptBytes <= byte_length_) && - (byte_length_ < (2 * kMaxScriptBytes))) { - // Try to split the last two fragments in half - put_soft_limit = byte_length_ / 2; - } - - - int spanscript; // The script of this span - int sc = UNKNOWN_ULSCRIPT; // The script of next character - int tlen = 0; - int plen = 0; - - script_buffer_[0] = ' '; // Always a space at front of output - script_buffer_[1] = '\0'; - int take = 0; - int put = 1; // Start after the initial space - - // Build offsets from span->text back to start_byte_ + span->offset - // This mapping reflects deletion of non-letters, expansion of - // entities, etc. - map2original_.Clear(); - map2original_.Delete(span->offset); // So that MapBack(0) gives offset - - // Get to the first real non-tag letter or entity that is a letter - int skip = SkipToFrontOfSpan(next_byte_, byte_length_, &spanscript); - next_byte_ += skip; - byte_length_ -= skip; - - if (skip != 1) { - map2original_.Delete(skip); - map2original_.Insert(1); - } else { - map2original_.Copy(1); - } - if (byte_length_ <= 0) { - map2original_.Reset(); - return false; // No more letters to be found - } - - // There is at least one letter, so we know the script for this span - span->ulscript = (ULScript)spanscript; - - - // Go over alternating spans of same-script letters and non-letters, - // copying letters to buffer with single spaces for each run of non-letters - while (take < byte_length_) { - // Copy run of letters in same script (&LS | LS)* - int letter_count = 0; // Keep track of word length - bool need_break = false; - - while (take < byte_length_) { - // We are at a letter, nonletter, tag, or entity - if (IsSpecial(next_byte_[take]) && !is_plain_text_) { - if (next_byte_[take] == '<') { - // Begining of tag - sc = 0; - break; - } else if (next_byte_[take] == '>') { - // Unexpected end of tag - sc = 0; - break; - } else if (next_byte_[take] == '&') { - // Copy entity, no advance - EntityToBuffer(next_byte_ + take, byte_length_ - take, - script_buffer_ + put, &tlen, &plen); - sc = GetUTF8LetterScriptNum(script_buffer_ + put); - } - } else { - // Real letter, safely copy up to 4 bytes, increment by 1..4 - // Will update by 1..4 bytes at Advance, below - tlen = plen = UTF8OneCharLen(next_byte_ + take); - if (take < (byte_length_ - 3)) { - // X86 fast case, does unaligned load/store - UNALIGNED_STORE32(script_buffer_ + put, - UNALIGNED_LOAD32(next_byte_ + take)); - - } else { - // Slow case, happens 1-3 times per input document - memcpy(script_buffer_ + put, next_byte_ + take, plen); - } - sc = GetUTF8LetterScriptNum(next_byte_ + take); - } - - // Allow continue across a single letter in a different script: - // A B D = three scripts, c = common script, i = inherited script, - // - = don't care, ( = take position before the += below - // AAA(A- continue - // - // AAA(BA continue - // AAA(BB break - // AAA(Bc continue (breaks after B) - // AAA(BD break - // AAA(Bi break - // - // AAA(c- break - // - // AAA(i- continue - // - - if ((sc != spanscript) && (sc != ULScript_Inherited)) { - // Might need to break this script span - if (sc == ULScript_Common) { - need_break = true; - } else { - // Look at next following character, ignoring entity as Common - int sc2 = GetUTF8LetterScriptNum(next_byte_ + take + tlen); - if ((sc2 != ULScript_Common) && (sc2 != spanscript)) { - // We found a non-trivial change of script - if (one_script_only_) { - need_break = true; - } - } - } - } - if (need_break) {break;} // Non-letter or letter in wrong script - - take += tlen; // Advance - put += plen; // Advance - - // Update the offset map to reflect take/put lengths - if (tlen == plen) { - map2original_.Copy(tlen); - } else if (tlen < plen) { - map2original_.Copy(tlen); - map2original_.Insert(plen - tlen); - } else { // plen < tlen - map2original_.Copy(plen); - map2original_.Delete(tlen - plen); - } - - ++letter_count; - if (put >= kMaxScriptBytes) { - // Buffer is full - span->truncated = true; - break; - } - } // End while letters - - // Do run of non-letters (tag | &NL | NL)* - while (take < byte_length_) { - // Do fast scan to next interesting byte - tlen = ScanToLetterOrSpecial(next_byte_ + take, byte_length_ - take); - take += tlen; - map2original_.Delete(tlen); - if (take >= byte_length_) {break;} // Might have scanned to end - - // We are at a letter, nonletter, tag, or entity - if (IsSpecial(next_byte_[take]) && !is_plain_text_) { - if (next_byte_[take] == '<') { - // Begining of tag; skip to end and go around again - tlen = ScanToPossibleLetter(next_byte_ + take, byte_length_ - take, - exit_state_); - sc = 0; - } else if (next_byte_[take] == '>') { - // Unexpected end of tag; skip it and go around again - tlen = 1; // Over the > - sc = 0; - } else if (next_byte_[take] == '&') { - // Expand entity, no advance - EntityToBuffer(next_byte_ + take, byte_length_ - take, - script_buffer_ + put, &tlen, &plen); - sc = GetUTF8LetterScriptNum(script_buffer_ + put); - } - } else { - // Update 1..4 - tlen = UTF8OneCharLen(next_byte_ + take); - sc = GetUTF8LetterScriptNum(next_byte_ + take); - } - if (sc != 0) {break;} // Letter found - take += tlen; // Else advance - map2original_.Delete(tlen); - } // End while not-letters - - script_buffer_[put++] = ' '; - map2original_.Insert(1); - - // Letter in wrong script ? - if ((sc != spanscript) && (sc != ULScript_Inherited)) {break;} - if (put >= put_soft_limit) { - // Buffer is almost full - span->truncated = true; - break; - } - } - - // Almost done. Back up to a character boundary if needed - while ((0 < take) && (take < byte_length_) && - ((next_byte_[take] & 0xc0) == 0x80)) { - // Back up over continuation byte - --take; - --put; - } - - // Update input position - next_byte_ += take; - byte_length_ -= take; - - // Put four more spaces/NUL. Worst case is abcd _ _ _ \0 - // kMaxScriptBytes | | put - script_buffer_[put + 0] = ' '; - script_buffer_[put + 1] = ' '; - script_buffer_[put + 2] = ' '; - script_buffer_[put + 3] = '\0'; - map2original_.Insert(4); - map2original_.Reset(); - - span->text_bytes = put; // Does not include the last four chars above - return true; -} - -// Force Latin, Cyrillic, Armenian, Greek scripts to be lowercase -// List changes with each version of Unicode, so just always lowercase -// Unicode 6.2.0: -// ARMENIAN COPTIC CYRILLIC DESERET GEORGIAN GLAGOLITIC GREEK LATIN -void ScriptScanner::LowerScriptSpan(LangSpan* span) { - // If needed, lowercase all the text. If we do it sooner, might miss - // lowercasing an entity such as Á - // We only need to do this for Latn and Cyrl scripts - map2uplow_.Clear(); - // Full Unicode lowercase of the entire buffer, including - // four pad bytes off the end. - // Ahhh. But the last byte 0x00 is not interchange-valid, so we do 3 pad - // bytes and put the 0x00 in explicitly. - // Build an offset map from script_buffer_lower_ back to script_buffer_ - int consumed, filled, changed; - StringPiece istr(span->text, span->text_bytes + 3); - StringPiece ostr(script_buffer_lower_, kMaxScriptLowerBuffer); - - UTF8GenericReplace(&utf8repl_lettermarklower_obj, - istr, ostr, is_plain_text_, - &consumed, &filled, &changed, &map2uplow_); - script_buffer_lower_[filled] = '\0'; - span->text = script_buffer_lower_; - span->text_bytes = filled - 3; - map2uplow_.Reset(); -} - -// Copy next run of same-script non-tag letters to buffer [NUL terminated] -// Force Latin, Cyrillic, Greek scripts to be lowercase -// Buffer ALWAYS has leading space and trailing space space space NUL -bool ScriptScanner::GetOneScriptSpanLower(LangSpan* span) { - bool ok = GetOneScriptSpan(span); - LowerScriptSpan(span); - return ok; -} - - -// Maps byte offset in most recent GetOneScriptSpan/Lower -// span->text [0..text_bytes] into an additional byte offset from -// span->offset, to get back to corresponding text in the original -// input buffer. -// text_offset must be the first byte -// of a UTF-8 character, or just beyond the last character. Normally this -// routine is called with the first byte of an interesting range and -// again with the first byte of the following range. -int ScriptScanner::MapBack(int text_offset) { - return map2original_.MapBack(map2uplow_.MapBack(text_offset)); -} - - -// Gets lscript number for letters; always returns -// 0 (common script) for non-letters -int GetUTF8LetterScriptNum(const char* src) { - int srclen = UTF8OneCharLen(src); - const uint8* usrc = reinterpret_cast<const uint8*>(src); - return UTF8GenericPropertyTwoByte(&utf8prop_lettermarkscriptnum_obj, - &usrc, &srclen); -} - -} // namespace CLD2 - - |