Revert "Add Basilisk"

This reverts commit e72ef92b5bdc43cd2584198e2e54e951b70299e8.

1 files changed, 0 insertions, 1086 deletions

diff --git a/application/basilisk/components/translation/cld2/internal/getonescriptspan.cc b/application/basilisk/components/translation/cld2/internal/getonescriptspan.cc
deleted file mode 100644
index 6bdd4871b..000000000
--- a/application/basilisk/components/translation/cld2/internal/getonescriptspan.cc
+++ /dev/null
@@ -1,1086 +0,0 @@
-// Copyright 2013 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//
-// Author: dsites@google.com (Dick Sites)
-//
-
-
-#include "getonescriptspan.h"
-#include <string.h>
-
-#include "fixunicodevalue.h"
-#include "lang_script.h"
-#include "port.h"
-#include "utf8statetable.h"
-
-#include "utf8prop_lettermarkscriptnum.h"
-#include "utf8repl_lettermarklower.h"
-#include "utf8scannot_lettermarkspecial.h"
-
-
-namespace CLD2 {
-
-// Alphabetical order for binary search, from
-// generated_entities.cc
-extern const int kNameToEntitySize;
-extern const CharIntPair kNameToEntity[];
-
-static const int kMaxUpToWordBoundary = 50;       // span < this make longer,
-                                                  // else make shorter
-static const int kMaxAdvanceToWordBoundary = 10;  // +/- this many bytes
-                                                  // to round to word boundary,
-                                                  // direction above
-
-static const char kSpecialSymbol[256] = {       // true for < > &
-  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
-  0,0,0,0,0,0,1,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,1,0,1,0,
-  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
-  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
-
-  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
-  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
-  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
-  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
-};
-
-
-
-#define LT 0      // <
-#define GT 1      // >
-#define EX 2      // !
-#define HY 3      // -
-#define QU 4      // "
-#define AP 5      // '
-#define SL 6      // /
-#define S_ 7
-#define C_ 8
-#define R_ 9
-#define I_ 10
-#define P_ 11
-#define T_ 12
-#define Y_ 13
-#define L_ 14
-#define E_ 15
-#define CR 16     // <cr> or <lf>
-#define NL 17     // non-letter: ASCII whitespace, digit, punctuation
-#define PL 18     // possible letter, incl. &
-#define xx 19     // <unused>
-
-// Map byte to one of ~20 interesting categories for cheap tag parsing
-static const uint8 kCharToSub[256] = {
-  NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,CR,NL, NL,CR,NL,NL,
-  NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL,
-  NL,EX,QU,NL, NL,NL,PL,AP, NL,NL,NL,NL, NL,HY,NL,SL,
-  NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL, LT,NL,GT,NL,
-
-  PL,PL,PL,C_, PL,E_,PL,PL, PL,I_,PL,PL, L_,PL,PL,PL,
-  P_,PL,R_,S_, T_,PL,PL,PL, PL,Y_,PL,NL, NL,NL,NL,NL,
-  PL,PL,PL,C_, PL,E_,PL,PL, PL,I_,PL,PL, L_,PL,PL,PL,
-  P_,PL,R_,S_, T_,PL,PL,PL, PL,Y_,PL,NL, NL,NL,NL,NL,
-
-  NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL,
-  NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL,
-  NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL,
-  NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL,
-
-  PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL,
-  PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL,
-  PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL,
-  PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL,
-};
-
-#undef LT
-#undef GT
-#undef EX
-#undef HY
-#undef QU
-#undef AP
-#undef SL
-#undef S_
-#undef C_
-#undef R_
-#undef I_
-#undef P_
-#undef T_
-#undef Y_
-#undef L_
-#undef E_
-#undef CR
-#undef NL
-#undef PL
-#undef xx
-
-
-#define OK 0
-#define X_ 1
-
-
-static const int kMaxExitStateLettersMarksOnly = 1;
-static const int kMaxExitStateAllText = 2;
-
-
-// State machine to do cheap parse of non-letter strings incl. tags
-// advances <tag>
-//          |    |
-// advances <tag> ... </tag>  for <script> <style>
-//          |               |
-// advances <!-- ... <tag> ... -->
-//          |                     |
-// advances <tag
-//          ||  (0)
-// advances <tag <tag2>
-//          ||  (0)
-//
-// We start in state [0] at a non-letter and make at least one transition
-// When scanning for just letters, arriving back at state [0] or [1] exits
-//   the state machine.
-// When scanning for any non-tag text, arriving at state [2] also exits
-static const uint8 kTagParseTbl_0[] = {
-// <  >  !  -   "  '  /  S   C  R  I  P   T  Y  L  E  CR NL PL xx
-   3, 2, 2, 2,  2, 2, 2,OK, OK,OK,OK,OK, OK,OK,OK,OK,  2, 2,OK,X_, // [0] OK    exit state
-  X_,X_,X_,X_, X_,X_,X_,X_, X_,X_,X_,X_, X_,X_,X_,X_, X_,X_,X_,X_, // [1] error exit state
-   3, 2, 2, 2,  2, 2, 2,OK, OK,OK,OK,OK, OK,OK,OK,OK,  2, 2,OK,X_, // [2] NL*   [exit state]
-  X_, 2, 4, 9, 10,11, 9,13,  9, 9, 9, 9,  9, 9, 9, 9,  9, 9, 9,X_, // [3] <
-  X_, 2, 9, 5, 10,11, 9, 9,  9, 9, 9, 9,  9, 9, 9, 9,  9, 9, 9,X_, // [4] <!
-  X_, 2, 9, 6, 10,11, 9, 9,  9, 9, 9, 9,  9, 9, 9, 9,  9, 9, 9,X_, // [5] <!-
-   6, 6, 6, 7,  6, 6, 6, 6,  6, 6, 6, 6,  6, 6, 6, 6,  6, 6, 6,X_, // [6] <!--.*
-   6, 6, 6, 8,  6, 6, 6, 6,  6, 6, 6, 6,  6, 6, 6, 6,  6, 6, 6,X_, // [7] <!--.*-
-   6, 2, 6, 8,  6, 6, 6, 6,  6, 6, 6, 6,  6, 6, 6, 6,  6, 6, 6,X_, // [8] <!--.*--
-  X_, 2, 9, 9, 10,11, 9, 9,  9, 9, 9, 9,  9, 9, 9, 9,  9, 9, 9,X_, // [9] <.*
-  10,10,10,10,  9,10,10,10, 10,10,10,10, 10,10,10,10, 12,10,10,X_, // [10] <.*"
-  11,11,11,11, 11, 9,11,11, 11,11,11,11, 11,11,11,11, 12,11,11,X_, // [11] <.*'
-  X_, 2,12,12, 12,12,12,12, 12,12,12,12, 12,12,12,12, 12,12,12,X_, // [12] <.* no " '
-
-// <  >  !  -   "  '  /  S   C  R  I  P   T  Y  L  E  CR NL PL xx
-  X_, 2, 9, 9, 10,11, 9, 9, 14, 9, 9, 9, 28, 9, 9, 9,  9, 9, 9,X_, // [13] <S
-  X_, 2, 9, 9, 10,11, 9, 9,  9,15, 9, 9,  9, 9, 9, 9,  9, 9, 9,X_, // [14] <SC
-  X_, 2, 9, 9, 10,11, 9, 9,  9, 9,16, 9,  9, 9, 9, 9,  9, 9, 9,X_, // [15] <SCR
-  X_, 2, 9, 9, 10,11, 9, 9,  9, 9, 9,17,  9, 9, 9, 9,  9, 9, 9,X_, // [16] <SCRI
-  X_, 2, 9, 9, 10,11, 9, 9,  9, 9, 9, 9, 18, 9, 9, 9,  9, 9, 9,X_, // [17] <SCRIP
-  X_,19, 9, 9, 10,11, 9, 9,  9, 9, 9, 9,  9, 9, 9, 9, 19,19, 9,X_, // [18] <SCRIPT
-  20,19,19,19, 19,19,19,19, 19,19,19,19, 19,19,19,19, 19,19,19,X_, // [19] <SCRIPT .*
-  19,19,19,19, 19,19,21,19, 19,19,19,19, 19,19,19,19, 19,19,19,X_, // [20] <SCRIPT .*<
-  19,19,19,19, 19,19,19,22, 19,19,19,19, 19,19,19,19, 21,21,19,X_, // [21] <SCRIPT .*</ allow SP CR LF
-  19,19,19,19, 19,19,19,19, 23,19,19,19, 19,19,19,19, 19,19,19,X_, // [22] <SCRIPT .*</S
-  19,19,19,19, 19,19,19,19, 19,24,19,19, 19,19,19,19, 19,19,19,X_, // [23] <SCRIPT .*</SC
-  19,19,19,19, 19,19,19,19, 19,19,25,19, 19,19,19,19, 19,19,19,X_, // [24] <SCRIPT .*</SCR
-  19,19,19,19, 19,19,19,19, 19,19,19,26, 19,19,19,19, 19,19,19,X_, // [25] <SCRIPT .*</SCRI
-  19,19,19,19, 19,19,19,19, 19,19,19,19, 27,19,19,19, 19,19,19,X_, // [26] <SCRIPT .*</SCRIP
-  19, 2,19,19, 19,19,19,19, 19,19,19,19, 19,19,19,19, 19,19,19,X_, // [27] <SCRIPT .*</SCRIPT
-
-// <  >  !  -   "  '  /  S   C  R  I  P   T  Y  L  E  CR NL PL xx
-  X_, 2, 9, 9, 10,11, 9, 9,  9, 9, 9, 9,  9,29, 9, 9,  9, 9, 9,X_, // [28] <ST
-  X_, 2, 9, 9, 10,11, 9, 9,  9, 9, 9, 9,  9, 9,30, 9,  9, 9, 9,X_, // [29] <STY
-  X_, 2, 9, 9, 10,11, 9, 9,  9, 9, 9, 9,  9, 9, 9,31,  9, 9, 9,X_, // [30] <STYL
-  X_,32, 9, 9, 10,11, 9, 9,  9, 9, 9, 9,  9, 9, 9, 9, 32,32, 9,X_, // [31] <STYLE
-  33,32,32,32, 32,32,32,32, 32,32,32,32, 32,32,32,32, 32,32,32,X_, // [32] <STYLE .*
-  32,32,32,32, 32,32,34,32, 32,32,32,32, 32,32,32,32, 32,32,32,X_, // [33] <STYLE .*<
-  32,32,32,32, 32,32,32,35, 32,32,32,32, 32,32,32,32, 34,34,32,X_, // [34] <STYLE .*</ allow SP CR LF
-  32,32,32,32, 32,32,32,32, 32,32,32,32, 36,32,32,32, 32,32,32,X_, // [35] <STYLE .*</S
-  32,32,32,32, 32,32,32,32, 32,32,32,32, 32,37,32,32, 32,32,32,X_, // [36] <STYLE .*</ST
-  32,32,32,32, 32,32,32,32, 32,32,32,32, 32,32,38,32, 32,32,32,X_, // [37] <STYLE .*</STY
-  32,32,32,32, 32,32,32,32, 32,32,32,32, 32,32,32,39, 32,32,32,X_, // [38] <STYLE .*</STYL
-  32, 2,32,32, 32,32,32,32, 32,32,32,32, 32,32,32,32, 32,32,32,X_, // [39] <STYLE .*</STYLE
-};
-
-#undef OK
-#undef X_
-
-enum
-{
-  UTFmax        = 4,            // maximum bytes per rune
-  Runesync      = 0x80,         // cannot represent part of a UTF sequence (<)
-  Runeself      = 0x80,         // rune and UTF sequences are the same (<)
-  Runeerror     = 0xFFFD,       // decoding error in UTF
-  Runemax       = 0x10FFFF,     // maximum rune value
-};
-
-// Debugging. Not thread safe.
-static char gDisplayPiece[32];
-const uint8 gCharlen[16] = {1,1,1,1, 1,1,1,1, 1,1,1,1, 2,2,3,4};
-char* DisplayPiece(const char* next_byte_, int byte_length_) {
-  // Copy up to 8 UTF-8 chars to buffer
-  int k = 0;    // byte count
-  int n = 0;    // character count
-  for (int i = 0; i < byte_length_; ++i) {
-    char c = next_byte_[i];
-    if ((c & 0xc0) != 0x80) {
-      // Beginning of a UTF-8 character
-      int charlen = gCharlen[static_cast<uint8>(c) >> 4];
-      if (i + charlen > byte_length_) {break;} // Not enough room for full char
-      if (k >= (32 - 7)) {break;}   // Not necessarily enough room
-      if (n >= 8) {break;}          // Enough characters already
-      ++n;
-    }
-    if (c == '<') {
-      memcpy(&gDisplayPiece[k], "&lt;", 4); k += 4;
-    } else if (c == '>') {
-      memcpy(&gDisplayPiece[k], "&gt;", 4); k += 4;
-    } else if (c == '&') {
-      memcpy(&gDisplayPiece[k], "&amp;", 5); k += 5;
-    } else if (c == '\'') {
-      memcpy(&gDisplayPiece[k], "&apos;", 6); k += 6;
-    } else if (c == '"') {
-      memcpy(&gDisplayPiece[k], "&quot;", 6); k += 6;
-    } else {
-      gDisplayPiece[k++] = c;
-    }
-  }
-  gDisplayPiece[k++] = '\0';
-  return gDisplayPiece;
-}
-
-
-
-// runetochar copies (encodes) one rune, pointed to by r, to at most
-// UTFmax bytes starting at s and returns the number of bytes generated.
-int runetochar(char *str, const char32 *rune) {
-  // Convert to unsigned for range check.
-  unsigned long c;
-
-  // 1 char 00-7F
-  c = *rune;
-  if(c <= 0x7F) {
-    str[0] = c;
-    return 1;
-  }
-
-  // 2 char 0080-07FF
-  if(c <= 0x07FF) {
-    str[0] = 0xC0 | (c >> 1*6);
-    str[1] = 0x80 | (c & 0x3F);
-    return 2;
-  }
-
-  // Range check
-  if (c > Runemax) {
-    c = Runeerror;
-  }
-
-  // 3 char 0800-FFFF
-  if (c <= 0xFFFF) {
-    str[0] = 0xE0 |  (c >> 2*6);
-    str[1] = 0x80 | ((c >> 1*6) & 0x3F);
-    str[2] = 0x80 |  (c & 0x3F);
-    return 3;
-  }
-
-  // 4 char 10000-1FFFFF
-  str[0] = 0xF0 | (c >> 3*6);
-  str[1] = 0x80 | ((c >> 2*6) & 0x3F);
-  str[2] = 0x80 | ((c >> 1*6) & 0x3F);
-  str[3] = 0x80 | (c & 0x3F);
-  return 4;
-}
-
-
-
-// Useful for converting an entity to an ascii value.
-// RETURNS unicode value, or -1 if entity isn't valid.  Don't include & or ;
-int LookupEntity(const char* entity_name, int entity_len) {
-  // Make a C string
-  if (entity_len >= 16) {return -1;}    // All real entities are shorter
-  char temp[16];
-  memcpy(temp, entity_name, entity_len);
-  temp[entity_len] = '\0';
-  int match = BinarySearch(temp, 0, kNameToEntitySize, kNameToEntity);
-  if (match >= 0) {return kNameToEntity[match].i;}
-  return -1;
-}
-
-bool ascii_isdigit(char c) {
-  return ('0' <= c) && (c <= '9');
-}
-bool ascii_isxdigit(char c) {
-  if (('0' <= c) && (c <= '9')) {return true;}
-  if (('a' <= c) && (c <= 'f')) {return true;}
-  if (('A' <= c) && (c <= 'F')) {return true;}
-  return false;
-}
-bool ascii_isalnum(char c) {
-  if (('0' <= c) && (c <= '9')) {return true;}
-  if (('a' <= c) && (c <= 'z')) {return true;}
-  if (('A' <= c) && (c <= 'Z')) {return true;}
-  return false;
-}
-int hex_digit_to_int(char c) {
-  if (('0' <= c) && (c <= '9')) {return c - '0';}
-  if (('a' <= c) && (c <= 'f')) {return c - 'a' + 10;}
-  if (('A' <= c) && (c <= 'F')) {return c - 'A' + 10;}
-  return 0;
-}
-
-static int32 strto32_base10(const char* nptr, const char* limit,
-                            const char **endptr) {
-  *endptr = nptr;
-  while (nptr < limit && *nptr == '0') {
-    ++nptr;
-  }
-  if (nptr == limit || !ascii_isdigit(*nptr))
-    return -1;
-  const char* end_digits_run = nptr;
-  while (end_digits_run < limit && ascii_isdigit(*end_digits_run)) {
-    ++end_digits_run;
-  }
-  *endptr = end_digits_run;
-  const int num_digits = end_digits_run - nptr;
-  // kint32max == 2147483647.
-  if (num_digits < 9 ||
-      (num_digits == 10 && memcmp(nptr, "2147483647", 10) <= 0)) {
-    int value = 0;
-    for (; nptr < end_digits_run; ++nptr) {
-      value *= 10;
-      value += *nptr - '0';
-    }
-    // Overflow past the last valid unicode codepoint
-    // (0x10ffff) is converted to U+FFFD by FixUnicodeValue().
-    return FixUnicodeValue(value);
-  } else {
-    // Overflow: can't fit in an int32;
-    // returns the replacement character 0xFFFD.
-    return 0xFFFD;
-  }
-}
-
-static int32 strto32_base16(const char* nptr, const char* limit,
-                            const char **endptr) {
-  *endptr = nptr;
-  while (nptr < limit && *nptr == '0') {
-    ++nptr;
-  }
-  if (nptr == limit || !ascii_isxdigit(*nptr)) {
-    return -1;
-  }
-  const char* end_xdigits_run = nptr;
-  while (end_xdigits_run < limit && ascii_isxdigit(*end_xdigits_run)) {
-    ++end_xdigits_run;
-  }
-  *endptr = end_xdigits_run;
-  const int num_xdigits = end_xdigits_run - nptr;
-  // kint32max == 0x7FFFFFFF.
-  if (num_xdigits < 8 || (num_xdigits == 8 && nptr[0] < '8')) {
-    int value = 0;
-    for (; nptr < end_xdigits_run; ++nptr) {
-      value <<= 4;
-      value += hex_digit_to_int(*nptr);
-    }
-    // Overflow past the last valid unicode codepoint
-    // (0x10ffff) is converted to U+FFFD by FixUnicodeValue().
-    return FixUnicodeValue(value);
-  } else {
-    // Overflow: can't fit in an int32;
-    // returns the replacement character 0xFFFD.
-    return 0xFFFD;
-  }
-}
-
-// Unescape the current character pointed to by src.  SETS the number
-// of chars read for the conversion (in UTF8).  If src isn't a valid entity,
-// just consume the & and RETURN -1.  If src doesn't point to & -- which it
-// should -- set src_consumed to 0 and RETURN -1.
-int ReadEntity(const char* src, int srcn, int* src_consumed) {
-  const char* const srcend = src + srcn;
-
-  if (srcn == 0 || *src != '&') {      // input should start with an ampersand
-    *src_consumed = 0;
-    return -1;
-  }
-  *src_consumed = 1;                   // we'll get the & at least
-
-  // The standards are a bit unclear on when an entity ends.  Certainly a ";"
-  // ends one, but spaces probably do too.  We follow the lead of both IE and
-  // Netscape, which as far as we can tell end numeric entities (1st case below)
-  // at any non-digit, and end character entities (2nd case) at any non-alnum.
-  const char* entstart, *entend;  // where the entity starts and ends
-  entstart = src + 1;             // read past the &
-  int entval;                     // UCS2 value of the entity
-  if ( *entstart == '#' ) {       // -- 1st case: numeric entity
-    if ( entstart + 2 >= srcend ) {
-      return -1;                  // no way a legitimate number could fit
-    } else if ( entstart[1] == 'x' || entstart[1] == 'X' ) {   // hex numeric
-      entval = strto32_base16(entstart + 2, srcend, &entend);
-    } else {                                  // decimal numeric entity
-      entval = strto32_base10(entstart+1, srcend, &entend);
-    }
-    if (entval == -1 || entend > srcend) {
-      return -1;                 // not entirely correct, but close enough
-    }
-  } else {                       // -- 2nd case: character entity
-    for (entend = entstart;
-         entend < srcend && ascii_isalnum(*entend);
-         ++entend ) {
-      // entity consists of alphanumeric chars
-    }
-    entval = LookupEntity(entstart, entend - entstart);
-    if (entval < 0) {
-      return -1;  // not a legal entity name
-    }
-    // Now we do a strange-seeming IE6-compatibility check: if entval is
-    // >= 256, it *must* be followed by a semicolon or it's not considered
-    // an entity.  The problem is lots of the newfangled entity names, like
-    // "lang", also occur in URL CGI arguments: "/search?q=test&lang=en".
-    // When these links are written in HTML, it would be really bad if the
-    // "&lang" were treated as an entity, which is what the spec says
-    // *should* happen (even when the HTML is inside an "A HREF" tag!)
-    // IE ignores the spec for these new, high-value entities, so we do too.
-    if ( entval >= 256 && !(entend < srcend && *entend == ';') ) {
-      return -1;                 // make non-;-terminated entity illegal
-    }
-  }
-
-  // Finally, figure out how much src was consumed
-  if ( entend < srcend && *entend == ';' ) {
-    entend++;                    // standard says ; terminator is special
-  }
-  *src_consumed = entend - src;
-  return entval;
-}
-
-
-// Src points to '&'
-// Writes entity value to dst. Returns take(src), put(dst) byte counts
-void EntityToBuffer(const char* src, int len, char* dst,
-                    int* tlen, int* plen) {
-  char32 entval = ReadEntity(src, len, tlen);
-
-  // ReadEntity does this already: entval = FixUnicodeValue(entval);
-
-  // Convert UTF-32 to UTF-8
-  if (entval > 0) {
-    *plen = runetochar(dst, &entval);
-  } else {
-    // Illegal entity; ignore the '&'
-    *tlen = 1;
-    *plen = 0;
-  }
-}
-
-// Returns true if character is < > or &, none of which are letters
-bool inline IsSpecial(char c) {
-  if ((c & 0xe0) == 0x20) {
-    return kSpecialSymbol[static_cast<uint8>(c)];
-  }
-  return false;
-}
-
-// Quick Skip to next letter or < > & or to end of string (eos)
-// Always return is_letter for eos
-int ScanToLetterOrSpecial(const char* src, int len) {
-  int bytes_consumed;
-  StringPiece str(src, len);
-  UTF8GenericScan(&utf8scannot_lettermarkspecial_obj, str, &bytes_consumed);
-  return bytes_consumed;
-}
-
-
-
-
-// src points to non-letter, such as tag-opening '<'
-// Return length from here to next possible letter
-// On another < before >, return 1
-// advances <tag>
-//          |    |
-// advances <tag> ... </tag>  for <script> <style>
-//          |               |
-// advances <!-- ... <tag> ... -->
-//          |                     |
-// advances <tag
-//          |    | end of string
-// advances <tag <tag2>
-//          ||
-int ScanToPossibleLetter(const char* isrc, int len, int max_exit_state) {
-  const uint8* src = reinterpret_cast<const uint8*>(isrc);
-  const uint8* srclimit = src + len;
-  const uint8* tagParseTbl = kTagParseTbl_0;
-  int e = 0;
-  while (src < srclimit) {
-    e = tagParseTbl[kCharToSub[*src++]];
-    if (e <= max_exit_state) {
-      // We overshot by one byte
-      --src;
-      break;
-    }
-    tagParseTbl = &kTagParseTbl_0[e * 20];
-  }
-
-  if (src >= srclimit) {
-    // We fell off the end of the text.
-    // It looks like the most common case for this is a truncated file, not
-    // mismatched angle brackets. So we pretend that the last char was '>'
-    return len;
-  }
-
-  // OK to be in state 0 or state 2 at exit
-  if ((e != 0) && (e != 2)) {
-    // Error, '<' followed by '<'
-    // We want to back up to first <, then advance by one byte past it
-    int offset = src - reinterpret_cast<const uint8*>(isrc);
-
-    // Backscan to first '<' and return enough length to just get past it
-    --offset;   // back up over the second '<', which caused us to stop
-    while ((0 < offset) && (isrc[offset] != '<')) {
-      // Find the first '<', which is unmatched
-      --offset;
-    }
-    // skip to just beyond first '<'
-    return offset + 1;
-  }
-
-  return src - reinterpret_cast<const uint8*>(isrc);
-}
-
-
-ScriptScanner::ScriptScanner(const char* buffer,
-                             int buffer_length,
-                             bool is_plain_text)
-  : start_byte_(buffer),
-  next_byte_(buffer),
-  next_byte_limit_(buffer + buffer_length),
-  byte_length_(buffer_length),
-  is_plain_text_(is_plain_text),
-  letters_marks_only_(true),
-  one_script_only_(true),
-  exit_state_(kMaxExitStateLettersMarksOnly) {
-    script_buffer_ = new char[kMaxScriptBuffer];
-    script_buffer_lower_ = new char[kMaxScriptLowerBuffer];
-    map2original_.Clear();    // map from script_buffer_ to buffer
-    map2uplow_.Clear();       // map from script_buffer_lower_ to script_buffer_
-}
-
-// Extended version to allow spans of any non-tag text and spans of mixed script
-ScriptScanner::ScriptScanner(const char* buffer,
-                             int buffer_length,
-                             bool is_plain_text,
-                             bool any_text,
-                             bool any_script)
-  : start_byte_(buffer),
-  next_byte_(buffer),
-  next_byte_limit_(buffer + buffer_length),
-  byte_length_(buffer_length),
-  is_plain_text_(is_plain_text),
-  letters_marks_only_(!any_text),
-  one_script_only_(!any_script),
-  exit_state_(any_text ? kMaxExitStateAllText : kMaxExitStateLettersMarksOnly) {
-    script_buffer_ = new char[kMaxScriptBuffer];
-    script_buffer_lower_ = new char[kMaxScriptLowerBuffer];
-    map2original_.Clear();    // map from script_buffer_ to buffer
-    map2uplow_.Clear();       // map from script_buffer_lower_ to script_buffer_
-}
-
-
-ScriptScanner::~ScriptScanner() {
-  delete[] script_buffer_;
-  delete[] script_buffer_lower_;
-}
-
-
-
-
-// Get to the first real non-tag letter or entity that is a letter
-// Sets script of that letter
-// Return len if no more letters
-int ScriptScanner::SkipToFrontOfSpan(const char* src, int len, int* script) {
-  int sc = UNKNOWN_ULSCRIPT;
-  int skip = 0;
-  int tlen, plen;
-
-  // Do run of non-letters (tag | &NL | NL)*
-  tlen = 0;
-  while (skip < len) {
-    // Do fast scan to next interesting byte
-    // int oldskip = skip;
-    skip += ScanToLetterOrSpecial(src + skip, len - skip);
-
-    // Check for no more letters/specials
-    if (skip >= len) {
-      // All done
-      *script = sc;
-      return len;
-    }
-
-    // We are at a letter, nonletter, tag, or entity
-    if (IsSpecial(src[skip]) && !is_plain_text_) {
-      if (src[skip] == '<') {
-        // Begining of tag; skip to end and go around again
-        tlen = ScanToPossibleLetter(src + skip, len - skip,
-                                    exit_state_);
-        sc = 0;
-      } else if (src[skip] == '>') {
-        // Unexpected end of tag; skip it and go around again
-        tlen = 1;         // Over the >
-        sc = 0;
-      } else if (src[skip] == '&') {
-        // Expand entity, no advance
-        char temp[4];
-        EntityToBuffer(src + skip, len - skip,
-                       temp, &tlen, &plen);
-        sc = GetUTF8LetterScriptNum(temp);
-      }
-    } else {
-      // Update 1..4 bytes
-      tlen = UTF8OneCharLen(src + skip);
-      sc = GetUTF8LetterScriptNum(src + skip);
-    }
-    if (sc != 0) {break;}           // Letter found
-    skip += tlen;                   // Else advance
-  }
-
-  *script = sc;
-  return skip;
-}
-
-
-// These are for ASCII-only tag names
-// Compare one letter uplow to c, ignoring case of uplowp
-inline bool EqCase(char uplow, char c) {
-  return (uplow | 0x20) == c;
-}
-
-// These are for ASCII-only tag names
-// Return true for space / < > etc. all less than 0x40
-inline bool NeqLetter(char c) {
-  return c < 0x40;
-}
-
-// These are for ASCII-only tag names
-// Return true for space \n false for \r
-inline bool WS(char c) {
-  return (c == ' ') || (c == '\n');
-}
-
-// Canonical CR or LF
-static const char LF = '\n';
-
-
-// The naive loop scans from next_byte_ to script_buffer_ until full.
-// But this can leave an awkward hard-to-identify short fragment at the
-// end of the input. We would prefer to make the next-to-last fragment
-// shorter and the last fragment longer.
-
-// Copy next run of non-tag characters to buffer [NUL terminated]
-// This just replaces tags with space or \n and removes entities.
-// Tags <br> <p> and <tr> are replaced with \n. Non-letter sequences
-// including \r or \n are replaced by \n. All other tags and skipped text
-// are replaced with ASCII space.
-//
-// Buffer ALWAYS has leading space and trailing space space space NUL
-bool ScriptScanner::GetOneTextSpan(LangSpan* span) {
-  span->text = script_buffer_;
-  span->text_bytes = 0;
-  span->offset = next_byte_ - start_byte_;
-  span->ulscript = UNKNOWN_ULSCRIPT;
-  span->lang = UNKNOWN_LANGUAGE;
-  span->truncated = false;
-
-  int put_soft_limit = kMaxScriptBytes - kWithinScriptTail;
-  if ((kMaxScriptBytes <= byte_length_) &&
-      (byte_length_ < (2 * kMaxScriptBytes))) {
-    // Try to split the last two fragments in half
-    put_soft_limit = byte_length_ / 2;
-  }
-
-  script_buffer_[0] = ' ';  // Always a space at front of output
-  script_buffer_[1] = '\0';
-  int take = 0;
-  int put = 1;              // Start after the initial space
-  int tlen, plen;
-
-  if (byte_length_ <= 0) {
-    return false;          // No more text to be found
-  }
-
-  // Go over alternating spans of text and tags,
-  // copying letters to buffer with single spaces for each run of non-letters
-  bool last_byte_was_space = false;
-  while (take < byte_length_) {
-    char c = next_byte_[take];
-    if (c == '\r') {c = LF;}      // Canonical CR or LF
-    if (c == '\n') {c = LF;}      // Canonical CR or LF
-
-    if (IsSpecial(c) && !is_plain_text_) {
-      if (c == '<') {
-        // Replace tag with space
-        c = ' ';                      // for almost-full test below
-        // or if <p> <br> <tr>, replace with \n
-        if (take < (byte_length_ - 3)) {
-          if (EqCase(next_byte_[take + 1], 'p') &&
-              NeqLetter(next_byte_[take + 2])) {
-            c = LF;
-          }
-          if (EqCase(next_byte_[take + 1], 'b') &&
-              EqCase(next_byte_[take + 2], 'r') &&
-              NeqLetter(next_byte_[take + 3])) {
-            c = LF;
-          }
-          if (EqCase(next_byte_[take + 1], 't') &&
-              EqCase(next_byte_[take + 2], 'r') &&
-              NeqLetter(next_byte_[take + 3])) {
-            c = LF;
-          }
-        }
-        // Begining of tag; skip to end and go around again
-        tlen = 1 + ScanToPossibleLetter(next_byte_ + take, byte_length_ - take,
-                                    exit_state_);
-        // Copy one byte, compressing spaces
-        if (!last_byte_was_space || !WS(c)) {
-          script_buffer_[put++] = c;      // Advance dest
-          last_byte_was_space = WS(c);
-        }
-      } else if (c == '>') {
-        // Unexpected end of tag; copy it and go around again
-        tlen = 1;         // Over the >
-        script_buffer_[put++] = c;    // Advance dest
-      } else if (c == '&') {
-        // Expand entity, no advance
-        EntityToBuffer(next_byte_ + take, byte_length_ - take,
-                       script_buffer_ + put, &tlen, &plen);
-        put += plen;                  // Advance dest
-      }
-      take += tlen;                   // Advance source
-    } else {
-      // Copy one byte, compressing spaces
-      if (!last_byte_was_space || !WS(c)) {
-        script_buffer_[put++] = c;      // Advance dest
-        last_byte_was_space = WS(c);
-      }
-      ++take;                         // Advance source
-    }
-
-    if (WS(c) &&
-        (put >= put_soft_limit)) {
-      // Buffer is almost full
-      span->truncated = true;
-      break;
-    }
-    if (put >= kMaxScriptBytes) {
-      // Buffer is completely full
-      span->truncated = true;
-      break;
-    }
-  }
-
-  // Almost done. Back up to a character boundary if needed
-  while ((0 < take) && ((next_byte_[take] & 0xc0) == 0x80)) {
-    // Back up over continuation byte
-    --take;
-    --put;
-  }
-
-  // Update input position
-  next_byte_ += take;
-  byte_length_ -= take;
-
-  // Put four more spaces/NUL. Worst case is abcd _ _ _ \0
-  //                          kMaxScriptBytes |   | put
-  script_buffer_[put + 0] = ' ';
-  script_buffer_[put + 1] = ' ';
-  script_buffer_[put + 2] = ' ';
-  script_buffer_[put + 3] = '\0';
-
-  span->text_bytes = put;       // Does not include the last four chars above
-  return true;
-}
-
-
-// Copy next run of same-script non-tag letters to buffer [NUL terminated]
-// Buffer ALWAYS has leading space and trailing space space space NUL
-bool ScriptScanner::GetOneScriptSpan(LangSpan* span) {
-  if (!letters_marks_only_) {
-    // Return non-tag text, including punctuation and digits
-    return GetOneTextSpan(span);
-  }
-
-  span->text = script_buffer_;
-  span->text_bytes = 0;
-  span->offset = next_byte_ - start_byte_;
-  span->ulscript = UNKNOWN_ULSCRIPT;
-  span->lang = UNKNOWN_LANGUAGE;
-  span->truncated = false;
-
-  // struct timeval script_start, script_mid, script_end;
-
-  int put_soft_limit = kMaxScriptBytes - kWithinScriptTail;
-  if ((kMaxScriptBytes <= byte_length_) &&
-      (byte_length_ < (2 * kMaxScriptBytes))) {
-    // Try to split the last two fragments in half
-    put_soft_limit = byte_length_ / 2;
-  }
-
-
-  int spanscript;           // The script of this span
-  int sc = UNKNOWN_ULSCRIPT;  // The script of next character
-  int tlen = 0;
-  int plen = 0;
-
-  script_buffer_[0] = ' ';  // Always a space at front of output
-  script_buffer_[1] = '\0';
-  int take = 0;
-  int put = 1;              // Start after the initial space
-
-  // Build offsets from span->text back to start_byte_ + span->offset
-  // This mapping reflects deletion of non-letters, expansion of
-  // entities, etc.
-  map2original_.Clear();
-  map2original_.Delete(span->offset);   // So that MapBack(0) gives offset
-
-  // Get to the first real non-tag letter or entity that is a letter
-  int skip = SkipToFrontOfSpan(next_byte_, byte_length_, &spanscript);
-  next_byte_ += skip;
-  byte_length_ -= skip;
-
-  if (skip != 1) {
-    map2original_.Delete(skip);
-    map2original_.Insert(1);
-  } else {
-    map2original_.Copy(1);
-  }
-  if (byte_length_ <= 0) {
-    map2original_.Reset();
-    return false;               // No more letters to be found
-  }
-
-  // There is at least one letter, so we know the script for this span
-  span->ulscript = (ULScript)spanscript;
-
-
-  // Go over alternating spans of same-script letters and non-letters,
-  // copying letters to buffer with single spaces for each run of non-letters
-  while (take < byte_length_) {
-    // Copy run of letters in same script (&LS | LS)*
-    int letter_count = 0;              // Keep track of word length
-    bool need_break = false;
-
-    while (take < byte_length_) {
-      // We are at a letter, nonletter, tag, or entity
-      if (IsSpecial(next_byte_[take]) && !is_plain_text_) {
-        if (next_byte_[take] == '<') {
-          // Begining of tag
-          sc = 0;
-          break;
-        } else if (next_byte_[take] == '>') {
-          // Unexpected end of tag
-          sc = 0;
-          break;
-        } else if (next_byte_[take] == '&') {
-          // Copy entity, no advance
-          EntityToBuffer(next_byte_ + take, byte_length_ - take,
-                         script_buffer_ + put, &tlen, &plen);
-          sc = GetUTF8LetterScriptNum(script_buffer_ + put);
-        }
-      } else {
-        // Real letter, safely copy up to 4 bytes, increment by 1..4
-        // Will update by 1..4 bytes at Advance, below
-        tlen = plen = UTF8OneCharLen(next_byte_ + take);
-        if (take < (byte_length_ - 3)) {
-          // X86 fast case, does unaligned load/store
-          UNALIGNED_STORE32(script_buffer_ + put,
-                            UNALIGNED_LOAD32(next_byte_ + take));
-
-        } else {
-          // Slow case, happens 1-3 times per input document
-          memcpy(script_buffer_ + put, next_byte_ + take, plen);
-        }
-        sc = GetUTF8LetterScriptNum(next_byte_ + take);
-      }
-
-      // Allow continue across a single letter in a different script:
-      // A B D = three scripts, c = common script, i = inherited script,
-      // - = don't care, ( = take position before the += below
-      //  AAA(A-    continue
-      //
-      //  AAA(BA    continue
-      //  AAA(BB    break
-      //  AAA(Bc    continue (breaks after B)
-      //  AAA(BD    break
-      //  AAA(Bi    break
-      //
-      //  AAA(c-    break
-      //
-      //  AAA(i-    continue
-      //
-
-      if ((sc != spanscript) && (sc != ULScript_Inherited)) {
-        // Might need to break this script span
-        if (sc == ULScript_Common) {
-          need_break = true;
-        } else {
-          // Look at next following character, ignoring entity as Common
-          int sc2 = GetUTF8LetterScriptNum(next_byte_ + take + tlen);
-          if ((sc2 != ULScript_Common) && (sc2 != spanscript)) {
-            // We found a non-trivial change of script
-            if (one_script_only_) {
-              need_break = true;
-            }
-          }
-        }
-      }
-      if (need_break) {break;}  // Non-letter or letter in wrong script
-
-      take += tlen;                   // Advance
-      put += plen;                    // Advance
-
-      // Update the offset map to reflect take/put lengths
-      if (tlen == plen) {
-        map2original_.Copy(tlen);
-      } else if (tlen < plen) {
-        map2original_.Copy(tlen);
-        map2original_.Insert(plen - tlen);
-      } else {    // plen < tlen
-        map2original_.Copy(plen);
-        map2original_.Delete(tlen - plen);
-      }
-
-      ++letter_count;
-      if (put >= kMaxScriptBytes) {
-        // Buffer is full
-        span->truncated = true;
-        break;
-      }
-    }     // End while letters
-
-    // Do run of non-letters (tag | &NL | NL)*
-    while (take < byte_length_) {
-      // Do fast scan to next interesting byte
-      tlen = ScanToLetterOrSpecial(next_byte_ + take, byte_length_ - take);
-      take += tlen;
-      map2original_.Delete(tlen);
-      if (take >= byte_length_) {break;}    // Might have scanned to end
-
-      // We are at a letter, nonletter, tag, or entity
-      if (IsSpecial(next_byte_[take]) && !is_plain_text_) {
-        if (next_byte_[take] == '<') {
-          // Begining of tag; skip to end and go around again
-          tlen = ScanToPossibleLetter(next_byte_ + take, byte_length_ - take,
-                                      exit_state_);
-          sc = 0;
-        } else if (next_byte_[take] == '>') {
-          // Unexpected end of tag; skip it and go around again
-          tlen = 1;         // Over the >
-          sc = 0;
-        } else if (next_byte_[take] == '&') {
-          // Expand entity, no advance
-          EntityToBuffer(next_byte_ + take, byte_length_ - take,
-                         script_buffer_ + put, &tlen, &plen);
-          sc = GetUTF8LetterScriptNum(script_buffer_ + put);
-        }
-      } else {
-        // Update 1..4
-        tlen = UTF8OneCharLen(next_byte_ + take);
-        sc = GetUTF8LetterScriptNum(next_byte_ + take);
-      }
-      if (sc != 0) {break;}           // Letter found
-      take += tlen;                   // Else advance
-      map2original_.Delete(tlen);
-    }     // End while not-letters
-
-    script_buffer_[put++] = ' ';
-    map2original_.Insert(1);
-
-    // Letter in wrong script ?
-    if ((sc != spanscript) && (sc != ULScript_Inherited)) {break;}
-    if (put >= put_soft_limit) {
-      // Buffer is almost full
-      span->truncated = true;
-      break;
-    }
-  }
-
-  // Almost done. Back up to a character boundary if needed
-  while ((0 < take) && (take < byte_length_) &&
-         ((next_byte_[take] & 0xc0) == 0x80)) {
-    // Back up over continuation byte
-    --take;
-    --put;
-  }
-
-  // Update input position
-  next_byte_ += take;
-  byte_length_ -= take;
-
-  // Put four more spaces/NUL. Worst case is abcd _ _ _ \0
-  //                          kMaxScriptBytes |   | put
-  script_buffer_[put + 0] = ' ';
-  script_buffer_[put + 1] = ' ';
-  script_buffer_[put + 2] = ' ';
-  script_buffer_[put + 3] = '\0';
-  map2original_.Insert(4);
-  map2original_.Reset();
-
-  span->text_bytes = put;       // Does not include the last four chars above
-  return true;
-}
-
-// Force Latin, Cyrillic, Armenian, Greek scripts to be lowercase
-// List changes with each version of Unicode, so just always lowercase
-// Unicode 6.2.0:
-//   ARMENIAN COPTIC CYRILLIC DESERET GEORGIAN GLAGOLITIC GREEK LATIN
-void ScriptScanner::LowerScriptSpan(LangSpan* span) {
-  // If needed, lowercase all the text. If we do it sooner, might miss
-  // lowercasing an entity such as &Aacute;
-  // We only need to do this for Latn and Cyrl scripts
-  map2uplow_.Clear();
-  // Full Unicode lowercase of the entire buffer, including
-  // four pad bytes off the end.
-  // Ahhh. But the last byte 0x00 is not interchange-valid, so we do 3 pad
-  // bytes and put the 0x00 in explicitly.
-  // Build an offset map from script_buffer_lower_ back to script_buffer_
-  int consumed, filled, changed;
-  StringPiece istr(span->text, span->text_bytes + 3);
-  StringPiece ostr(script_buffer_lower_, kMaxScriptLowerBuffer);
-
-  UTF8GenericReplace(&utf8repl_lettermarklower_obj,
-                            istr, ostr, is_plain_text_,
-                            &consumed, &filled, &changed, &map2uplow_);
-  script_buffer_lower_[filled] = '\0';
-  span->text = script_buffer_lower_;
-  span->text_bytes = filled - 3;
-  map2uplow_.Reset();
-}
-
-// Copy next run of same-script non-tag letters to buffer [NUL terminated]
-// Force Latin, Cyrillic, Greek scripts to be lowercase
-// Buffer ALWAYS has leading space and trailing space space space NUL
-bool ScriptScanner::GetOneScriptSpanLower(LangSpan* span) {
-  bool ok = GetOneScriptSpan(span);
-  LowerScriptSpan(span);
-  return ok;
-}
-
-
-// Maps byte offset in most recent GetOneScriptSpan/Lower
-// span->text [0..text_bytes] into an additional byte offset from
-// span->offset, to get back to corresponding text in the original
-// input buffer.
-// text_offset must be the first byte
-// of a UTF-8 character, or just beyond the last character. Normally this
-// routine is called with the first byte of an interesting range and
-// again with the first byte of the following range.
-int ScriptScanner::MapBack(int text_offset) {
-  return map2original_.MapBack(map2uplow_.MapBack(text_offset));
-}
-
-
-// Gets lscript number for letters; always returns
-//   0 (common script) for non-letters
-int GetUTF8LetterScriptNum(const char* src) {
-  int srclen = UTF8OneCharLen(src);
-  const uint8* usrc = reinterpret_cast<const uint8*>(src);
-  return UTF8GenericPropertyTwoByte(&utf8prop_lettermarkscriptnum_obj,
-                                    &usrc, &srclen);
-}
-
-}  // namespace CLD2
-
-