path: root/browser/components/translation/cld2/internal/getonescriptspan.cc

// Copyright 2013 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//
// Author: dsites@google.com (Dick Sites)
//


#include "getonescriptspan.h"
#include <string.h>

#include "fixunicodevalue.h"
#include "lang_script.h"
#include "port.h"
#include "utf8statetable.h"

#include "utf8prop_lettermarkscriptnum.h"
#include "utf8repl_lettermarklower.h"
#include "utf8scannot_lettermarkspecial.h"


namespace CLD2 {

// Alphabetical order for binary search, from
// generated_entities.cc
extern const int kNameToEntitySize;
extern const CharIntPair kNameToEntity[];

static const int kMaxUpToWordBoundary = 50;       // span < this make longer,
                                                  // else make shorter
static const int kMaxAdvanceToWordBoundary = 10;  // +/- this many bytes
                                                  // to round to word boundary,
                                                  // direction above

static const char kSpecialSymbol[256] = {       // true for < > &
  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,1,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,1,0,1,0,
  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,

  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
};



#define LT 0      // <
#define GT 1      // >
#define EX 2      // !
#define HY 3      // -
#define QU 4      // "
#define AP 5      // '
#define SL 6      // /
#define S_ 7
#define C_ 8
#define R_ 9
#define I_ 10
#define P_ 11
#define T_ 12
#define Y_ 13
#define L_ 14
#define E_ 15
#define CR 16     // <cr> or <lf>
#define NL 17     // non-letter: ASCII whitespace, digit, punctuation
#define PL 18     // possible letter, incl. &
#define xx 19     // <unused>

// Map byte to one of ~20 interesting categories for cheap tag parsing
static const uint8 kCharToSub[256] = {
  NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,CR,NL, NL,CR,NL,NL,
  NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL,
  NL,EX,QU,NL, NL,NL,PL,AP, NL,NL,NL,NL, NL,HY,NL,SL,
  NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL, LT,NL,GT,NL,

  PL,PL,PL,C_, PL,E_,PL,PL, PL,I_,PL,PL, L_,PL,PL,PL,
  P_,PL,R_,S_, T_,PL,PL,PL, PL,Y_,PL,NL, NL,NL,NL,NL,
  PL,PL,PL,C_, PL,E_,PL,PL, PL,I_,PL,PL, L_,PL,PL,PL,
  P_,PL,R_,S_, T_,PL,PL,PL, PL,Y_,PL,NL, NL,NL,NL,NL,

  NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL,
  NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL,
  NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL,
  NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL,

  PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL,
  PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL,
  PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL,
  PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL,
};

#undef LT
#undef GT
#undef EX
#undef HY
#undef QU
#undef AP
#undef SL
#undef S_
#undef C_
#undef R_
#undef I_
#undef P_
#undef T_
#undef Y_
#undef L_
#undef E_
#undef CR
#undef NL
#undef PL
#undef xx


#define OK 0
#define X_ 1


static const int kMaxExitStateLettersMarksOnly = 1;
static const int kMaxExitStateAllText = 2;


// State machine to do cheap parse of non-letter strings incl. tags
// advances <tag>
//          |    |
// advances <tag> ... </tag>  for <script> <style>
//          |               |
// advances <!-- ... <tag> ... -->
//          |                     |
// advances <tag
//          ||  (0)
// advances <tag <tag2>
//          ||  (0)
//
// We start in state [0] at a non-letter and make at least one transition
// When scanning for just letters, arriving back at state [0] or [1] exits
//   the state machine.
// When scanning for any non-tag text, arriving at state [2] also exits
static const uint8 kTagParseTbl_0[] = {
// <  >  !  -   "  '  /  S   C  R  I  P   T  Y  L  E  CR NL PL xx
   3, 2, 2, 2,  2, 2, 2,OK, OK,OK,OK,OK, OK,OK,OK,OK,  2, 2,OK,X_, // [0] OK    exit state
  X_,X_,X_,X_, X_,X_,X_,X_, X_,X_,X_,X_, X_,X_,X_,X_, X_,X_,X_,X_, // [1] error exit state
   3, 2, 2, 2,  2, 2, 2,OK, OK,OK,OK,OK, OK,OK,OK,OK,  2, 2,OK,X_, // [2] NL*   [exit state]
  X_, 2, 4, 9, 10,11, 9,13,  9, 9, 9, 9,  9, 9, 9, 9,  9, 9, 9,X_, // [3] <
  X_, 2, 9, 5, 10,11, 9, 9,  9, 9, 9, 9,  9, 9, 9, 9,  9, 9, 9,X_, // [4] <!
  X_, 2, 9, 6, 10,11, 9, 9,  9, 9, 9, 9,  9, 9, 9, 9,  9, 9, 9,X_, // [5] <!-
   6, 6, 6, 7,  6, 6, 6, 6,  6, 6, 6, 6,  6, 6, 6, 6,  6, 6, 6,X_, // [6] <!--.*
   6, 6, 6, 8,  6, 6, 6, 6,  6, 6, 6, 6,  6, 6, 6, 6,  6, 6, 6,X_, // [7] <!--.*-
   6, 2, 6, 8,  6, 6, 6, 6,  6, 6, 6, 6,  6, 6, 6, 6,  6, 6, 6,X_, // [8] <!--.*--
  X_, 2, 9, 9, 10,11, 9, 9,  9, 9, 9, 9,  9, 9, 9, 9,  9, 9, 9,X_, // [9] <.*
  10,10,10,10,  9,10,10,10, 10,10,10,10, 10,10,10,10, 12,10,10,X_, // [10] <.*"
  11,11,11,11, 11, 9,11,11, 11,11,11,11, 11,11,11,11, 12,11,11,X_, // [11] <.*'
  X_, 2,12,12, 12,12,12,12, 12,12,12,12, 12,12,12,12, 12,12,12,X_, // [12] <.* no " '

// <  >  !  -   "  '  /  S   C  R  I  P   T  Y  L  E  CR NL PL xx
  X_, 2, 9, 9, 10,11, 9, 9, 14, 9, 9, 9, 28, 9, 9, 9,  9, 9, 9,X_, // [13] <S
  X_, 2, 9, 9, 10,11, 9, 9,  9,15, 9, 9,  9, 9, 9, 9,  9, 9, 9,X_, // [14] <SC
  X_, 2, 9, 9, 10,11, 9, 9,  9, 9,16, 9,  9, 9, 9, 9,  9, 9, 9,X_, // [15] <SCR
  X_, 2, 9, 9, 10,11, 9, 9,  9, 9, 9,17,  9, 9, 9, 9,  9, 9, 9,X_, // [16] <SCRI
  X_, 2, 9, 9, 10,11, 9, 9,  9, 9, 9, 9, 18, 9, 9, 9,  9, 9, 9,X_, // [17] <SCRIP
  X_,19, 9, 9, 10,11, 9, 9,  9, 9, 9, 9,  9, 9, 9, 9, 19,19, 9,X_, // [18] <SCRIPT
  20,19,19,19, 19,19,19,19, 19,19,19,19, 19,19,19,19, 19,19,19,X_, // [19] <SCRIPT .*
  19,19,19,19, 19,19,21,19, 19,19,19,19, 19,19,19,19, 19,19,19,X_, // [20] <SCRIPT .*<
  19,19,19,19, 19,19,19,22, 19,19,19,19, 19,19,19,19, 21,21,19,X_, // [21] <SCRIPT .*</ allow SP CR LF
  19,19,19,19, 19,19,19,19, 23,19,19,19, 19,19,19,19, 19,19,19,X_, // [22] <SCRIPT .*</S
  19,19,19,19, 19,19,19,19, 19,24,19,19, 19,19,19,19, 19,19,19,X_, // [23] <SCRIPT .*</SC
  19,19,19,19, 19,19,19,19, 19,19,25,19, 19,19,19,19, 19,19,19,X_, // [24] <SCRIPT .*</SCR
  19,19,19,19, 19,19,19,19, 19,19,19,26, 19,19,19,19, 19,19,19,X_, // [25] <SCRIPT .*</SCRI
  19,19,19,19, 19,19,19,19, 19,19,19,19, 27,19,19,19, 19,19,19,X_, // [26] <SCRIPT .*</SCRIP
  19, 2,19,19, 19,19,19,19, 19,19,19,19, 19,19,19,19, 19,19,19,X_, // [27] <SCRIPT .*</SCRIPT

// <  >  !  -   "  '  /  S   C  R  I  P   T  Y  L  E  CR NL PL xx
  X_, 2, 9, 9, 10,11, 9, 9,  9, 9, 9, 9,  9,29, 9, 9,  9, 9, 9,X_, // [28] <ST
  X_, 2, 9, 9, 10,11, 9, 9,  9, 9, 9, 9,  9, 9,30, 9,  9, 9, 9,X_, // [29] <STY
  X_, 2, 9, 9, 10,11, 9, 9,  9, 9, 9, 9,  9, 9, 9,31,  9, 9, 9,X_, // [30] <STYL
  X_,32, 9, 9, 10,11, 9, 9,  9, 9, 9, 9,  9, 9, 9, 9, 32,32, 9,X_, // [31] <STYLE
  33,32,32,32, 32,32,32,32, 32,32,32,32, 32,32,32,32, 32,32,32,X_, // [32] <STYLE .*
  32,32,32,32, 32,32,34,32, 32,32,32,32, 32,32,32,32, 32,32,32,X_, // [33] <STYLE .*<
  32,32,32,32, 32,32,32,35, 32,32,32,32, 32,32,32,32, 34,34,32,X_, // [34] <STYLE .*</ allow SP CR LF
  32,32,32,32, 32,32,32,32, 32,32,32,32, 36,32,32,32, 32,32,32,X_, // [35] <STYLE .*</S
  32,32,32,32, 32,32,32,32, 32,32,32,32, 32,37,32,32, 32,32,32,X_, // [36] <STYLE .*</ST
  32,32,32,32, 32,32,32,32, 32,32,32,32, 32,32,38,32, 32,32,32,X_, // [37] <STYLE .*</STY
  32,32,32,32, 32,32,32,32, 32,32,32,32, 32,32,32,39, 32,32,32,X_, // [38] <STYLE .*</STYL
  32, 2,32,32, 32,32,32,32, 32,32,32,32, 32,32,32,32, 32,32,32,X_, // [39] <STYLE .*</STYLE
};

#undef OK
#undef X_

enum
{
  UTFmax        = 4,            // maximum bytes per rune
  Runesync      = 0x80,         // cannot represent part of a UTF sequence (<)
  Runeself      = 0x80,         // rune and UTF sequences are the same (<)
  Runeerror     = 0xFFFD,       // decoding error in UTF
  Runemax       = 0x10FFFF,     // maximum rune value
};

// Debugging. Not thread safe.
static char gDisplayPiece[32];
const uint8 gCharlen[16] = {1,1,1,1, 1,1,1,1, 1,1,1,1, 2,2,3,4};
char* DisplayPiece(const char* next_byte_, int byte_length_) {
  // Copy up to 8 UTF-8 chars to buffer
  int k = 0;    // byte count
  int n = 0;    // character count
  for (int i = 0; i < byte_length_; ++i) {
    char c = next_byte_[i];
    if ((c & 0xc0) != 0x80) {
      // Beginning of a UTF-8 character
      int charlen = gCharlen[static_cast<uint8>(c) >> 4];
      if (i + charlen > byte_length_) {break;} // Not enough room for full char
      if (k >= (32 - 7)) {break;}   // Not necessarily enough room
      if (n >= 8) {break;}          // Enough characters already
      ++n;
    }
    if (c == '<') {
      memcpy(&gDisplayPiece[k], "&lt;", 4); k += 4;
    } else if (c == '>') {
      memcpy(&gDisplayPiece[k], "&gt;", 4); k += 4;
    } else if (c == '&') {
      memcpy(&gDisplayPiece[k], "&amp;", 5); k += 5;
    } else if (c == '\'') {
      memcpy(&gDisplayPiece[k], "&apos;", 6); k += 6;
    } else if (c == '"') {
      memcpy(&gDisplayPiece[k], "&quot;", 6); k += 6;
    } else {
      gDisplayPiece[k++] = c;
    }
  }
  gDisplayPiece[k++] = '\0';
  return gDisplayPiece;
}



// runetochar copies (encodes) one rune, pointed to by r, to at most
// UTFmax bytes starting at s and returns the number of bytes generated.
int runetochar(char *str, const char32 *rune) {
  // Convert to unsigned for range check.
  unsigned long c;

  // 1 char 00-7F
  c = *rune;
  if(c <= 0x7F) {
    str[0] = c;
    return 1;
  }

  // 2 char 0080-07FF
  if(c <= 0x07FF) {
    str[0] = 0xC0 | (c >> 1*6);
    str[1] = 0x80 | (c & 0x3F);
    return 2;
  }

  // Range check
  if (c > Runemax) {
    c = Runeerror;
  }

  // 3 char 0800-FFFF
  if (c <= 0xFFFF) {
    str[0] = 0xE0 |  (c >> 2*6);
    str[1] = 0x80 | ((c >> 1*6) & 0x3F);
    str[2] = 0x80 |  (c & 0x3F);
    return 3;
  }

  // 4 char 10000-1FFFFF
  str[0] = 0xF0 | (c >> 3*6);
  str[1] = 0x80 | ((c >> 2*6) & 0x3F);
  str[2] = 0x80 | ((c >> 1*6) & 0x3F);
  str[3] = 0x80 | (c & 0x3F);
  return 4;
}



// Useful for converting an entity to an ascii value.
// RETURNS unicode value, or -1 if entity isn't valid.  Don't include & or ;
int LookupEntity(const char* entity_name, int entity_len) {
  // Make a C string
  if (entity_len >= 16) {return -1;}    // All real entities are shorter
  char temp[16];
  memcpy(temp, entity_name, entity_len);
  temp[entity_len] = '\0';
  int match = BinarySearch(temp, 0, kNameToEntitySize, kNameToEntity);
  if (match >= 0) {return kNameToEntity[match].i;}
  return -1;
}

bool ascii_isdigit(char c) {
  return ('0' <= c) && (c <= '9');
}
bool ascii_isxdigit(char c) {
  if (('0' <= c) && (c <= '9')) {return true;}
  if (('a' <= c) && (c <= 'f')) {return true;}
  if (('A' <= c) && (c <= 'F')) {return true;}
  return false;
}
bool ascii_isalnum(char c) {
  if (('0' <= c) && (c <= '9')) {return true;}
  if (('a' <= c) && (c <= 'z')) {return true;}
  if (('A' <= c) && (c <= 'Z')) {return true;}
  return false;
}
int hex_digit_to_int(char c) {
  if (('0' <= c) && (c <= '9')) {return c - '0';}
  if (('a' <= c) && (c <= 'f')) {return c - 'a' + 10;}
  if (('A' <= c) && (c <= 'F')) {return c - 'A' + 10;}
  return 0;
}

static int32 strto32_base10(const char* nptr, const char* limit,
                            const char **endptr) {
  *endptr = nptr;
  while (nptr < limit && *nptr == '0') {
    ++nptr;
  }
  if (nptr == limit || !ascii_isdigit(*nptr))
    return -1;
  const char* end_digits_run = nptr;
  while (end_digits_run < limit && ascii_isdigit(*end_digits_run)) {
    ++end_digits_run;
  }
  *endptr = end_digits_run;
  const int num_digits = end_digits_run - nptr;
  // kint32max == 2147483647.
  if (num_digits < 9 ||
      (num_digits == 10 && memcmp(nptr, "2147483647", 10) <= 0)) {
    int value = 0;
    for (; nptr < end_digits_run; ++nptr) {
      value *= 10;
      value += *nptr - '0';
    }
    // Overflow past the last valid unicode codepoint
    // (0x10ffff) is converted to U+FFFD by FixUnicodeValue().
    return FixUnicodeValue(value);
  } else {
    // Overflow: can't fit in an int32;
    // returns the replacement character 0xFFFD.
    return 0xFFFD;
  }
}

static int32 strto32_base16(const char* nptr, const char* limit,
                            const char **endptr) {
  *endptr = nptr;
  while (nptr < limit && *nptr == '0') {
    ++nptr;
  }
  if (nptr == limit || !ascii_isxdigit(*nptr)) {
    return -1;
  }
  const char* end_xdigits_run = nptr;
  while (end_xdigits_run < limit && ascii_isxdigit(*end_xdigits_run)) {
    ++end_xdigits_run;
  }
  *endptr = end_xdigits_run;
  const int num_xdigits = end_xdigits_run - nptr;
  // kint32max == 0x7FFFFFFF.
  if (num_xdigits < 8 || (num_xdigits == 8 && nptr[0] < '8')) {
    int value = 0;
    for (; nptr < end_xdigits_run; ++nptr) {
      value <<= 4;
      value += hex_digit_to_int(*nptr);
    }
    // Overflow past the last valid unicode codepoint
    // (0x10ffff) is converted to U+FFFD by FixUnicodeValue().
    return FixUnicodeValue(value);
  } else {
    // Overflow: can't fit in an int32;
    // returns the replacement character 0xFFFD.
    return 0xFFFD;
  }
}

// Unescape the current character pointed to by src.  SETS the number
// of chars read for the conversion (in UTF8).  If src isn't a valid entity,
// just consume the & and RETURN -1.  If src doesn't point to & -- which it
// should -- set src_consumed to 0 and RETURN -1.
int ReadEntity(const char* src, int srcn, int* src_consumed) {
  const char* const srcend = src + srcn;

  if (srcn == 0 || *src != '&') {      // input should start with an ampersand
    *src_consumed = 0;
    return -1;
  }
  *src_consumed = 1;                   // we'll get the & at least

  // The standards are a bit unclear on when an entity ends.  Certainly a ";"
  // ends one, but spaces probably do too.  We follow the lead of both IE and
  // Netscape, which as far as we can tell end numeric entities (1st case below)
  // at any non-digit, and end character entities (2nd case) at any non-alnum.
  const char* entstart, *entend;  // where the entity starts and ends
  entstart = src + 1;             // read past the &
  int entval;                     // UCS2 value of the entity
  if ( *entstart == '#' ) {       // -- 1st case: numeric entity
    if ( entstart + 2 >= srcend ) {
      return -1;                  // no way a legitimate number could fit
    } else if ( entstart[1] == 'x' || entstart[1] == 'X' ) {   // hex numeric
      entval = strto32_base16(entstart + 2, srcend, &entend);
    } else {                                  // decimal numeric entity
      entval = strto32_base10(entstart+1, srcend, &entend);
    }
    if (entval == -1 || entend > srcend) {
      return -1;                 // not entirely correct, but close enough
    }
  } else {                       // -- 2nd case: character entity
    for (entend = entstart;
         entend < srcend && ascii_isalnum(*entend);
         ++entend ) {
      // entity consists of alphanumeric chars
    }
    entval = LookupEntity(entstart, entend - entstart);
    if (entval < 0) {
      return -1;  // not a legal entity name
    }
    // Now we do a strange-seeming IE6-compatibility check: if entval is
    // >= 256, it *must* be followed by a semicolon or it's not considered
    // an entity.  The problem is lots of the newfangled entity names, like
    // "lang", also occur in URL CGI arguments: "/search?q=test&lang=en".
    // When these links are written in HTML, it would be really bad if the
    // "&lang" were treated as an entity, which is what the spec says
    // *should* happen (even when the HTML is inside an "A HREF" tag!)
    // IE ignores the spec for these new, high-value entities, so we do too.
    if ( entval >= 256 && !(entend < srcend && *entend == ';') ) {
      return -1;                 // make non-;-terminated entity illegal
    }
  }

  // Finally, figure out how much src was consumed
  if ( entend < srcend && *entend == ';' ) {
    entend++;                    // standard says ; terminator is special
  }
  *src_consumed = entend - src;
  return entval;
}


// Src points to '&'
// Writes entity value to dst. Returns take(src), put(dst) byte counts
void EntityToBuffer(const char* src, int len, char* dst,
                    int* tlen, int* plen) {
  char32 entval = ReadEntity(src, len, tlen);

  // ReadEntity does this already: entval = FixUnicodeValue(entval);

  // Convert UTF-32 to UTF-8
  if (entval > 0) {
    *plen = runetochar(dst, &entval);
  } else {
    // Illegal entity; ignore the '&'
    *tlen = 1;
    *plen = 0;
  }
}

// Returns true if character is < > or &, none of which are letters
bool inline IsSpecial(char c) {
  if ((c & 0xe0) == 0x20) {
    return kSpecialSymbol[static_cast<uint8>(c)];
  }
  return false;
}

// Quick Skip to next letter or < > & or to end of string (eos)
// Always return is_letter for eos
int ScanToLetterOrSpecial(const char* src, int len) {
  int bytes_consumed;
  StringPiece str(src, len);
  UTF8GenericScan(&utf8scannot_lettermarkspecial_obj, str, &bytes_consumed);
  return bytes_consumed;
}




// src points to non-letter, such as tag-opening '<'
// Return length from here to next possible letter
// On another < before >, return 1
// advances <tag>
//          |    |
// advances <tag> ... </tag>  for <script> <style>
//          |               |
// advances <!-- ... <tag> ... -->
//          |                     |
// advances <tag
//          |    | end of string
// advances <tag <tag2>
//          ||
int ScanToPossibleLetter(const char* isrc, int len, int max_exit_state) {
  const uint8* src = reinterpret_cast<const uint8*>(isrc);
  const uint8* srclimit = src + len;
  const uint8* tagParseTbl = kTagParseTbl_0;
  int e = 0;
  while (src < srclimit) {
    e = tagParseTbl[kCharToSub[*src++]];
    if (e <= max_exit_state) {
      // We overshot by one byte
      --src;
      break;
    }
    tagParseTbl = &kTagParseTbl_0[e * 20];
  }

  if (src >= srclimit) {
    // We fell off the end of the text.
    // It looks like the most common case for this is a truncated file, not
    // mismatched angle brackets. So we pretend that the last char was '>'
    return len;
  }

  // OK to be in state 0 or state 2 at exit
  if ((e != 0) && (e != 2)) {
    // Error, '<' followed by '<'
    // We want to back up to first <, then advance by one byte past it
    int offset = src - reinterpret_cast<const uint8*>(isrc);

    // Backscan to first '<' and return enough length to just get past it
    --offset;   // back up over the second '<', which caused us to stop
    while ((0 < offset) && (isrc[offset] != '<')) {
      // Find the first '<', which is unmatched
      --offset;
    }
    // skip to just beyond first '<'
    return offset + 1;
  }

  return src - reinterpret_cast<const uint8*>(isrc);
}


ScriptScanner::ScriptScanner(const char* buffer,
                             int buffer_length,
                             bool is_plain_text)
  : start_byte_(buffer),
  next_byte_(buffer),
  next_byte_limit_(buffer + buffer_length),
  byte_length_(buffer_length),
  is_plain_text_(is_plain_text),
  letters_marks_only_(true),
  one_script_only_(true),
  exit_state_(kMaxExitStateLettersMarksOnly) {
    script_buffer_ = new char[kMaxScriptBuffer];
    script_buffer_lower_ = new char[kMaxScriptLowerBuffer];
    map2original_.Clear();    // map from script_buffer_ to buffer
    map2uplow_.Clear();       // map from script_buffer_lower_ to script_buffer_
}

// Extended version to allow spans of any non-tag text and spans of mixed script
ScriptScanner::ScriptScanner(const char* buffer,
                             int buffer_length,
                             bool is_plain_text,
                             bool any_text,
                             bool any_script)
  : start_byte_(buffer),
  next_byte_(buffer),
  next_byte_limit_(buffer + buffer_length),
  byte_length_(buffer_length),
  is_plain_text_(is_plain_text),
  letters_marks_only_(!any_text),
  one_script_only_(!any_script),
  exit_state_(any_text ? kMaxExitStateAllText : kMaxExitStateLettersMarksOnly) {
    script_buffer_ = new char[kMaxScriptBuffer];
    script_buffer_lower_ = new char[kMaxScriptLowerBuffer];
    map2original_.Clear();    // map from script_buffer_ to buffer
    map2uplow_.Clear();       // map from script_buffer_lower_ to script_buffer_
}


ScriptScanner::~ScriptScanner() {
  delete[] script_buffer_;
  delete[] script_buffer_lower_;
}




// Get to the first real non-tag letter or entity that is a letter
// Sets script of that letter
// Return len if no more letters
int ScriptScanner::SkipToFrontOfSpan(const char* src, int len, int* script) {
  int sc = UNKNOWN_ULSCRIPT;
  int skip = 0;
  int tlen, plen;

  // Do run of non-letters (tag | &NL | NL)*
  tlen = 0;
  while (skip < len) {
    // Do fast scan to next interesting byte
    // int oldskip = skip;
    skip += ScanToLetterOrSpecial(src + skip, len - skip);

    // Check for no more letters/specials
    if (skip >= len) {
      // All done
      *script = sc;
      return len;
    }

    // We are at a letter, nonletter, tag, or entity
    if (IsSpecial(src[skip]) && !is_plain_text_) {
      if (src[skip] == '<') {
        // Begining of tag; skip to end and go around again
        tlen = ScanToPossibleLetter(src + skip, len - skip,
                                    exit_state_);
        sc = 0;
      } else if (src[skip] == '>') {
        // Unexpected end of tag; skip it and go around again
        tlen = 1;         // Over the >
        sc = 0;
      } else if (src[skip] == '&') {
        // Expand entity, no advance
        char temp[4];
        EntityToBuffer(src + skip, len - skip,
                       temp, &tlen, &plen);
        sc = GetUTF8LetterScriptNum(temp);
      }
    } else {
      // Update 1..4 bytes
      tlen = UTF8OneCharLen(src + skip);
      sc = GetUTF8LetterScriptNum(src + skip);
    }
    if (sc != 0) {break;}           // Letter found
    skip += tlen;                   // Else advance
  }

  *script = sc;
  return skip;
}


// These are for ASCII-only tag names
// Compare one letter uplow to c, ignoring case of uplowp
inline bool EqCase(char uplow, char c) {
  return (uplow | 0x20) == c;
}

// These are for ASCII-only tag names
// Return true for space / < > etc. all less than 0x40
inline bool NeqLetter(char c) {
  return c < 0x40;
}

// These are for ASCII-only tag names
// Return true for space \n false for \r
inline bool WS(char c) {
  return (c == ' ') || (c == '\n');
}

// Canonical CR or LF
static const char LF = '\n';


// The naive loop scans from next_byte_ to script_buffer_ until full.
// But this can leave an awkward hard-to-identify short fragment at the
// end of the input. We would prefer to make the next-to-last fragment
// shorter and the last fragment longer.

// Copy next run of non-tag characters to buffer [NUL terminated]
// This just replaces tags with space or \n and removes entities.
// Tags <br> <p> and <tr> are replaced with \n. Non-letter sequences
// including \r or \n are replaced by \n. All other tags and skipped text
// are replaced with ASCII space.
//
// Buffer ALWAYS has leading space and trailing space space space NUL
bool ScriptScanner::GetOneTextSpan(LangSpan* span) {
  span->text = script_buffer_;
  span->text_bytes = 0;
  span->offset = next_byte_ - start_byte_;
  span->ulscript = UNKNOWN_ULSCRIPT;
  span->lang = UNKNOWN_LANGUAGE;
  span->truncated = false;

  int put_soft_limit = kMaxScriptBytes - kWithinScriptTail;
  if ((kMaxScriptBytes <= byte_length_) &&
      (byte_length_ < (2 * kMaxScriptBytes))) {
    // Try to split the last two fragments in half
    put_soft_limit = byte_length_ / 2;
  }

  script_buffer_[0] = ' ';  // Always a space at front of output
  script_buffer_[1] = '\0';
  int take = 0;
  int put = 1;              // Start after the initial space
  int tlen, plen;

  if (byte_length_ <= 0) {
    return false;          // No more text to be found
  }

  // Go over alternating spans of text and tags,
  // copying letters to buffer with single spaces for each run of non-letters
  bool last_byte_was_space = false;
  while (take < byte_length_) {
    char c = next_byte_[take];
    if (c == '\r') {c = LF;}      // Canonical CR or LF
    if (c == '\n') {c = LF;}      // Canonical CR or LF

    if (IsSpecial(c) && !is_plain_text_) {
      if (c == '<') {
        // Replace tag with space
        c = ' ';                      // for almost-full test below
        // or if <p> <br> <tr>, replace with \n
        if (take < (byte_length_ - 3)) {
          if (EqCase(next_byte_[take + 1], 'p') &&
              NeqLetter(next_byte_[take + 2])) {
            c = LF;
          }
          if (EqCase(next_byte_[take + 1], 'b') &&
              EqCase(next_byte_[take + 2], 'r') &&
              NeqLetter(next_byte_[take + 3])) {
            c = LF;
          }
          if (EqCase(next_byte_[take + 1], 't') &&
              EqCase(next_byte_[take + 2], 'r') &&
              NeqLetter(next_byte_[take + 3])) {
            c = LF;
          }
        }
        // Begining of tag; skip to end and go around again
        tlen = 1 + ScanToPossibleLetter(next_byte_ + take, byte_length_ - take,
                                    exit_state_);
        // Copy one byte, compressing spaces
        if (!last_byte_was_space || !WS(c)) {
          script_buffer_[put++] = c;      // Advance dest
          last_byte_was_space = WS(c);
        }
      } else if (c == '>') {
        // Unexpected end of tag; copy it and go around again
        tlen = 1;         // Over the >
        script_buffer_[put++] = c;    // Advance dest
      } else if (c == '&') {
        // Expand entity, no advance
        EntityToBuffer(next_byte_ + take, byte_length_ - take,
                       script_buffer_ + put, &tlen, &plen);
        put += plen;                  // Advance dest
      }
      take += tlen;                   // Advance source
    } else {
      // Copy one byte, compressing spaces
      if (!last_byte_was_space || !WS(c)) {
        script_buffer_[put++] = c;      // Advance dest
        last_byte_was_space = WS(c);
      }
      ++take;                         // Advance source
    }

    if (WS(c) &&
        (put >= put_soft_limit)) {
      // Buffer is almost full
      span->truncated = true;
      break;
    }
    if (put >= kMaxScriptBytes) {
      // Buffer is completely full
      span->truncated = true;
      break;
    }
  }

  // Almost done. Back up to a character boundary if needed
  while ((0 < take) && ((next_byte_[take] & 0xc0) == 0x80)) {
    // Back up over continuation byte
    --take;
    --put;
  }

  // Update input position
  next_byte_ += take;
  byte_length_ -= take;

  // Put four more spaces/NUL. Worst case is abcd _ _ _ \0
  //                          kMaxScriptBytes |   | put
  script_buffer_[put + 0] = ' ';
  script_buffer_[put + 1] = ' ';
  script_buffer_[put + 2] = ' ';
  script_buffer_[put + 3] = '\0';

  span->text_bytes = put;       // Does not include the last four chars above
  return true;
}


// Copy next run of same-script non-tag letters to buffer [NUL terminated]
// Buffer ALWAYS has leading space and trailing space space space NUL
bool ScriptScanner::GetOneScriptSpan(LangSpan* span) {
  if (!letters_marks_only_) {
    // Return non-tag text, including punctuation and digits
    return GetOneTextSpan(span);
  }

  span->text = script_buffer_;
  span->text_bytes = 0;
  span->offset = next_byte_ - start_byte_;
  span->ulscript = UNKNOWN_ULSCRIPT;
  span->lang = UNKNOWN_LANGUAGE;
  span->truncated = false;

  // struct timeval script_start, script_mid, script_end;

  int put_soft_limit = kMaxScriptBytes - kWithinScriptTail;
  if ((kMaxScriptBytes <= byte_length_) &&
      (byte_length_ < (2 * kMaxScriptBytes))) {
    // Try to split the last two fragments in half
    put_soft_limit = byte_length_ / 2;
  }


  int spanscript;           // The script of this span
  int sc = UNKNOWN_ULSCRIPT;  // The script of next character
  int tlen = 0;
  int plen = 0;

  script_buffer_[0] = ' ';  // Always a space at front of output
  script_buffer_[1] = '\0';
  int take = 0;
  int put = 1;              // Start after the initial space

  // Build offsets from span->text back to start_byte_ + span->offset
  // This mapping reflects deletion of non-letters, expansion of
  // entities, etc.
  map2original_.Clear();
  map2original_.Delete(span->offset);   // So that MapBack(0) gives offset

  // Get to the first real non-tag letter or entity that is a letter
  int skip = SkipToFrontOfSpan(next_byte_, byte_length_, &spanscript);
  next_byte_ += skip;
  byte_length_ -= skip;

  if (skip != 1) {
    map2original_.Delete(skip);
    map2original_.Insert(1);
  } else {
    map2original_.Copy(1);
  }
  if (byte_length_ <= 0) {
    map2original_.Reset();
    return false;               // No more letters to be found
  }

  // There is at least one letter, so we know the script for this span
  span->ulscript = (ULScript)spanscript;


  // Go over alternating spans of same-script letters and non-letters,
  // copying letters to buffer with single spaces for each run of non-letters
  while (take < byte_length_) {
    // Copy run of letters in same script (&LS | LS)*
    int letter_count = 0;              // Keep track of word length
    bool need_break = false;

    while (take < byte_length_) {
      // We are at a letter, nonletter, tag, or entity
      if (IsSpecial(next_byte_[take]) && !is_plain_text_) {
        if (next_byte_[take] == '<') {
          // Begining of tag
          sc = 0;
          break;
        } else if (next_byte_[take] == '>') {
          // Unexpected end of tag
          sc = 0;
          break;
        } else if (next_byte_[take] == '&') {
          // Copy entity, no advance
          EntityToBuffer(next_byte_ + take, byte_length_ - take,
                         script_buffer_ + put, &tlen, &plen);
          sc = GetUTF8LetterScriptNum(script_buffer_ + put);
        }
      } else {
        // Real letter, safely copy up to 4 bytes, increment by 1..4
        // Will update by 1..4 bytes at Advance, below
        tlen = plen = UTF8OneCharLen(next_byte_ + take);
        if (take < (byte_length_ - 3)) {
          // X86 fast case, does unaligned load/store
          UNALIGNED_STORE32(script_buffer_ + put,
                            UNALIGNED_LOAD32(next_byte_ + take));

        } else {
          // Slow case, happens 1-3 times per input document
          memcpy(script_buffer_ + put, next_byte_ + take, plen);
        }
        sc = GetUTF8LetterScriptNum(next_byte_ + take);
      }

      // Allow continue across a single letter in a different script:
      // A B D = three scripts, c = common script, i = inherited script,
      // - = don't care, ( = take position before the += below
      //  AAA(A-    continue
      //
      //  AAA(BA    continue
      //  AAA(BB    break
      //  AAA(Bc    continue (breaks after B)
      //  AAA(BD    break
      //  AAA(Bi    break
      //
      //  AAA(c-    break
      //
      //  AAA(i-    continue
      //

      if ((sc != spanscript) && (sc != ULScript_Inherited)) {
        // Might need to break this script span
        if (sc == ULScript_Common) {
          need_break = true;
        } else {
          // Look at next following character, ignoring entity as Common
          int sc2 = GetUTF8LetterScriptNum(next_byte_ + take + tlen);
          if ((sc2 != ULScript_Common) && (sc2 != spanscript)) {
            // We found a non-trivial change of script
            if (one_script_only_) {
              need_break = true;
            }
          }
        }
      }
      if (need_break) {break;}  // Non-letter or letter in wrong script

      take += tlen;                   // Advance
      put += plen;                    // Advance

      // Update the offset map to reflect take/put lengths
      if (tlen == plen) {
        map2original_.Copy(tlen);
      } else if (tlen < plen) {
        map2original_.Copy(tlen);
        map2original_.Insert(plen - tlen);
      } else {    // plen < tlen
        map2original_.Copy(plen);
        map2original_.Delete(tlen - plen);
      }

      ++letter_count;
      if (put >= kMaxScriptBytes) {
        // Buffer is full
        span->truncated = true;
        break;
      }
    }     // End while letters

    // Do run of non-letters (tag | &NL | NL)*
    while (take < byte_length_) {
      // Do fast scan to next interesting byte
      tlen = ScanToLetterOrSpecial(next_byte_ + take, byte_length_ - take);
      take += tlen;
      map2original_.Delete(tlen);
      if (take >= byte_length_) {break;}    // Might have scanned to end

      // We are at a letter, nonletter, tag, or entity
      if (IsSpecial(next_byte_[take]) && !is_plain_text_) {
        if (next_byte_[take] == '<') {
          // Begining of tag; skip to end and go around again
          tlen = ScanToPossibleLetter(next_byte_ + take, byte_length_ - take,
                                      exit_state_);
          sc = 0;
        } else if (next_byte_[take] == '>') {
          // Unexpected end of tag; skip it and go around again
          tlen = 1;         // Over the >
          sc = 0;
        } else if (next_byte_[take] == '&') {
          // Expand entity, no advance
          EntityToBuffer(next_byte_ + take, byte_length_ - take,
                         script_buffer_ + put, &tlen, &plen);
          sc = GetUTF8LetterScriptNum(script_buffer_ + put);
        }
      } else {
        // Update 1..4
        tlen = UTF8OneCharLen(next_byte_ + take);
        sc = GetUTF8LetterScriptNum(next_byte_ + take);
      }
      if (sc != 0) {break;}           // Letter found
      take += tlen;                   // Else advance
      map2original_.Delete(tlen);
    }     // End while not-letters

    script_buffer_[put++] = ' ';
    map2original_.Insert(1);

    // Letter in wrong script ?
    if ((sc != spanscript) && (sc != ULScript_Inherited)) {break;}
    if (put >= put_soft_limit) {
      // Buffer is almost full
      span->truncated = true;
      break;
    }
  }

  // Almost done. Back up to a character boundary if needed
  while ((0 < take) && (take < byte_length_) &&
         ((next_byte_[take] & 0xc0) == 0x80)) {
    // Back up over continuation byte
    --take;
    --put;
  }

  // Update input position
  next_byte_ += take;
  byte_length_ -= take;

  // Put four more spaces/NUL. Worst case is abcd _ _ _ \0
  //                          kMaxScriptBytes |   | put
  script_buffer_[put + 0] = ' ';
  script_buffer_[put + 1] = ' ';
  script_buffer_[put + 2] = ' ';
  script_buffer_[put + 3] = '\0';
  map2original_.Insert(4);
  map2original_.Reset();

  span->text_bytes = put;       // Does not include the last four chars above
  return true;
}

// Force Latin, Cyrillic, Armenian, Greek scripts to be lowercase
// List changes with each version of Unicode, so just always lowercase
// Unicode 6.2.0:
//   ARMENIAN COPTIC CYRILLIC DESERET GEORGIAN GLAGOLITIC GREEK LATIN
void ScriptScanner::LowerScriptSpan(LangSpan* span) {
  // If needed, lowercase all the text. If we do it sooner, might miss
  // lowercasing an entity such as &Aacute;
  // We only need to do this for Latn and Cyrl scripts
  map2uplow_.Clear();
  // Full Unicode lowercase of the entire buffer, including
  // four pad bytes off the end.
  // Ahhh. But the last byte 0x00 is not interchange-valid, so we do 3 pad
  // bytes and put the 0x00 in explicitly.
  // Build an offset map from script_buffer_lower_ back to script_buffer_
  int consumed, filled, changed;
  StringPiece istr(span->text, span->text_bytes + 3);
  StringPiece ostr(script_buffer_lower_, kMaxScriptLowerBuffer);

  UTF8GenericReplace(&utf8repl_lettermarklower_obj,
                            istr, ostr, is_plain_text_,
                            &consumed, &filled, &changed, &map2uplow_);
  script_buffer_lower_[filled] = '\0';
  span->text = script_buffer_lower_;
  span->text_bytes = filled - 3;
  map2uplow_.Reset();
}

// Copy next run of same-script non-tag letters to buffer [NUL terminated]
// Force Latin, Cyrillic, Greek scripts to be lowercase
// Buffer ALWAYS has leading space and trailing space space space NUL
bool ScriptScanner::GetOneScriptSpanLower(LangSpan* span) {
  bool ok = GetOneScriptSpan(span);
  LowerScriptSpan(span);
  return ok;
}


// Maps byte offset in most recent GetOneScriptSpan/Lower
// span->text [0..text_bytes] into an additional byte offset from
// span->offset, to get back to corresponding text in the original
// input buffer.
// text_offset must be the first byte
// of a UTF-8 character, or just beyond the last character. Normally this
// routine is called with the first byte of an interesting range and
// again with the first byte of the following range.
int ScriptScanner::MapBack(int text_offset) {
  return map2original_.MapBack(map2uplow_.MapBack(text_offset));
}


// Gets lscript number for letters; always returns
//   0 (common script) for non-letters
int GetUTF8LetterScriptNum(const char* src) {
  int srclen = UTF8OneCharLen(src);
  const uint8* usrc = reinterpret_cast<const uint8*>(src);
  return UTF8GenericPropertyTwoByte(&utf8prop_lettermarkscriptnum_obj,
                                    &usrc, &srclen);
}

}  // namespace CLD2