Add m-esr52 at 52.6.0

1 files changed, 2192 insertions, 0 deletions

diff --git a/extensions/spellcheck/hunspell/src/suggestmgr.cxx b/extensions/spellcheck/hunspell/src/suggestmgr.cxx
new file mode 100644
index 000000000..17becd758
--- /dev/null
+++ b/extensions/spellcheck/hunspell/src/suggestmgr.cxx
@@ -0,0 +1,2192 @@
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is Hunspell, based on MySpell.
+ *
+ * The Initial Developers of the Original Code are
+ * Kevin Hendricks (MySpell) and Németh László (Hunspell).
+ * Portions created by the Initial Developers are Copyright (C) 2002-2005
+ * the Initial Developers. All Rights Reserved.
+ *
+ * Contributor(s): David Einstein, Davide Prina, Giuseppe Modugno,
+ * Gianluca Turconi, Simon Brouwer, Noll János, Bíró Árpád,
+ * Goldman Eleonóra, Sarlós Tamás, Bencsáth Boldizsár, Halácsy Péter,
+ * Dvornik László, Gefferth András, Nagy Viktor, Varga Dániel, Chris Halls,
+ * Rene Engelhard, Bram Moolenaar, Dafydd Jones, Harri Pitkänen
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+/*
+ * Copyright 2002 Kevin B. Hendricks, Stratford, Ontario, Canada
+ * And Contributors.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * 3. All modifications to the source code must be clearly marked as
+ *    such.  Binary redistributions based on modified source code
+ *    must be clearly marked as modified versions in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY KEVIN B. HENDRICKS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL
+ * KEVIN B. HENDRICKS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <ctype.h>
+
+#include "suggestmgr.hxx"
+#include "htypes.hxx"
+#include "csutil.hxx"
+
+const w_char W_VLINE = {'\0', '|'};
+
+SuggestMgr::SuggestMgr(const char* tryme, int maxn, AffixMgr* aptr) {
+  // register affix manager and check in string of chars to
+  // try when building candidate suggestions
+  pAMgr = aptr;
+
+  csconv = NULL;
+
+  ckeyl = 0;
+  ckey = NULL;
+  ckey_utf = NULL;
+
+  ctryl = 0;
+  ctry = NULL;
+  ctry_utf = NULL;
+
+  utf8 = 0;
+  langnum = 0;
+  complexprefixes = 0;
+
+  maxSug = maxn;
+  nosplitsugs = 0;
+  maxngramsugs = MAXNGRAMSUGS;
+  maxcpdsugs = MAXCOMPOUNDSUGS;
+
+  if (pAMgr) {
+    langnum = pAMgr->get_langnum();
+    ckey = pAMgr->get_key_string();
+    nosplitsugs = pAMgr->get_nosplitsugs();
+    if (pAMgr->get_maxngramsugs() >= 0)
+      maxngramsugs = pAMgr->get_maxngramsugs();
+    utf8 = pAMgr->get_utf8();
+    if (pAMgr->get_maxcpdsugs() >= 0)
+      maxcpdsugs = pAMgr->get_maxcpdsugs();
+    if (!utf8) {
+      char* enc = pAMgr->get_encoding();
+      csconv = get_current_cs(enc);
+      free(enc);
+    }
+    complexprefixes = pAMgr->get_complexprefixes();
+  }
+
+  if (ckey) {
+    if (utf8) {
+      std::vector<w_char> t;
+      ckeyl = u8_u16(t, ckey);
+      ckey_utf = (w_char*)malloc(ckeyl * sizeof(w_char));
+      if (ckey_utf)
+        memcpy(ckey_utf, &t[0], ckeyl * sizeof(w_char));
+      else
+        ckeyl = 0;
+    } else {
+      ckeyl = strlen(ckey);
+    }
+  }
+
+  if (tryme) {
+    ctry = mystrdup(tryme);
+    if (ctry)
+      ctryl = strlen(ctry);
+    if (ctry && utf8) {
+      std::vector<w_char> t;
+      ctryl = u8_u16(t, tryme);
+      ctry_utf = (w_char*)malloc(ctryl * sizeof(w_char));
+      if (ctry_utf)
+        memcpy(ctry_utf, &t[0], ctryl * sizeof(w_char));
+      else
+        ctryl = 0;
+    }
+  }
+}
+
+SuggestMgr::~SuggestMgr() {
+  pAMgr = NULL;
+  if (ckey)
+    free(ckey);
+  ckey = NULL;
+  if (ckey_utf)
+    free(ckey_utf);
+  ckey_utf = NULL;
+  ckeyl = 0;
+  if (ctry)
+    free(ctry);
+  ctry = NULL;
+  if (ctry_utf)
+    free(ctry_utf);
+  ctry_utf = NULL;
+  ctryl = 0;
+  maxSug = 0;
+#ifdef MOZILLA_CLIENT
+  delete[] csconv;
+#endif
+}
+
+int SuggestMgr::testsug(char** wlst,
+                        const char* candidate,
+                        int wl,
+                        int ns,
+                        int cpdsuggest,
+                        int* timer,
+                        clock_t* timelimit) {
+  int cwrd = 1;
+  if (ns == maxSug)
+    return maxSug;
+  for (int k = 0; k < ns; k++) {
+    if (strcmp(candidate, wlst[k]) == 0) {
+      cwrd = 0;
+      break;
+    }
+  }
+  if ((cwrd) && checkword(candidate, wl, cpdsuggest, timer, timelimit)) {
+    wlst[ns] = mystrdup(candidate);
+    if (wlst[ns] == NULL) {
+      for (int j = 0; j < ns; j++)
+        free(wlst[j]);
+      return -1;
+    }
+    ns++;
+  }
+  return ns;
+}
+
+// generate suggestions for a misspelled word
+//    pass in address of array of char * pointers
+// onlycompoundsug: probably bad suggestions (need for ngram sugs, too)
+
+int SuggestMgr::suggest(char*** slst,
+                        const char* w,
+                        int nsug,
+                        int* onlycompoundsug) {
+  int nocompoundtwowords = 0;
+  char** wlst;
+  std::vector<w_char> word_utf;
+  int wl = 0;
+  int nsugorig = nsug;
+  std::string w2;
+  const char* word = w;
+  int oldSug = 0;
+
+  // word reversing wrapper for complex prefixes
+  if (complexprefixes) {
+    w2.assign(w);
+    if (utf8)
+      reverseword_utf(w2);
+    else
+      reverseword(w2);
+    word = w2.c_str();
+  }
+
+  if (*slst) {
+    wlst = *slst;
+  } else {
+    wlst = (char**)malloc(maxSug * sizeof(char*));
+    if (wlst == NULL)
+      return -1;
+    for (int i = 0; i < maxSug; i++) {
+      wlst[i] = NULL;
+    }
+  }
+
+  if (utf8) {
+    wl = u8_u16(word_utf, word);
+    if (wl == -1) {
+      *slst = wlst;
+      return nsug;
+    }
+  }
+
+  for (int cpdsuggest = 0; (cpdsuggest < 2) && (nocompoundtwowords == 0);
+       cpdsuggest++) {
+    // limit compound suggestion
+    if (cpdsuggest > 0)
+      oldSug = nsug;
+
+    // suggestions for an uppercase word (html -> HTML)
+    if ((nsug < maxSug) && (nsug > -1)) {
+      nsug = (utf8) ? capchars_utf(wlst, &word_utf[0], wl, nsug, cpdsuggest)
+                    : capchars(wlst, word, nsug, cpdsuggest);
+    }
+
+    // perhaps we made a typical fault of spelling
+    if ((nsug < maxSug) && (nsug > -1) &&
+        (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) {
+      nsug = replchars(wlst, word, nsug, cpdsuggest);
+    }
+
+    // perhaps we made chose the wrong char from a related set
+    if ((nsug < maxSug) && (nsug > -1) &&
+        (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) {
+      nsug = mapchars(wlst, word, nsug, cpdsuggest);
+    }
+
+    // only suggest compound words when no other suggestion
+    if ((cpdsuggest == 0) && (nsug > nsugorig))
+      nocompoundtwowords = 1;
+
+    // did we swap the order of chars by mistake
+    if ((nsug < maxSug) && (nsug > -1) &&
+        (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) {
+      nsug = (utf8) ? swapchar_utf(wlst, &word_utf[0], wl, nsug, cpdsuggest)
+                    : swapchar(wlst, word, nsug, cpdsuggest);
+    }
+
+    // did we swap the order of non adjacent chars by mistake
+    if ((nsug < maxSug) && (nsug > -1) &&
+        (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) {
+      nsug = (utf8) ? longswapchar_utf(wlst, &word_utf[0], wl, nsug, cpdsuggest)
+                    : longswapchar(wlst, word, nsug, cpdsuggest);
+    }
+
+    // did we just hit the wrong key in place of a good char (case and keyboard)
+    if ((nsug < maxSug) && (nsug > -1) &&
+        (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) {
+      nsug = (utf8) ? badcharkey_utf(wlst, &word_utf[0], wl, nsug, cpdsuggest)
+                    : badcharkey(wlst, word, nsug, cpdsuggest);
+    }
+
+    // did we add a char that should not be there
+    if ((nsug < maxSug) && (nsug > -1) &&
+        (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) {
+      nsug = (utf8) ? extrachar_utf(wlst, &word_utf[0], wl, nsug, cpdsuggest)
+                    : extrachar(wlst, word, nsug, cpdsuggest);
+    }
+
+    // did we forgot a char
+    if ((nsug < maxSug) && (nsug > -1) &&
+        (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) {
+      nsug = (utf8) ? forgotchar_utf(wlst, &word_utf[0], wl, nsug, cpdsuggest)
+                    : forgotchar(wlst, word, nsug, cpdsuggest);
+    }
+
+    // did we move a char
+    if ((nsug < maxSug) && (nsug > -1) &&
+        (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) {
+      nsug = (utf8) ? movechar_utf(wlst, &word_utf[0], wl, nsug, cpdsuggest)
+                    : movechar(wlst, word, nsug, cpdsuggest);
+    }
+
+    // did we just hit the wrong key in place of a good char
+    if ((nsug < maxSug) && (nsug > -1) &&
+        (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) {
+      nsug = (utf8) ? badchar_utf(wlst, &word_utf[0], wl, nsug, cpdsuggest)
+                    : badchar(wlst, word, nsug, cpdsuggest);
+    }
+
+    // did we double two characters
+    if ((nsug < maxSug) && (nsug > -1) &&
+        (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) {
+      nsug = (utf8) ? doubletwochars_utf(wlst, &word_utf[0], wl, nsug, cpdsuggest)
+                    : doubletwochars(wlst, word, nsug, cpdsuggest);
+    }
+
+    // perhaps we forgot to hit space and two words ran together
+    if (!nosplitsugs && (nsug < maxSug) && (nsug > -1) &&
+        (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) {
+      nsug = twowords(wlst, word, nsug, cpdsuggest);
+    }
+
+  }  // repeating ``for'' statement compounding support
+
+  if (nsug < 0) {
+    // we ran out of memory - we should free up as much as possible
+    for (int i = 0; i < maxSug; i++)
+      if (wlst[i] != NULL)
+        free(wlst[i]);
+    free(wlst);
+    wlst = NULL;
+  }
+
+  if (!nocompoundtwowords && (nsug > 0) && onlycompoundsug)
+    *onlycompoundsug = 1;
+
+  *slst = wlst;
+  return nsug;
+}
+
+// suggestions for an uppercase word (html -> HTML)
+int SuggestMgr::capchars_utf(char** wlst,
+                             const w_char* word,
+                             int wl,
+                             int ns,
+                             int cpdsuggest) {
+  std::vector<w_char> candidate_utf(word, word + wl);
+  mkallcap_utf(candidate_utf, langnum);
+  std::string candidate;
+  u16_u8(candidate, candidate_utf);
+  return testsug(wlst, candidate.c_str(), candidate.size(), ns, cpdsuggest, NULL,
+                 NULL);
+}
+
+// suggestions for an uppercase word (html -> HTML)
+int SuggestMgr::capchars(char** wlst,
+                         const char* word,
+                         int ns,
+                         int cpdsuggest) {
+  std::string candidate(word);
+  mkallcap(candidate, csconv);
+  return testsug(wlst, candidate.c_str(), candidate.size(), ns, cpdsuggest, NULL,
+                 NULL);
+}
+
+// suggestions for when chose the wrong char out of a related set
+int SuggestMgr::mapchars(char** wlst,
+                         const char* word,
+                         int ns,
+                         int cpdsuggest) {
+  std::string candidate;
+  clock_t timelimit;
+  int timer;
+
+  int wl = strlen(word);
+  if (wl < 2 || !pAMgr)
+    return ns;
+
+  int nummap = pAMgr->get_nummap();
+  struct mapentry* maptable = pAMgr->get_maptable();
+  if (maptable == NULL)
+    return ns;
+
+  timelimit = clock();
+  timer = MINTIMER;
+  return map_related(word, candidate, 0, wlst, cpdsuggest, ns,
+                     maptable, nummap, &timer, &timelimit);
+}
+
+int SuggestMgr::map_related(const char* word,
+                            std::string& candidate,
+                            int wn,
+                            char** wlst,
+                            int cpdsuggest,
+                            int ns,
+                            const mapentry* maptable,
+                            int nummap,
+                            int* timer,
+                            clock_t* timelimit) {
+  if (*(word + wn) == '\0') {
+    int cwrd = 1;
+    for (int m = 0; m < ns; m++) {
+      if (candidate == wlst[m]) {
+        cwrd = 0;
+        break;
+      }
+    }
+    if ((cwrd) && checkword(candidate.c_str(), candidate.size(), cpdsuggest, timer, timelimit)) {
+      if (ns < maxSug) {
+        wlst[ns] = mystrdup(candidate.c_str());
+        if (wlst[ns] == NULL)
+          return -1;
+        ns++;
+      }
+    }
+    return ns;
+  }
+  int in_map = 0;
+  for (int j = 0; j < nummap; j++) {
+    for (int k = 0; k < maptable[j].len; k++) {
+      int len = strlen(maptable[j].set[k]);
+      if (strncmp(maptable[j].set[k], word + wn, len) == 0) {
+        in_map = 1;
+        size_t cn = candidate.size();
+        for (int l = 0; l < maptable[j].len; l++) {
+          candidate.resize(cn);
+          candidate.append(maptable[j].set[l]);
+          ns = map_related(word, candidate, wn + len, wlst,
+                           cpdsuggest, ns, maptable, nummap, timer, timelimit);
+          if (!(*timer))
+            return ns;
+        }
+      }
+    }
+  }
+  if (!in_map) {
+    candidate.push_back(*(word + wn));
+    ns = map_related(word, candidate, wn + 1, wlst, cpdsuggest, ns,
+                     maptable, nummap, timer, timelimit);
+  }
+  return ns;
+}
+
+// suggestions for a typical fault of spelling, that
+// differs with more, than 1 letter from the right form.
+int SuggestMgr::replchars(char** wlst,
+                          const char* word,
+                          int ns,
+                          int cpdsuggest) {
+  std::string candidate;
+  int wl = strlen(word);
+  if (wl < 2 || !pAMgr)
+    return ns;
+  int numrep = pAMgr->get_numrep();
+  struct replentry* reptable = pAMgr->get_reptable();
+  if (reptable == NULL)
+    return ns;
+  for (int i = 0; i < numrep; i++) {
+    const char* r = word;
+    // search every occurence of the pattern in the word
+    while ((r = strstr(r, reptable[i].pattern)) != NULL &&
+           (!reptable[i].end || strlen(r) == strlen(reptable[i].pattern)) &&
+           (!reptable[i].start || r == word)) {
+      candidate.assign(word);
+      candidate.resize(r - word);
+      candidate.append(reptable[i].pattern2);
+      int lenp = strlen(reptable[i].pattern);
+      candidate.append(r + lenp);
+      ns = testsug(wlst, candidate.c_str(), candidate.size(), ns, cpdsuggest, NULL,
+                   NULL);
+      if (ns == -1)
+        return -1;
+      // check REP suggestions with space
+      size_t sp = candidate.find(' ');
+      if (sp != std::string::npos) {
+        size_t prev = 0;
+        while (sp != std::string::npos) {
+          std::string prev_chunk = candidate.substr(prev, sp - prev);
+          if (checkword(prev_chunk.c_str(), prev_chunk.size(), 0, NULL, NULL)) {
+            int oldns = ns;
+            std::string post_chunk = candidate.substr(sp + 1);
+            ns = testsug(wlst, post_chunk.c_str(), post_chunk.size(), ns, cpdsuggest, NULL,
+                         NULL);
+            if (ns == -1)
+              return -1;
+            if (oldns < ns) {
+              free(wlst[ns - 1]);
+              wlst[ns - 1] = mystrdup(candidate.c_str());
+              if (!wlst[ns - 1])
+                return -1;
+            }
+          }
+          prev = sp + 1;
+          sp = candidate.find(' ', prev);
+        }
+      }
+      r++;  // search for the next letter
+    }
+  }
+  return ns;
+}
+
+// perhaps we doubled two characters (pattern aba -> ababa, for example vacation
+// -> vacacation)
+int SuggestMgr::doubletwochars(char** wlst,
+                               const char* word,
+                               int ns,
+                               int cpdsuggest) {
+  int state = 0;
+  int wl = strlen(word);
+  if (wl < 5 || !pAMgr)
+    return ns;
+  for (int i = 2; i < wl; i++) {
+    if (word[i] == word[i - 2]) {
+      state++;
+      if (state == 3) {
+        std::string candidate(word, word + i - 1);
+        candidate.insert(candidate.end(), word + i + 1, word + wl);
+        ns = testsug(wlst, candidate.c_str(), candidate.size(), ns, cpdsuggest, NULL, NULL);
+        if (ns == -1)
+          return -1;
+        state = 0;
+      }
+    } else {
+      state = 0;
+    }
+  }
+  return ns;
+}
+
+// perhaps we doubled two characters (pattern aba -> ababa, for example vacation
+// -> vacacation)
+int SuggestMgr::doubletwochars_utf(char** wlst,
+                                   const w_char* word,
+                                   int wl,
+                                   int ns,
+                                   int cpdsuggest) {
+  int state = 0;
+  if (wl < 5 || !pAMgr)
+    return ns;
+  for (int i = 2; i < wl; i++) {
+    if (word[i] == word[i - 2]) {
+      state++;
+      if (state == 3) {
+        std::vector<w_char> candidate_utf(word, word + i - 1);
+        candidate_utf.insert(candidate_utf.end(), word + i + 1, word + wl);
+        std::string candidate;
+        u16_u8(candidate, candidate_utf);
+        ns = testsug(wlst, candidate.c_str(), candidate.size(), ns, cpdsuggest, NULL,
+                     NULL);
+        if (ns == -1)
+          return -1;
+        state = 0;
+      }
+    } else {
+      state = 0;
+    }
+  }
+  return ns;
+}
+
+// error is wrong char in place of correct one (case and keyboard related
+// version)
+int SuggestMgr::badcharkey(char** wlst,
+                           const char* word,
+                           int ns,
+                           int cpdsuggest) {
+  std::string candidate(word);
+
+  // swap out each char one by one and try uppercase and neighbor
+  // keyboard chars in its place to see if that makes a good word
+  for (size_t i = 0; i < candidate.size(); ++i) {
+    char tmpc = candidate[i];
+    // check with uppercase letters
+    candidate[i] = csconv[((unsigned char)tmpc)].cupper;
+    if (tmpc != candidate[i]) {
+      ns = testsug(wlst, candidate.c_str(), candidate.size(), ns, cpdsuggest, NULL, NULL);
+      if (ns == -1)
+        return -1;
+      candidate[i] = tmpc;
+    }
+    // check neighbor characters in keyboard string
+    if (!ckey)
+      continue;
+    char* loc = strchr(ckey, tmpc);
+    while (loc) {
+      if ((loc > ckey) && (*(loc - 1) != '|')) {
+        candidate[i] = *(loc - 1);
+        ns = testsug(wlst, candidate.c_str(), candidate.size(), ns, cpdsuggest, NULL, NULL);
+        if (ns == -1)
+          return -1;
+      }
+      if ((*(loc + 1) != '|') && (*(loc + 1) != '\0')) {
+        candidate[i] = *(loc + 1);
+        ns = testsug(wlst, candidate.c_str(), candidate.size(), ns, cpdsuggest, NULL, NULL);
+        if (ns == -1)
+          return -1;
+      }
+      loc = strchr(loc + 1, tmpc);
+    }
+    candidate[i] = tmpc;
+  }
+  return ns;
+}
+
+// error is wrong char in place of correct one (case and keyboard related
+// version)
+int SuggestMgr::badcharkey_utf(char** wlst,
+                               const w_char* word,
+                               int wl,
+                               int ns,
+                               int cpdsuggest) {
+  std::string candidate;
+  std::vector<w_char> candidate_utf(word, word + wl);
+  // swap out each char one by one and try all the tryme
+  // chars in its place to see if that makes a good word
+  for (int i = 0; i < wl; i++) {
+    w_char tmpc = candidate_utf[i];
+    // check with uppercase letters
+    candidate_utf[i] = upper_utf(candidate_utf[i], 1);
+    if (tmpc != candidate_utf[i]) {
+      u16_u8(candidate, candidate_utf);
+      ns = testsug(wlst, candidate.c_str(), candidate.size(), ns, cpdsuggest, NULL,
+                   NULL);
+      if (ns == -1)
+        return -1;
+      candidate_utf[i] = tmpc;
+    }
+    // check neighbor characters in keyboard string
+    if (!ckey)
+      continue;
+    w_char* loc = ckey_utf;
+    while ((loc < (ckey_utf + ckeyl)) && *loc != tmpc)
+      loc++;
+    while (loc < (ckey_utf + ckeyl)) {
+      if ((loc > ckey_utf) && *(loc - 1) != W_VLINE) {
+        candidate_utf[i] = *(loc - 1);
+        u16_u8(candidate, candidate_utf);
+        ns = testsug(wlst, candidate.c_str(), candidate.size(), ns, cpdsuggest, NULL,
+                     NULL);
+        if (ns == -1)
+          return -1;
+      }
+      if (((loc + 1) < (ckey_utf + ckeyl)) && (*(loc + 1) != W_VLINE)) {
+        candidate_utf[i] = *(loc + 1);
+        u16_u8(candidate, candidate_utf);
+        ns = testsug(wlst, candidate.c_str(), candidate.size(), ns, cpdsuggest, NULL,
+                     NULL);
+        if (ns == -1)
+          return -1;
+      }
+      do {
+        loc++;
+      } while ((loc < (ckey_utf + ckeyl)) && *loc != tmpc);
+    }
+    candidate_utf[i] = tmpc;
+  }
+  return ns;
+}
+
+// error is wrong char in place of correct one
+int SuggestMgr::badchar(char** wlst, const char* word, int ns, int cpdsuggest) {
+  std::string candidate(word);
+  clock_t timelimit = clock();
+  int timer = MINTIMER;
+  // swap out each char one by one and try all the tryme
+  // chars in its place to see if that makes a good word
+  for (int j = 0; j < ctryl; j++) {
+    for (std::string::reverse_iterator aI = candidate.rbegin(), aEnd = candidate.rend(); aI != aEnd; ++aI) {
+      char tmpc = *aI;
+      if (ctry[j] == tmpc)
+        continue;
+      *aI = ctry[j];
+      ns = testsug(wlst, candidate.c_str(), candidate.size(), ns, cpdsuggest, &timer, &timelimit);
+      if (ns == -1)
+        return -1;
+      if (!timer)
+        return ns;
+      *aI = tmpc;
+    }
+  }
+  return ns;
+}
+
+// error is wrong char in place of correct one
+int SuggestMgr::badchar_utf(char** wlst,
+                            const w_char* word,
+                            int wl,
+                            int ns,
+                            int cpdsuggest) {
+  std::vector<w_char> candidate_utf(word, word + wl);
+  std::string candidate;
+  clock_t timelimit = clock();
+  int timer = MINTIMER;
+  // swap out each char one by one and try all the tryme
+  // chars in its place to see if that makes a good word
+  for (int j = 0; j < ctryl; j++) {
+    for (int i = wl - 1; i >= 0; i--) {
+      w_char tmpc = candidate_utf[i];
+      if (tmpc == ctry_utf[j])
+        continue;
+      candidate_utf[i] = ctry_utf[j];
+      u16_u8(candidate, candidate_utf);
+      ns = testsug(wlst, candidate.c_str(), candidate.size(), ns, cpdsuggest, &timer,
+                   &timelimit);
+      if (ns == -1)
+        return -1;
+      if (!timer)
+        return ns;
+      candidate_utf[i] = tmpc;
+    }
+  }
+  return ns;
+}
+
+// error is word has an extra letter it does not need
+int SuggestMgr::extrachar_utf(char** wlst,
+                              const w_char* word,
+                              int wl,
+                              int ns,
+                              int cpdsuggest) {
+  std::vector<w_char> candidate_utf(word, word + wl);
+  if (candidate_utf.size() < 2)
+    return ns;
+  // try omitting one char of word at a time
+  for (size_t i = 0; i < candidate_utf.size(); ++i) {
+    size_t index = candidate_utf.size() - 1 - i;
+    w_char tmpc = candidate_utf[index];
+    candidate_utf.erase(candidate_utf.begin() + index);
+    std::string candidate;
+    u16_u8(candidate, candidate_utf);
+    ns = testsug(wlst, candidate.c_str(), candidate.size(), ns, cpdsuggest, NULL, NULL);
+    if (ns == -1)
+      return -1;
+    candidate_utf.insert(candidate_utf.begin() + index, tmpc);
+  }
+  return ns;
+}
+
+// error is word has an extra letter it does not need
+int SuggestMgr::extrachar(char** wlst,
+                          const char* word,
+                          int ns,
+                          int cpdsuggest) {
+  std::string candidate(word);
+  if (candidate.size() < 2)
+    return ns;
+  // try omitting one char of word at a time
+  for (size_t i = 0; i < candidate.size(); ++i) {
+    size_t index = candidate.size() - 1 - i;
+    char tmpc = candidate[index];
+    candidate.erase(candidate.begin() + index);
+    ns = testsug(wlst, candidate.c_str(), candidate.size(), ns, cpdsuggest, NULL, NULL);
+    if (ns == -1)
+      return -1;
+    candidate.insert(candidate.begin() + index, tmpc);
+  }
+  return ns;
+}
+
+// error is missing a letter it needs
+int SuggestMgr::forgotchar(char** wlst,
+                           const char* word,
+                           int ns,
+                           int cpdsuggest) {
+  std::string candidate(word);
+  clock_t timelimit = clock();
+  int timer = MINTIMER;
+
+  // try inserting a tryme character before every letter (and the null
+  // terminator)
+  for (int k = 0; k < ctryl; ++k) {
+    for (size_t i = 0; i <= candidate.size(); ++i) {
+      size_t index = candidate.size() - i;
+      candidate.insert(candidate.begin() + index, ctry[k]);
+      ns = testsug(wlst, candidate.c_str(), candidate.size(), ns, cpdsuggest, &timer, &timelimit);
+      if (ns == -1)
+        return -1;
+      if (!timer)
+        return ns;
+      candidate.erase(candidate.begin() + index);
+    }
+  }
+  return ns;
+}
+
+// error is missing a letter it needs
+int SuggestMgr::forgotchar_utf(char** wlst,
+                               const w_char* word,
+                               int wl,
+                               int ns,
+                               int cpdsuggest) {
+  std::vector<w_char> candidate_utf(word, word + wl);
+  clock_t timelimit = clock();
+  int timer = MINTIMER;
+
+  // try inserting a tryme character at the end of the word and before every
+  // letter
+  for (int k = 0; k < ctryl; ++k) {
+    for (size_t i = 0; i <= candidate_utf.size(); ++i) {
+      size_t index = candidate_utf.size() - i;
+      candidate_utf.insert(candidate_utf.begin() + index, ctry_utf[k]);
+      std::string candidate;
+      u16_u8(candidate, candidate_utf);
+      ns = testsug(wlst, candidate.c_str(), candidate.size(), ns, cpdsuggest, &timer,
+                   &timelimit);
+      if (ns == -1)
+        return -1;
+      if (!timer)
+        return ns;
+      candidate_utf.erase(candidate_utf.begin() + index);
+    }
+  }
+  return ns;
+}
+
+/* error is should have been two words */
+int SuggestMgr::twowords(char** wlst,
+                         const char* word,
+                         int ns,
+                         int cpdsuggest) {
+  int c1, c2;
+  int forbidden = 0;
+  int cwrd;
+
+  int wl = strlen(word);
+  if (wl < 3)
+    return ns;
+
+  if (langnum == LANG_hu)
+    forbidden = check_forbidden(word, wl);
+
+  char* candidate = (char*)malloc(wl + 2);
+  strcpy(candidate + 1, word);
+
+  // split the string into two pieces after every char
+  // if both pieces are good words make them a suggestion
+  for (char* p = candidate + 1; p[1] != '\0'; p++) {
+    p[-1] = *p;
+    // go to end of the UTF-8 character
+    while (utf8 && ((p[1] & 0xc0) == 0x80)) {
+      *p = p[1];
+      p++;
+    }
+    if (utf8 && p[1] == '\0')
+      break;  // last UTF-8 character
+    *p = '\0';
+    c1 = checkword(candidate, strlen(candidate), cpdsuggest, NULL, NULL);
+    if (c1) {
+      c2 = checkword((p + 1), strlen(p + 1), cpdsuggest, NULL, NULL);
+      if (c2) {
+        *p = ' ';
+
+        // spec. Hungarian code (need a better compound word support)
+        if ((langnum == LANG_hu) && !forbidden &&
+            // if 3 repeating letter, use - instead of space
+            (((p[-1] == p[1]) &&
+              (((p > candidate + 1) && (p[-1] == p[-2])) || (p[-1] == p[2]))) ||
+             // or multiple compounding, with more, than 6 syllables
+             ((c1 == 3) && (c2 >= 2))))
+          *p = '-';
+
+        cwrd = 1;
+        for (int k = 0; k < ns; k++) {
+          if (strcmp(candidate, wlst[k]) == 0) {
+            cwrd = 0;
+            break;
+          }
+        }
+        if (ns < maxSug) {
+          if (cwrd) {
+            wlst[ns] = mystrdup(candidate);
+            if (wlst[ns] == NULL) {
+              free(candidate);
+              return -1;
+            }
+            ns++;
+          }
+        } else {
+          free(candidate);
+          return ns;
+        }
+        // add two word suggestion with dash, if TRY string contains
+        // "a" or "-"
+        // NOTE: cwrd doesn't modified for REP twoword sugg.
+        if (ctry && (strchr(ctry, 'a') || strchr(ctry, '-')) &&
+            mystrlen(p + 1) > 1 && mystrlen(candidate) - mystrlen(p) > 1) {
+          *p = '-';
+          for (int k = 0; k < ns; k++) {
+            if (strcmp(candidate, wlst[k]) == 0) {
+              cwrd = 0;
+              break;
+            }
+          }
+          if (ns < maxSug) {
+            if (cwrd) {
+              wlst[ns] = mystrdup(candidate);
+              if (wlst[ns] == NULL) {
+                free(candidate);
+                return -1;
+              }
+              ns++;
+            }
+          } else {
+            free(candidate);
+            return ns;
+          }
+        }
+      }
+    }
+  }
+  free(candidate);
+  return ns;
+}
+
+// error is adjacent letter were swapped
+int SuggestMgr::swapchar(char** wlst,
+                         const char* word,
+                         int ns,
+                         int cpdsuggest) {
+  std::string candidate(word);
+  if (candidate.size() < 2)
+    return ns;
+
+  // try swapping adjacent chars one by one
+  for (size_t i = 0; i < candidate.size() - 1; ++i) {
+    std::swap(candidate[i], candidate[i+1]);
+    ns = testsug(wlst, candidate.c_str(), candidate.size(), ns, cpdsuggest, NULL, NULL);
+    if (ns == -1)
+      return -1;
+    std::swap(candidate[i], candidate[i+1]);
+  }
+
+  // try double swaps for short words
+  // ahev -> have, owudl -> would
+  if (candidate.size() == 4 || candidate.size() == 5) {
+    candidate[0] = word[1];
+    candidate[1] = word[0];
+    candidate[2] = word[2];
+    candidate[candidate.size() - 2] = word[candidate.size() - 1];
+    candidate[candidate.size() - 1] = word[candidate.size() - 2];
+    ns = testsug(wlst, candidate.c_str(), candidate.size(), ns, cpdsuggest, NULL, NULL);
+    if (ns == -1)
+      return -1;
+    if (candidate.size() == 5) {
+      candidate[0] = word[0];
+      candidate[1] = word[2];
+      candidate[2] = word[1];
+      ns = testsug(wlst, candidate.c_str(), candidate.size(), ns, cpdsuggest, NULL, NULL);
+      if (ns == -1)
+        return -1;
+    }
+  }
+
+  return ns;
+}
+
+// error is adjacent letter were swapped
+int SuggestMgr::swapchar_utf(char** wlst,
+                             const w_char* word,
+                             int wl,
+                             int ns,
+                             int cpdsuggest) {
+  std::vector<w_char> candidate_utf(word, word + wl);
+  if (candidate_utf.size() < 2)
+    return ns;
+
+  std::string candidate;
+  // try swapping adjacent chars one by one
+  for (size_t i = 0; i < candidate_utf.size() - 1; ++i) {
+    std::swap(candidate_utf[i], candidate_utf[i+1]);
+    u16_u8(candidate, candidate_utf);
+    ns = testsug(wlst, candidate.c_str(), candidate.size(), ns, cpdsuggest, NULL, NULL);
+    if (ns == -1)
+      return -1;
+    std::swap(candidate_utf[i], candidate_utf[i+1]);
+  }
+
+  // try double swaps for short words
+  // ahev -> have, owudl -> would, suodn -> sound
+  if (candidate_utf.size() == 4 || candidate_utf.size() == 5) {
+    candidate_utf[0] = word[1];
+    candidate_utf[1] = word[0];
+    candidate_utf[2] = word[2];
+    candidate_utf[candidate_utf.size() - 2] = word[candidate_utf.size() - 1];
+    candidate_utf[candidate_utf.size() - 1] = word[candidate_utf.size() - 2];
+    u16_u8(candidate, candidate_utf);
+    ns = testsug(wlst, candidate.c_str(), candidate.size(), ns, cpdsuggest, NULL, NULL);
+    if (ns == -1)
+      return -1;
+    if (candidate_utf.size() == 5) {
+      candidate_utf[0] = word[0];
+      candidate_utf[1] = word[2];
+      candidate_utf[2] = word[1];
+      u16_u8(candidate, candidate_utf);
+      ns = testsug(wlst, candidate.c_str(), candidate.size(), ns, cpdsuggest, NULL, NULL);
+      if (ns == -1)
+        return -1;
+    }
+  }
+  return ns;
+}
+
+// error is not adjacent letter were swapped
+int SuggestMgr::longswapchar(char** wlst,
+                             const char* word,
+                             int ns,
+                             int cpdsuggest) {
+  std::string candidate(word);
+  // try swapping not adjacent chars one by one
+  for (std::string::iterator p = candidate.begin(); p < candidate.end(); ++p) {
+    for (std::string::iterator q = candidate.begin(); q < candidate.end(); ++q) {
+      if (abs(std::distance(q, p)) > 1) {
+        std::swap(*p, *q);
+        ns = testsug(wlst, candidate.c_str(), candidate.size(), ns, cpdsuggest, NULL, NULL);
+        if (ns == -1)
+          return -1;
+        std::swap(*p, *q);
+      }
+    }
+  }
+  return ns;
+}
+
+// error is adjacent letter were swapped
+int SuggestMgr::longswapchar_utf(char** wlst,
+                                 const w_char* word,
+                                 int wl,
+                                 int ns,
+                                 int cpdsuggest) {
+  std::vector<w_char> candidate_utf(word, word + wl);
+  // try swapping not adjacent chars
+  for (std::vector<w_char>::iterator p = candidate_utf.begin(); p < candidate_utf.end(); ++p) {
+    for (std::vector<w_char>::iterator q = candidate_utf.begin(); q < candidate_utf.end(); ++q) {
+      if (abs(std::distance(q, p)) > 1) {
+        std::swap(*p, *q);
+        std::string candidate;
+        u16_u8(candidate, candidate_utf);
+        ns = testsug(wlst, candidate.c_str(), candidate.size(), ns, cpdsuggest, NULL,
+                     NULL);
+        if (ns == -1)
+          return -1;
+        std::swap(*p, *q);
+      }
+    }
+  }
+  return ns;
+}
+
+// error is a letter was moved
+int SuggestMgr::movechar(char** wlst,
+                         const char* word,
+                         int ns,
+                         int cpdsuggest) {
+  std::string candidate(word);
+  if (candidate.size() < 2)
+    return ns;
+
+  // try moving a char
+  for (std::string::iterator p = candidate.begin(); p < candidate.end(); ++p) {
+    for (std::string::iterator q = p + 1; q < candidate.end() && std::distance(p, q) < 10; ++q) {
+      std::swap(*q, *(q - 1));
+      if (std::distance(p, q) < 2)
+        continue;  // omit swap char
+      ns = testsug(wlst, candidate.c_str(), candidate.size(), ns, cpdsuggest, NULL, NULL);
+      if (ns == -1)
+        return -1;
+    }
+    std::copy(word, word + candidate.size(), candidate.begin());
+  }
+
+  for (std::string::reverse_iterator p = candidate.rbegin(), pEnd = candidate.rend() - 1; p != pEnd; ++p) {
+    for (std::string::reverse_iterator q = p + 1, qEnd = candidate.rend(); q != qEnd && std::distance(p, q) < 10; ++q) {
+      std::swap(*q, *(q - 1));
+      if (std::distance(p, q) < 2)
+        continue;  // omit swap char
+      ns = testsug(wlst, candidate.c_str(), candidate.size(), ns, cpdsuggest, NULL, NULL);
+      if (ns == -1)
+        return -1;
+    }
+    std::copy(word, word + candidate.size(), candidate.begin());
+  }
+
+  return ns;
+}
+
+// error is a letter was moved
+int SuggestMgr::movechar_utf(char** wlst,
+                             const w_char* word,
+                             int wl,
+                             int ns,
+                             int cpdsuggest) {
+  std::vector<w_char> candidate_utf(word, word + wl);
+  if (candidate_utf.size() < 2)
+    return ns;
+
+  // try moving a char
+  for (std::vector<w_char>::iterator p = candidate_utf.begin(); p < candidate_utf.end(); ++p) {
+    for (std::vector<w_char>::iterator q = p + 1; q < candidate_utf.end() && std::distance(p, q) < 10; ++q) {
+      std::swap(*q, *(q - 1));
+      if (std::distance(p, q) < 2)
+        continue;  // omit swap char
+      std::string candidate;
+      u16_u8(candidate, candidate_utf);
+      ns = testsug(wlst, candidate.c_str(), candidate.size(), ns, cpdsuggest, NULL,
+                   NULL);
+      if (ns == -1)
+        return -1;
+    }
+    std::copy(word, word + candidate_utf.size(), candidate_utf.begin());
+  }
+
+  for (std::vector<w_char>::iterator p = candidate_utf.begin() + candidate_utf.size() - 1; p > candidate_utf.begin(); --p) {
+    for (std::vector<w_char>::iterator q = p - 1; q >= candidate_utf.begin() && std::distance(q, p) < 10; --q) {
+      std::swap(*q, *(q + 1));
+      if (std::distance(q, p) < 2)
+        continue;  // omit swap char
+      std::string candidate;
+      u16_u8(candidate, candidate_utf);
+      ns = testsug(wlst, candidate.c_str(), candidate.size(), ns, cpdsuggest, NULL,
+                   NULL);
+      if (ns == -1)
+        return -1;
+    }
+    std::copy(word, word + candidate_utf.size(), candidate_utf.begin());
+  }
+
+  return ns;
+}
+
+// generate a set of suggestions for very poorly spelled words
+int SuggestMgr::ngsuggest(char** wlst,
+                          const char* w,
+                          int ns,
+                          HashMgr** pHMgr,
+                          int md) {
+  int i, j;
+  int lval;
+  int sc;
+  int lp, lpphon;
+  int nonbmp = 0;
+
+  // exhaustively search through all root words
+  // keeping track of the MAX_ROOTS most similar root words
+  struct hentry* roots[MAX_ROOTS];
+  char* rootsphon[MAX_ROOTS];
+  int scores[MAX_ROOTS];
+  int scoresphon[MAX_ROOTS];
+  for (i = 0; i < MAX_ROOTS; i++) {
+    roots[i] = NULL;
+    scores[i] = -100 * i;
+    rootsphon[i] = NULL;
+    scoresphon[i] = -100 * i;
+  }
+  lp = MAX_ROOTS - 1;
+  lpphon = MAX_ROOTS - 1;
+  int low = NGRAM_LOWERING;
+
+  std::string w2;
+  const char* word = w;
+
+  // word reversing wrapper for complex prefixes
+  if (complexprefixes) {
+    w2.assign(w);
+    if (utf8)
+      reverseword_utf(w2);
+    else
+      reverseword(w2);
+    word = w2.c_str();
+  }
+
+  std::vector<w_char> u8;
+  int nc = strlen(word);
+  int n = (utf8) ? u8_u16(u8, word) : nc;
+
+  // set character based ngram suggestion for words with non-BMP Unicode
+  // characters
+  if (n == -1) {
+    utf8 = 0;  // XXX not state-free
+    n = nc;
+    nonbmp = 1;
+    low = 0;
+  }
+
+  struct hentry* hp = NULL;
+  int col = -1;
+  phonetable* ph = (pAMgr) ? pAMgr->get_phonetable() : NULL;
+  std::string target;
+  std::string candidate;
+  if (ph) {
+    if (utf8) {
+      std::vector<w_char> _w;
+      u8_u16(_w, word);
+      mkallcap_utf(_w, langnum);
+      u16_u8(candidate, _w);
+    } else {
+      candidate.assign(word);
+      if (!nonbmp)
+        mkallcap(candidate, csconv);
+    }
+    target = phonet(candidate, *ph);  // XXX phonet() is 8-bit (nc, not n)
+  }
+
+  FLAG forbiddenword = pAMgr ? pAMgr->get_forbiddenword() : FLAG_NULL;
+  FLAG nosuggest = pAMgr ? pAMgr->get_nosuggest() : FLAG_NULL;
+  FLAG nongramsuggest = pAMgr ? pAMgr->get_nongramsuggest() : FLAG_NULL;
+  FLAG onlyincompound = pAMgr ? pAMgr->get_onlyincompound() : FLAG_NULL;
+
+  for (i = 0; i < md; i++) {
+    while (0 != (hp = (pHMgr[i])->walk_hashtable(col, hp))) {
+      if ((hp->astr) && (pAMgr) &&
+          (TESTAFF(hp->astr, forbiddenword, hp->alen) ||
+           TESTAFF(hp->astr, ONLYUPCASEFLAG, hp->alen) ||
+           TESTAFF(hp->astr, nosuggest, hp->alen) ||
+           TESTAFF(hp->astr, nongramsuggest, hp->alen) ||
+           TESTAFF(hp->astr, onlyincompound, hp->alen)))
+        continue;
+
+      sc = ngram(3, word, HENTRY_WORD(hp), NGRAM_LONGER_WORSE + low) +
+           leftcommonsubstring(word, HENTRY_WORD(hp));
+
+      // check special pronounciation
+      std::string f;
+      if ((hp->var & H_OPT_PHON) &&
+          copy_field(f, HENTRY_DATA(hp), MORPH_PHON)) {
+        int sc2 = ngram(3, word, f, NGRAM_LONGER_WORSE + low) +
+                  +leftcommonsubstring(word, f.c_str());
+        if (sc2 > sc)
+          sc = sc2;
+      }
+
+      int scphon = -20000;
+      if (ph && (sc > 2) && (abs(n - (int)hp->clen) <= 3)) {
+        if (utf8) {
+          std::vector<w_char> _w;
+          u8_u16(_w, HENTRY_WORD(hp));
+          mkallcap_utf(_w, langnum);
+          u16_u8(candidate, _w);
+        } else {
+          candidate.assign(HENTRY_WORD(hp));
+          mkallcap(candidate, csconv);
+        }
+        std::string target2 = phonet(candidate, *ph);
+        scphon = 2 * ngram(3, target, target2, NGRAM_LONGER_WORSE);
+      }
+
+      if (sc > scores[lp]) {
+        scores[lp] = sc;
+        roots[lp] = hp;
+        lval = sc;
+        for (j = 0; j < MAX_ROOTS; j++)
+          if (scores[j] < lval) {
+            lp = j;
+            lval = scores[j];
+          }
+      }
+
+      if (scphon > scoresphon[lpphon]) {
+        scoresphon[lpphon] = scphon;
+        rootsphon[lpphon] = HENTRY_WORD(hp);
+        lval = scphon;
+        for (j = 0; j < MAX_ROOTS; j++)
+          if (scoresphon[j] < lval) {
+            lpphon = j;
+            lval = scoresphon[j];
+          }
+      }
+    }
+  }
+
+  // find minimum threshold for a passable suggestion
+  // mangle original word three differnt ways
+  // and score them to generate a minimum acceptable score
+  int thresh = 0;
+  for (int sp = 1; sp < 4; sp++) {
+    if (utf8) {
+      for (int k = sp; k < n; k += 4) {
+        u8[k].l = '*';
+        u8[k].h = 0;
+      }
+      std::string mw;
+      u16_u8(mw, u8);
+      thresh = thresh + ngram(n, word, mw, NGRAM_ANY_MISMATCH + low);
+    } else {
+      std::string mw(word);
+      for (int k = sp; k < n; k += 4)
+        mw[k] = '*';
+      thresh = thresh + ngram(n, word, mw, NGRAM_ANY_MISMATCH + low);
+    }
+  }
+  thresh = thresh / 3;
+  thresh--;
+
+  // now expand affixes on each of these root words and
+  // and use length adjusted ngram scores to select
+  // possible suggestions
+  char* guess[MAX_GUESS];
+  char* guessorig[MAX_GUESS];
+  int gscore[MAX_GUESS];
+  for (i = 0; i < MAX_GUESS; i++) {
+    guess[i] = NULL;
+    guessorig[i] = NULL;
+    gscore[i] = -100 * i;
+  }
+
+  lp = MAX_GUESS - 1;
+
+  struct guessword* glst;
+  glst = (struct guessword*)calloc(MAX_WORDS, sizeof(struct guessword));
+  if (!glst) {
+    if (nonbmp)
+      utf8 = 1;
+    return ns;
+  }
+
+  for (i = 0; i < MAX_ROOTS; i++) {
+    if (roots[i]) {
+      struct hentry* rp = roots[i];
+
+      std::string f;
+      const char *field = NULL;
+      if ((rp->var & H_OPT_PHON) && copy_field(f, HENTRY_DATA(rp), MORPH_PHON))
+          field = f.c_str();
+      int nw = pAMgr->expand_rootword(
+          glst, MAX_WORDS, HENTRY_WORD(rp), rp->blen, rp->astr, rp->alen, word,
+          nc, field);
+
+      for (int k = 0; k < nw; k++) {
+        sc = ngram(n, word, glst[k].word, NGRAM_ANY_MISMATCH + low) +
+             leftcommonsubstring(word, glst[k].word);
+
+        if (sc > thresh) {
+          if (sc > gscore[lp]) {
+            if (guess[lp]) {
+              free(guess[lp]);
+              if (guessorig[lp]) {
+                free(guessorig[lp]);
+                guessorig[lp] = NULL;
+              }
+            }
+            gscore[lp] = sc;
+            guess[lp] = glst[k].word;
+            guessorig[lp] = glst[k].orig;
+            lval = sc;
+            for (j = 0; j < MAX_GUESS; j++)
+              if (gscore[j] < lval) {
+                lp = j;
+                lval = gscore[j];
+              }
+          } else {
+            free(glst[k].word);
+            if (glst[k].orig)
+              free(glst[k].orig);
+          }
+        } else {
+          free(glst[k].word);
+          if (glst[k].orig)
+            free(glst[k].orig);
+        }
+      }
+    }
+  }
+  free(glst);
+
+  // now we are done generating guesses
+  // sort in order of decreasing score
+
+  bubblesort(&guess[0], &guessorig[0], &gscore[0], MAX_GUESS);
+  if (ph)
+    bubblesort(&rootsphon[0], NULL, &scoresphon[0], MAX_ROOTS);
+
+  // weight suggestions with a similarity index, based on
+  // the longest common subsequent algorithm and resort
+
+  int is_swap = 0;
+  int re = 0;
+  double fact = 1.0;
+  if (pAMgr) {
+    int maxd = pAMgr->get_maxdiff();
+    if (maxd >= 0)
+      fact = (10.0 - maxd) / 5.0;
+  }
+
+  for (i = 0; i < MAX_GUESS; i++) {
+    if (guess[i]) {
+      // lowering guess[i]
+      std::string gl;
+      int len;
+      if (utf8) {
+        std::vector<w_char> _w;
+        len = u8_u16(_w, guess[i]);
+        mkallsmall_utf(_w, langnum);
+        u16_u8(gl, _w);
+      } else {
+        gl.assign(guess[i]);
+        if (!nonbmp)
+          mkallsmall(gl, csconv);
+        len = strlen(guess[i]);
+      }
+
+      int _lcs = lcslen(word, gl.c_str());
+
+      // same characters with different casing
+      if ((n == len) && (n == _lcs)) {
+        gscore[i] += 2000;
+        break;
+      }
+      // using 2-gram instead of 3, and other weightening
+
+      re = ngram(2, word, gl, NGRAM_ANY_MISMATCH + low + NGRAM_WEIGHTED) +
+           ngram(2, gl, word, NGRAM_ANY_MISMATCH + low + NGRAM_WEIGHTED);
+
+      gscore[i] =
+          // length of longest common subsequent minus length difference
+          2 * _lcs - abs((int)(n - len)) +
+          // weight length of the left common substring
+          leftcommonsubstring(word, gl.c_str()) +
+          // weight equal character positions
+          (!nonbmp && commoncharacterpositions(word, gl.c_str(), &is_swap)
+               ? 1
+               : 0) +
+          // swap character (not neighboring)
+          ((is_swap) ? 10 : 0) +
+          // ngram
+          ngram(4, word, gl, NGRAM_ANY_MISMATCH + low) +
+          // weighted ngrams
+          re +
+          // different limit for dictionaries with PHONE rules
+          (ph ? (re < len * fact ? -1000 : 0)
+              : (re < (n + len) * fact ? -1000 : 0));
+    }
+  }
+
+  bubblesort(&guess[0], &guessorig[0], &gscore[0], MAX_GUESS);
+
+  // phonetic version
+  if (ph)
+    for (i = 0; i < MAX_ROOTS; i++) {
+      if (rootsphon[i]) {
+        // lowering rootphon[i]
+        std::string gl;
+        int len;
+        if (utf8) {
+          std::vector<w_char> _w;
+          len = u8_u16(_w, rootsphon[i]);
+          mkallsmall_utf(_w, langnum);
+          u16_u8(gl, _w);
+        } else {
+          gl.assign(rootsphon[i]);
+          if (!nonbmp)
+            mkallsmall(gl, csconv);
+          len = strlen(rootsphon[i]);
+        }
+
+        // heuristic weigthing of ngram scores
+        scoresphon[i] += 2 * lcslen(word, gl) - abs((int)(n - len)) +
+                         // weight length of the left common substring
+                         leftcommonsubstring(word, gl.c_str());
+      }
+    }
+
+  if (ph)
+    bubblesort(&rootsphon[0], NULL, &scoresphon[0], MAX_ROOTS);
+
+  // copy over
+  int oldns = ns;
+
+  int same = 0;
+  for (i = 0; i < MAX_GUESS; i++) {
+    if (guess[i]) {
+      if ((ns < oldns + maxngramsugs) && (ns < maxSug) &&
+          (!same || (gscore[i] > 1000))) {
+        int unique = 1;
+        // leave only excellent suggestions, if exists
+        if (gscore[i] > 1000)
+          same = 1;
+        else if (gscore[i] < -100) {
+          same = 1;
+          // keep the best ngram suggestions, unless in ONLYMAXDIFF mode
+          if (ns > oldns || (pAMgr && pAMgr->get_onlymaxdiff())) {
+            free(guess[i]);
+            if (guessorig[i])
+              free(guessorig[i]);
+            continue;
+          }
+        }
+        for (j = 0; j < ns; j++) {
+          // don't suggest previous suggestions or a previous suggestion with
+          // prefixes or affixes
+          if ((!guessorig[i] && strstr(guess[i], wlst[j])) ||
+              (guessorig[i] && strstr(guessorig[i], wlst[j])) ||
+              // check forbidden words
+              !checkword(guess[i], strlen(guess[i]), 0, NULL, NULL)) {
+            unique = 0;
+            break;
+          }
+        }
+        if (unique) {
+          wlst[ns++] = guess[i];
+          if (guessorig[i]) {
+            free(guess[i]);
+            wlst[ns - 1] = guessorig[i];
+          }
+        } else {
+          free(guess[i]);
+          if (guessorig[i])
+            free(guessorig[i]);
+        }
+      } else {
+        free(guess[i]);
+        if (guessorig[i])
+          free(guessorig[i]);
+      }
+    }
+  }
+
+  oldns = ns;
+  if (ph)
+    for (i = 0; i < MAX_ROOTS; i++) {
+      if (rootsphon[i]) {
+        if ((ns < oldns + MAXPHONSUGS) && (ns < maxSug)) {
+          int unique = 1;
+          for (j = 0; j < ns; j++) {
+            // don't suggest previous suggestions or a previous suggestion with
+            // prefixes or affixes
+            if (strstr(rootsphon[i], wlst[j]) ||
+                // check forbidden words
+                !checkword(rootsphon[i], strlen(rootsphon[i]), 0, NULL, NULL)) {
+              unique = 0;
+              break;
+            }
+          }
+          if (unique) {
+            wlst[ns++] = mystrdup(rootsphon[i]);
+            if (!wlst[ns - 1])
+              return ns - 1;
+          }
+        }
+      }
+    }
+
+  if (nonbmp)
+    utf8 = 1;
+  return ns;
+}
+
+// see if a candidate suggestion is spelled correctly
+// needs to check both root words and words with affixes
+
+// obsolote MySpell-HU modifications:
+// return value 2 and 3 marks compounding with hyphen (-)
+// `3' marks roots without suffix
+int SuggestMgr::checkword(const char* word,
+                          int len,
+                          int cpdsuggest,
+                          int* timer,
+                          clock_t* timelimit) {
+  struct hentry* rv = NULL;
+  struct hentry* rv2 = NULL;
+  int nosuffix = 0;
+
+  // check time limit
+  if (timer) {
+    (*timer)--;
+    if (!(*timer) && timelimit) {
+      if ((clock() - *timelimit) > TIMELIMIT)
+        return 0;
+      *timer = MAXPLUSTIMER;
+    }
+  }
+
+  if (pAMgr) {
+    if (cpdsuggest == 1) {
+      if (pAMgr->get_compound()) {
+        struct hentry* rwords[100];  // buffer for COMPOUND pattern checking
+        rv = pAMgr->compound_check(word, len, 0, 0, 100, 0, NULL, (hentry**)&rwords, 0, 1,
+                                   0);  // EXT
+        if (rv &&
+            (!(rv2 = pAMgr->lookup(word)) || !rv2->astr ||
+             !(TESTAFF(rv2->astr, pAMgr->get_forbiddenword(), rv2->alen) ||
+               TESTAFF(rv2->astr, pAMgr->get_nosuggest(), rv2->alen))))
+          return 3;  // XXX obsolote categorisation + only ICONV needs affix
+                     // flag check?
+      }
+      return 0;
+    }
+
+    rv = pAMgr->lookup(word);
+
+    if (rv) {
+      if ((rv->astr) &&
+          (TESTAFF(rv->astr, pAMgr->get_forbiddenword(), rv->alen) ||
+           TESTAFF(rv->astr, pAMgr->get_nosuggest(), rv->alen)))
+        return 0;
+      while (rv) {
+        if (rv->astr &&
+            (TESTAFF(rv->astr, pAMgr->get_needaffix(), rv->alen) ||
+             TESTAFF(rv->astr, ONLYUPCASEFLAG, rv->alen) ||
+             TESTAFF(rv->astr, pAMgr->get_onlyincompound(), rv->alen))) {
+          rv = rv->next_homonym;
+        } else
+          break;
+      }
+    } else
+      rv = pAMgr->prefix_check(word, len,
+                               0);  // only prefix, and prefix + suffix XXX
+
+    if (rv) {
+      nosuffix = 1;
+    } else {
+      rv = pAMgr->suffix_check(word, len, 0, NULL, NULL, 0,
+                               NULL);  // only suffix
+    }
+
+    if (!rv && pAMgr->have_contclass()) {
+      rv = pAMgr->suffix_check_twosfx(word, len, 0, NULL, FLAG_NULL);
+      if (!rv)
+        rv = pAMgr->prefix_check_twosfx(word, len, 1, FLAG_NULL);
+    }
+
+    // check forbidden words
+    if ((rv) && (rv->astr) &&
+        (TESTAFF(rv->astr, pAMgr->get_forbiddenword(), rv->alen) ||
+         TESTAFF(rv->astr, ONLYUPCASEFLAG, rv->alen) ||
+         TESTAFF(rv->astr, pAMgr->get_nosuggest(), rv->alen) ||
+         TESTAFF(rv->astr, pAMgr->get_onlyincompound(), rv->alen)))
+      return 0;
+
+    if (rv) {  // XXX obsolote
+      if ((pAMgr->get_compoundflag()) &&
+          TESTAFF(rv->astr, pAMgr->get_compoundflag(), rv->alen))
+        return 2 + nosuffix;
+      return 1;
+    }
+  }
+  return 0;
+}
+
+int SuggestMgr::check_forbidden(const char* word, int len) {
+  struct hentry* rv = NULL;
+
+  if (pAMgr) {
+    rv = pAMgr->lookup(word);
+    if (rv && rv->astr &&
+        (TESTAFF(rv->astr, pAMgr->get_needaffix(), rv->alen) ||
+         TESTAFF(rv->astr, pAMgr->get_onlyincompound(), rv->alen)))
+      rv = NULL;
+    if (!(pAMgr->prefix_check(word, len, 1)))
+      rv = pAMgr->suffix_check(word, len, 0, NULL, NULL, 0,
+                               NULL);  // prefix+suffix, suffix
+    // check forbidden words
+    if ((rv) && (rv->astr) &&
+        TESTAFF(rv->astr, pAMgr->get_forbiddenword(), rv->alen))
+      return 1;
+  }
+  return 0;
+}
+
+char* SuggestMgr::suggest_morph(const char* w) {
+  char result[MAXLNLEN];
+  char* r = (char*)result;
+  char* st;
+
+  struct hentry* rv = NULL;
+
+  *result = '\0';
+
+  if (!pAMgr)
+    return NULL;
+
+  std::string w2;
+  const char* word = w;
+
+  // word reversing wrapper for complex prefixes
+  if (complexprefixes) {
+    w2.assign(w);
+    if (utf8)
+      reverseword_utf(w2);
+    else
+      reverseword(w2);
+    word = w2.c_str();
+  }
+
+  rv = pAMgr->lookup(word);
+
+  while (rv) {
+    if ((!rv->astr) ||
+        !(TESTAFF(rv->astr, pAMgr->get_forbiddenword(), rv->alen) ||
+          TESTAFF(rv->astr, pAMgr->get_needaffix(), rv->alen) ||
+          TESTAFF(rv->astr, pAMgr->get_onlyincompound(), rv->alen))) {
+      if (!HENTRY_FIND(rv, MORPH_STEM)) {
+        mystrcat(result, " ", MAXLNLEN);
+        mystrcat(result, MORPH_STEM, MAXLNLEN);
+        mystrcat(result, word, MAXLNLEN);
+      }
+      if (HENTRY_DATA(rv)) {
+        mystrcat(result, " ", MAXLNLEN);
+        mystrcat(result, HENTRY_DATA2(rv), MAXLNLEN);
+      }
+      mystrcat(result, "\n", MAXLNLEN);
+    }
+    rv = rv->next_homonym;
+  }
+
+  st = pAMgr->affix_check_morph(word, strlen(word));
+  if (st) {
+    mystrcat(result, st, MAXLNLEN);
+    free(st);
+  }
+
+  if (pAMgr->get_compound() && (*result == '\0')) {
+    struct hentry* rwords[100];  // buffer for COMPOUND pattern checking
+    pAMgr->compound_check_morph(word, strlen(word), 0, 0, 100, 0, NULL, (hentry**)&rwords, 0, &r,
+                                NULL);
+  }
+
+  return (*result) ? mystrdup(line_uniq(result, MSEP_REC)) : NULL;
+}
+
+/* affixation */
+char* SuggestMgr::suggest_hentry_gen(hentry* rv, const char* pattern) {
+  char result[MAXLNLEN];
+  *result = '\0';
+  int sfxcount = get_sfxcount(pattern);
+
+  if (get_sfxcount(HENTRY_DATA(rv)) > sfxcount)
+    return NULL;
+
+  if (HENTRY_DATA(rv)) {
+    char* aff = pAMgr->morphgen(HENTRY_WORD(rv), rv->blen, rv->astr, rv->alen,
+                                HENTRY_DATA(rv), pattern, 0);
+    if (aff) {
+      mystrcat(result, aff, MAXLNLEN);
+      mystrcat(result, "\n", MAXLNLEN);
+      free(aff);
+    }
+  }
+
+  // check all allomorphs
+  char allomorph[MAXLNLEN];
+  char* p = NULL;
+  if (HENTRY_DATA(rv))
+    p = (char*)strstr(HENTRY_DATA2(rv), MORPH_ALLOMORPH);
+  while (p) {
+    struct hentry* rv2 = NULL;
+    p += MORPH_TAG_LEN;
+    int plen = fieldlen(p);
+    strncpy(allomorph, p, plen);
+    allomorph[plen] = '\0';
+    rv2 = pAMgr->lookup(allomorph);
+    while (rv2) {
+      //            if (HENTRY_DATA(rv2) && get_sfxcount(HENTRY_DATA(rv2)) <=
+      //            sfxcount) {
+      if (HENTRY_DATA(rv2)) {
+        char* st = (char*)strstr(HENTRY_DATA2(rv2), MORPH_STEM);
+        if (st && (strncmp(st + MORPH_TAG_LEN, HENTRY_WORD(rv),
+                           fieldlen(st + MORPH_TAG_LEN)) == 0)) {
+          char* aff = pAMgr->morphgen(HENTRY_WORD(rv2), rv2->blen, rv2->astr,
+                                      rv2->alen, HENTRY_DATA(rv2), pattern, 0);
+          if (aff) {
+            mystrcat(result, aff, MAXLNLEN);
+            mystrcat(result, "\n", MAXLNLEN);
+            free(aff);
+          }
+        }
+      }
+      rv2 = rv2->next_homonym;
+    }
+    p = strstr(p + plen, MORPH_ALLOMORPH);
+  }
+
+  return (*result) ? mystrdup(result) : NULL;
+}
+
+char* SuggestMgr::suggest_gen(char** desc, int n, const char* pattern) {
+  if (n == 0 || !pAMgr)
+    return NULL;
+
+  std::string result2;
+  std::string newpattern;
+  struct hentry* rv = NULL;
+
+  // search affixed forms with and without derivational suffixes
+  while (1) {
+    for (int k = 0; k < n; k++) {
+      std::string result;
+
+      // add compound word parts (except the last one)
+      char* s = (char*)desc[k];
+      char* part = strstr(s, MORPH_PART);
+      if (part) {
+        char* nextpart = strstr(part + 1, MORPH_PART);
+        while (nextpart) {
+          std::string field;
+          copy_field(field, part, MORPH_PART);
+          result.append(field);
+          part = nextpart;
+          nextpart = strstr(part + 1, MORPH_PART);
+        }
+        s = part;
+      }
+
+      char** pl;
+      std::string tok(s);
+      size_t pos = tok.find(" | ");
+      while (pos != std::string::npos) {
+        tok[pos + 1] = MSEP_ALT;
+        pos = tok.find(" | ", pos);
+      }
+      int pln = line_tok(tok.c_str(), &pl, MSEP_ALT);
+      for (int i = 0; i < pln; i++) {
+        // remove inflectional and terminal suffixes
+        char* is = strstr(pl[i], MORPH_INFL_SFX);
+        if (is)
+          *is = '\0';
+        char* ts = strstr(pl[i], MORPH_TERM_SFX);
+        while (ts) {
+          *ts = '_';
+          ts = strstr(pl[i], MORPH_TERM_SFX);
+        }
+        char* st = strstr(s, MORPH_STEM);
+        if (st) {
+          copy_field(tok, st, MORPH_STEM);
+          rv = pAMgr->lookup(tok.c_str());
+          while (rv) {
+            std::string newpat(pl[i]);
+            newpat.append(pattern);
+            char* sg = suggest_hentry_gen(rv, newpat.c_str());
+            if (!sg)
+              sg = suggest_hentry_gen(rv, pattern);
+            if (sg) {
+              char** gen;
+              int genl = line_tok(sg, &gen, MSEP_REC);
+              free(sg);
+              sg = NULL;
+              for (int j = 0; j < genl; j++) {
+                result2.push_back(MSEP_REC);
+                result2.append(result);
+                if (strstr(pl[i], MORPH_SURF_PFX)) {
+                  std::string field;
+                  copy_field(field, pl[i], MORPH_SURF_PFX);
+                  result2.append(field);
+                }
+                result2.append(gen[j]);
+              }
+              freelist(&gen, genl);
+            }
+            rv = rv->next_homonym;
+          }
+        }
+      }
+      freelist(&pl, pln);
+    }
+
+    if (!result2.empty() || !strstr(pattern, MORPH_DERI_SFX))
+      break;
+
+    newpattern.assign(pattern);
+    mystrrep(newpattern, MORPH_DERI_SFX, MORPH_TERM_SFX);
+    pattern = newpattern.c_str();
+  }
+  return (!result2.empty() ? mystrdup(result2.c_str()) : NULL);
+}
+
+// generate an n-gram score comparing s1 and s2
+int SuggestMgr::ngram(int n,
+                      const std::string& s1,
+                      const std::string& s2,
+                      int opt) {
+  int nscore = 0;
+  int ns;
+  int l1;
+  int l2;
+  int test = 0;
+
+  if (utf8) {
+    std::vector<w_char> su1;
+    std::vector<w_char> su2;
+    l1 = u8_u16(su1, s1);
+    l2 = u8_u16(su2, s2);
+    if ((l2 <= 0) || (l1 == -1))
+      return 0;
+    // lowering dictionary word
+    if (opt & NGRAM_LOWERING)
+      mkallsmall_utf(su2, langnum);
+    for (int j = 1; j <= n; j++) {
+      ns = 0;
+      for (int i = 0; i <= (l1 - j); i++) {
+        int k = 0;
+        for (int l = 0; l <= (l2 - j); l++) {
+          for (k = 0; k < j; k++) {
+            w_char& c1 = su1[i + k];
+            w_char& c2 = su2[l + k];
+            if ((c1.l != c2.l) || (c1.h != c2.h))
+              break;
+          }
+          if (k == j) {
+            ns++;
+            break;
+          }
+        }
+        if (k != j && opt & NGRAM_WEIGHTED) {
+          ns--;
+          test++;
+          if (i == 0 || i == l1 - j)
+            ns--;  // side weight
+        }
+      }
+      nscore = nscore + ns;
+      if (ns < 2 && !(opt & NGRAM_WEIGHTED))
+        break;
+    }
+  } else {
+    l2 = s2.size();
+    if (l2 == 0)
+      return 0;
+    l1 = s1.size();
+    std::string t(s2);
+    if (opt & NGRAM_LOWERING)
+      mkallsmall(t, csconv);
+    for (int j = 1; j <= n; j++) {
+      ns = 0;
+      for (int i = 0; i <= (l1 - j); i++) {
+        std::string temp(s1.substr(i, j));
+        if (t.find(temp) != std::string::npos) {
+          ns++;
+        } else if (opt & NGRAM_WEIGHTED) {
+          ns--;
+          test++;
+          if (i == 0 || i == l1 - j)
+            ns--;  // side weight
+        }
+      }
+      nscore = nscore + ns;
+      if (ns < 2 && !(opt & NGRAM_WEIGHTED))
+        break;
+    }
+  }
+
+  ns = 0;
+  if (opt & NGRAM_LONGER_WORSE)
+    ns = (l2 - l1) - 2;
+  if (opt & NGRAM_ANY_MISMATCH)
+    ns = abs(l2 - l1) - 2;
+  ns = (nscore - ((ns > 0) ? ns : 0));
+  return ns;
+}
+
+// length of the left common substring of s1 and (decapitalised) s2
+int SuggestMgr::leftcommonsubstring(const char* s1, const char* s2) {
+  if (utf8) {
+    std::vector<w_char> su1;
+    std::vector<w_char> su2;
+    int l1 = u8_u16(su1, s1);
+    int l2 = u8_u16(su2, s2);
+    // decapitalize dictionary word
+    if (complexprefixes) {
+      if (su1[l1 - 1] == su2[l2 - 1])
+        return 1;
+    } else {
+      unsigned short idx = su2.empty() ? 0 : (su2[0].h << 8) + su2[0].l;
+      unsigned short otheridx = su1.empty() ? 0 : (su1[0].h << 8) + su1[0].l;
+      if (otheridx != idx && (otheridx != unicodetolower(idx, langnum)))
+        return 0;
+      int i;
+      for (i = 1; (i < l1) && (i < l2) && (su1[i].l == su2[i].l) &&
+                  (su1[i].h == su2[i].h);
+           i++)
+        ;
+      return i;
+    }
+  } else {
+    if (complexprefixes) {
+      int l1 = strlen(s1);
+      int l2 = strlen(s2);
+      if (l1 <= l2 && s2[l1 - 1] == s2[l2 - 1])
+        return 1;
+    } else if (csconv) {
+      const char* olds = s1;
+      // decapitalise dictionary word
+      if ((*s1 != *s2) && (*s1 != csconv[((unsigned char)*s2)].clower))
+        return 0;
+      do {
+        s1++;
+        s2++;
+      } while ((*s1 == *s2) && (*s1 != '\0'));
+      return (int)(s1 - olds);
+    }
+  }
+  return 0;
+}
+
+int SuggestMgr::commoncharacterpositions(const char* s1,
+                                         const char* s2,
+                                         int* is_swap) {
+  int num = 0;
+  int diff = 0;
+  int diffpos[2];
+  *is_swap = 0;
+  if (utf8) {
+    std::vector<w_char> su1;
+    std::vector<w_char> su2;
+    int l1 = u8_u16(su1, s1);
+    int l2 = u8_u16(su2, s2);
+
+    if (l1 <= 0 || l2 <= 0)
+      return 0;
+
+    // decapitalize dictionary word
+    if (complexprefixes) {
+      su2[l2 - 1] = lower_utf(su2[l2 - 1], langnum);
+    } else {
+      su2[0] = lower_utf(su2[0], langnum);
+    }
+    for (int i = 0; (i < l1) && (i < l2); i++) {
+      if (su1[i] == su2[i]) {
+        num++;
+      } else {
+        if (diff < 2)
+          diffpos[diff] = i;
+        diff++;
+      }
+    }
+    if ((diff == 2) && (l1 == l2) &&
+        (su1[diffpos[0]] == su2[diffpos[1]]) &&
+        (su1[diffpos[1]] == su2[diffpos[0]]))
+      *is_swap = 1;
+  } else {
+    size_t i;
+    std::string t(s2);
+    // decapitalize dictionary word
+    if (complexprefixes) {
+      size_t l2 = t.size();
+      t[l2 - 1] = csconv[(unsigned char)t[l2 - 1]].clower;
+    } else {
+      mkallsmall(t, csconv);
+    }
+    for (i = 0; (*(s1 + i) != 0) && i < t.size(); i++) {
+      if (*(s1 + i) == t[i]) {
+        num++;
+      } else {
+        if (diff < 2)
+          diffpos[diff] = i;
+        diff++;
+      }
+    }
+    if ((diff == 2) && (*(s1 + i) == 0) && i == t.size() &&
+        (*(s1 + diffpos[0]) == t[diffpos[1]]) &&
+        (*(s1 + diffpos[1]) == t[diffpos[0]]))
+      *is_swap = 1;
+  }
+  return num;
+}
+
+int SuggestMgr::mystrlen(const char* word) {
+  if (utf8) {
+    std::vector<w_char> w;
+    return u8_u16(w, word);
+  } else
+    return strlen(word);
+}
+
+// sort in decreasing order of score
+void SuggestMgr::bubblesort(char** rword, char** rword2, int* rsc, int n) {
+  int m = 1;
+  while (m < n) {
+    int j = m;
+    while (j > 0) {
+      if (rsc[j - 1] < rsc[j]) {
+        int sctmp = rsc[j - 1];
+        char* wdtmp = rword[j - 1];
+        rsc[j - 1] = rsc[j];
+        rword[j - 1] = rword[j];
+        rsc[j] = sctmp;
+        rword[j] = wdtmp;
+        if (rword2) {
+          wdtmp = rword2[j - 1];
+          rword2[j - 1] = rword2[j];
+          rword2[j] = wdtmp;
+        }
+        j--;
+      } else
+        break;
+    }
+    m++;
+  }
+  return;
+}
+
+// longest common subsequence
+void SuggestMgr::lcs(const char* s,
+                     const char* s2,
+                     int* l1,
+                     int* l2,
+                     char** result) {
+  int n, m;
+  std::vector<w_char> su;
+  std::vector<w_char> su2;
+  char* b;
+  char* c;
+  int i;
+  int j;
+  if (utf8) {
+    m = u8_u16(su, s);
+    n = u8_u16(su2, s2);
+  } else {
+    m = strlen(s);
+    n = strlen(s2);
+  }
+  c = (char*)malloc((m + 1) * (n + 1));
+  b = (char*)malloc((m + 1) * (n + 1));
+  if (!c || !b) {
+    if (c)
+      free(c);
+    if (b)
+      free(b);
+    *result = NULL;
+    return;
+  }
+  for (i = 1; i <= m; i++)
+    c[i * (n + 1)] = 0;
+  for (j = 0; j <= n; j++)
+    c[j] = 0;
+  for (i = 1; i <= m; i++) {
+    for (j = 1; j <= n; j++) {
+      if (((utf8) && (su[i - 1] == su2[j - 1])) ||
+          ((!utf8) && (s[i - 1] == s2[j - 1]))) {
+        c[i * (n + 1) + j] = c[(i - 1) * (n + 1) + j - 1] + 1;
+        b[i * (n + 1) + j] = LCS_UPLEFT;
+      } else if (c[(i - 1) * (n + 1) + j] >= c[i * (n + 1) + j - 1]) {
+        c[i * (n + 1) + j] = c[(i - 1) * (n + 1) + j];
+        b[i * (n + 1) + j] = LCS_UP;
+      } else {
+        c[i * (n + 1) + j] = c[i * (n + 1) + j - 1];
+        b[i * (n + 1) + j] = LCS_LEFT;
+      }
+    }
+  }
+  *result = b;
+  free(c);
+  *l1 = m;
+  *l2 = n;
+}
+
+int SuggestMgr::lcslen(const char* s, const char* s2) {
+  int m;
+  int n;
+  int i;
+  int j;
+  char* result;
+  int len = 0;
+  lcs(s, s2, &m, &n, &result);
+  if (!result)
+    return 0;
+  i = m;
+  j = n;
+  while ((i != 0) && (j != 0)) {
+    if (result[i * (n + 1) + j] == LCS_UPLEFT) {
+      len++;
+      i--;
+      j--;
+    } else if (result[i * (n + 1) + j] == LCS_UP) {
+      i--;
+    } else
+      j--;
+  }
+  free(result);
+  return len;
+}
+
+int SuggestMgr::lcslen(const std::string& s, const std::string& s2) {
+  return lcslen(s.c_str(), s2.c_str());
+}