Issue #1258 - Part 1: Import mailnews, ldap, and mork from comm-esr52.9.1

1 files changed, 1022 insertions, 0 deletions

diff --git a/mailnews/base/search/src/nsMsgLocalSearch.cpp b/mailnews/base/search/src/nsMsgLocalSearch.cpp
new file mode 100644
index 000000000..3ce510b6d
--- /dev/null
+++ b/mailnews/base/search/src/nsMsgLocalSearch.cpp
@@ -0,0 +1,1022 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Implementation of db search for POP and offline IMAP mail folders
+
+#include "msgCore.h"
+#include "nsIMsgDatabase.h"
+#include "nsMsgSearchCore.h"
+#include "nsMsgLocalSearch.h"
+#include "nsIStreamListener.h"
+#include "nsMsgSearchBoolExpression.h"
+#include "nsMsgSearchTerm.h"
+#include "nsMsgResultElement.h"
+#include "nsIDBFolderInfo.h"
+// Disable deprecation warnings generated by nsISupportsArray and associated
+// classes.
+#if defined(__GNUC__)
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+#elif defined(_MSC_VER)
+#pragma warning (disable : 4996)
+#endif
+#include "nsISupportsArray.h"
+#include "nsMsgBaseCID.h"
+#include "nsMsgSearchValue.h"
+#include "nsIMsgLocalMailFolder.h"
+#include "nsIMsgWindow.h"
+#include "nsIMsgHdr.h"
+#include "nsIMsgFilterPlugin.h"
+#include "nsMsgMessageFlags.h"
+#include "nsMsgUtils.h"
+#include "nsIMsgFolder.h"
+
+extern "C"
+{
+    extern int MK_MSG_SEARCH_STATUS;
+    extern int MK_MSG_CANT_SEARCH_IF_NO_SUMMARY;
+    extern int MK_MSG_SEARCH_HITS_NOT_IN_DB;
+}
+
+
+//----------------------------------------------------------------------------
+// Class definitions for the boolean expression structure....
+//----------------------------------------------------------------------------
+
+
+nsMsgSearchBoolExpression * nsMsgSearchBoolExpression::AddSearchTerm(nsMsgSearchBoolExpression * aOrigExpr, nsIMsgSearchTerm * aNewTerm, char * aEncodingStr)
+// appropriately add the search term to the current expression and return a pointer to the
+// new expression. The encodingStr is the IMAP/NNTP encoding string for newTerm.
+{
+    return aOrigExpr->leftToRightAddTerm(aNewTerm, aEncodingStr);
+}
+
+nsMsgSearchBoolExpression * nsMsgSearchBoolExpression::AddExpressionTree(nsMsgSearchBoolExpression * aOrigExpr, nsMsgSearchBoolExpression * aExpression, bool aBoolOp)
+{
+  if (!aOrigExpr->m_term && !aOrigExpr->m_leftChild && !aOrigExpr->m_rightChild)
+  {
+      // just use the original expression tree...
+      // delete the original since we have a new original to use
+      delete aOrigExpr;
+      return aExpression;
+  }
+
+  nsMsgSearchBoolExpression * newExpr = new nsMsgSearchBoolExpression (aOrigExpr, aExpression, aBoolOp);
+  return (newExpr) ? newExpr : aOrigExpr;
+}
+
+nsMsgSearchBoolExpression::nsMsgSearchBoolExpression()
+{
+    m_term = nullptr;
+    m_boolOp = nsMsgSearchBooleanOp::BooleanAND;
+    m_leftChild = nullptr;
+    m_rightChild = nullptr;
+}
+
+nsMsgSearchBoolExpression::nsMsgSearchBoolExpression (nsIMsgSearchTerm * newTerm, char * encodingStr)
+// we are creating an expression which contains a single search term (newTerm)
+// and the search term's IMAP or NNTP encoding value for online search expressions AND
+// a boolean evaluation value which is used for offline search expressions.
+{
+    m_term = newTerm;
+    m_encodingStr = encodingStr;
+    m_boolOp = nsMsgSearchBooleanOp::BooleanAND;
+
+    // this expression does not contain sub expressions
+    m_leftChild = nullptr;
+    m_rightChild = nullptr;
+}
+
+
+nsMsgSearchBoolExpression::nsMsgSearchBoolExpression (nsMsgSearchBoolExpression * expr1, nsMsgSearchBoolExpression * expr2, nsMsgSearchBooleanOperator boolOp)
+// we are creating an expression which contains two sub expressions and a boolean operator used to combine
+// them.
+{
+    m_leftChild = expr1;
+    m_rightChild = expr2;
+    m_boolOp = boolOp;
+
+    m_term = nullptr;
+}
+
+nsMsgSearchBoolExpression::~nsMsgSearchBoolExpression()
+{
+  // we must recursively destroy all sub expressions before we destroy ourself.....We leave search terms alone!
+  delete m_leftChild;
+  delete m_rightChild;
+}
+
+nsMsgSearchBoolExpression *
+nsMsgSearchBoolExpression::leftToRightAddTerm(nsIMsgSearchTerm * newTerm, char * encodingStr)
+{
+    // we have a base case where this is the first term being added to the expression:
+    if (!m_term && !m_leftChild && !m_rightChild)
+    {
+        m_term = newTerm;
+        m_encodingStr = encodingStr;
+        return this;
+    }
+
+    nsMsgSearchBoolExpression * tempExpr = new nsMsgSearchBoolExpression (newTerm,encodingStr);
+    if (tempExpr)  // make sure creation succeeded
+    {
+      bool booleanAnd;
+      newTerm->GetBooleanAnd(&booleanAnd);
+      nsMsgSearchBoolExpression * newExpr = new nsMsgSearchBoolExpression (this, tempExpr, booleanAnd);
+      if (newExpr)
+         return newExpr;
+      else
+         delete tempExpr;    // clean up memory allocation in case of failure
+    }
+    return this;   // in case we failed to create a new expression, return self
+}
+
+
+// returns true or false depending on what the current expression evaluates to.
+bool nsMsgSearchBoolExpression::OfflineEvaluate(nsIMsgDBHdr *msgToMatch, const char *defaultCharset,
+  nsIMsgSearchScopeTerm *scope, nsIMsgDatabase *db, const char *headers,
+  uint32_t headerSize, bool Filtering)
+{
+    bool result = true;    // always default to false positives
+    bool isAnd;
+
+    if (m_term) // do we contain just a search term?
+    {
+      nsMsgSearchOfflineMail::ProcessSearchTerm(msgToMatch, m_term,
+        defaultCharset, scope, db, headers, headerSize, Filtering, &result);
+      return result;
+    }
+
+    // otherwise we must recursively determine the value of our sub expressions
+
+    isAnd = (m_boolOp == nsMsgSearchBooleanOp::BooleanAND);
+
+    if (m_leftChild)
+    {
+        result = m_leftChild->OfflineEvaluate(msgToMatch, defaultCharset,
+          scope, db, headers, headerSize, Filtering);
+        if ( (result && !isAnd) || (!result && isAnd))
+          return result;
+    }
+
+    // If we got this far, either there was no leftChild (which is impossible)
+    // or we got (FALSE and OR) or (TRUE and AND) from the first result. That
+    // means the outcome depends entirely on the rightChild.
+    if (m_rightChild)
+        result = m_rightChild->OfflineEvaluate(msgToMatch, defaultCharset,
+          scope, db, headers, headerSize, Filtering);
+
+    return result;
+}
+
+// ### Maybe we can get rid of these because of our use of nsString???
+// constants used for online searching with IMAP/NNTP encoded search terms.
+// the + 1 is to account for null terminators we add at each stage of assembling the expression...
+const int sizeOfORTerm = 6+1;  // 6 bytes if we are combining two sub expressions with an OR term
+const int sizeOfANDTerm = 1+1; // 1 byte if we are combining two sub expressions with an AND term
+
+int32_t nsMsgSearchBoolExpression::CalcEncodeStrSize()
+// recursively examine each sub expression and calculate a final size for the entire IMAP/NNTP encoding
+{
+    if (!m_term && (!m_leftChild || !m_rightChild))   // is the expression empty?
+        return 0;
+    if (m_term)  // are we a leaf node?
+        return m_encodingStr.Length();
+    if (m_boolOp == nsMsgSearchBooleanOp::BooleanOR)
+        return sizeOfORTerm + m_leftChild->CalcEncodeStrSize() + m_rightChild->CalcEncodeStrSize();
+    if (m_boolOp == nsMsgSearchBooleanOp::BooleanAND)
+        return sizeOfANDTerm + m_leftChild->CalcEncodeStrSize() + m_rightChild->CalcEncodeStrSize();
+    return 0;
+}
+
+
+void nsMsgSearchBoolExpression::GenerateEncodeStr(nsCString * buffer)
+// recurively combine sub expressions to form a single IMAP/NNTP encoded string
+{
+    if ((!m_term && (!m_leftChild || !m_rightChild))) // is expression empty?
+        return;
+
+    if (m_term) // are we a leaf expression?
+    {
+        *buffer += m_encodingStr;
+        return;
+    }
+
+    // add encode strings of each sub expression
+    if (m_boolOp == nsMsgSearchBooleanOp::BooleanOR)
+    {
+        *buffer += " (OR";
+
+        m_leftChild->GenerateEncodeStr(buffer);  // insert left expression into the buffer
+        m_rightChild->GenerateEncodeStr(buffer);  // insert right expression into the buffer
+
+        // HACK ALERT!!! if last returned character in the buffer is now a ' ' then we need to remove it because we don't want
+        // a ' ' to preceded the closing paren in the OR encoding.
+        uint32_t lastCharPos = buffer->Length() - 1;
+        if (buffer->CharAt(lastCharPos) == ' ')
+        {
+            buffer->SetLength(lastCharPos);
+        }
+
+        *buffer += ')';
+    }
+    else if (m_boolOp == nsMsgSearchBooleanOp::BooleanAND)
+    {
+        m_leftChild->GenerateEncodeStr(buffer); // insert left expression
+        m_rightChild->GenerateEncodeStr(buffer);
+    }
+    return;
+}
+
+
+//-----------------------------------------------------------------------------
+//---------------- Adapter class for searching offline folders ----------------
+//-----------------------------------------------------------------------------
+
+
+NS_IMPL_ISUPPORTS_INHERITED(nsMsgSearchOfflineMail, nsMsgSearchAdapter, nsIUrlListener)
+
+nsMsgSearchOfflineMail::nsMsgSearchOfflineMail (nsIMsgSearchScopeTerm *scope, nsISupportsArray *termList) : nsMsgSearchAdapter (scope, termList)
+{
+}
+
+nsMsgSearchOfflineMail::~nsMsgSearchOfflineMail ()
+{
+  // Database should have been closed when the scope term finished.
+  CleanUpScope();
+  NS_ASSERTION(!m_db, "db not closed");
+}
+
+
+nsresult nsMsgSearchOfflineMail::ValidateTerms ()
+{
+  return nsMsgSearchAdapter::ValidateTerms ();
+}
+
+
+nsresult nsMsgSearchOfflineMail::OpenSummaryFile ()
+{
+    nsCOMPtr <nsIMsgDatabase> mailDB ;
+
+    nsresult err = NS_OK;
+    // do password protection of local cache thing.
+#ifdef DOING_FOLDER_CACHE_PASSWORDS
+    if (m_scope->m_folder && m_scope->m_folder->UserNeedsToAuthenticateForFolder(false) && m_scope->m_folder->GetMaster()->PromptForHostPassword(m_scope->m_frame->GetContext(), m_scope->m_folder) != 0)
+    {
+        m_scope->m_frame->StopRunning();
+        return SearchError_ScopeDone;
+    }
+#endif
+    nsCOMPtr <nsIDBFolderInfo>  folderInfo;
+    nsCOMPtr <nsIMsgFolder> scopeFolder;
+    err = m_scope->GetFolder(getter_AddRefs(scopeFolder));
+    if (NS_SUCCEEDED(err) && scopeFolder)
+    {
+      err = scopeFolder->GetDBFolderInfoAndDB(getter_AddRefs(folderInfo), getter_AddRefs(m_db));
+    }
+    else
+      return err; // not sure why m_folder wouldn't be set.
+
+    if (NS_SUCCEEDED(err))
+      return NS_OK;
+
+    if ((err == NS_MSG_ERROR_FOLDER_SUMMARY_MISSING) ||
+        (err == NS_MSG_ERROR_FOLDER_SUMMARY_OUT_OF_DATE))
+    {
+            nsCOMPtr<nsIMsgLocalMailFolder> localFolder = do_QueryInterface(scopeFolder, &err);
+            if (NS_SUCCEEDED(err) && localFolder)
+            {
+              nsCOMPtr<nsIMsgSearchSession> searchSession;
+              m_scope->GetSearchSession(getter_AddRefs(searchSession));
+              if (searchSession)
+              {
+                nsCOMPtr <nsIMsgWindow> searchWindow;
+
+                searchSession->GetWindow(getter_AddRefs(searchWindow));
+                searchSession->PauseSearch();
+                localFolder->ParseFolder(searchWindow, this);
+              }
+            }
+    }
+    else
+    {
+      NS_ASSERTION(false, "unexpected error opening db");
+    }
+
+    return err;
+}
+
+
+nsresult
+nsMsgSearchOfflineMail::MatchTermsForFilter(nsIMsgDBHdr *msgToMatch,
+                                            nsISupportsArray *termList,
+                                            const char *defaultCharset,
+                                            nsIMsgSearchScopeTerm * scope,
+                                            nsIMsgDatabase * db,
+                                            const char * headers,
+                                            uint32_t headerSize,
+                                            nsMsgSearchBoolExpression ** aExpressionTree,
+                                            bool *pResult)
+{
+    return MatchTerms(msgToMatch, termList, defaultCharset, scope, db, headers, headerSize, true, aExpressionTree, pResult);
+}
+
+// static method which matches a header against a list of search terms.
+nsresult
+nsMsgSearchOfflineMail::MatchTermsForSearch(nsIMsgDBHdr *msgToMatch,
+                                            nsISupportsArray* termList,
+                                            const char *defaultCharset,
+                                            nsIMsgSearchScopeTerm *scope,
+                                            nsIMsgDatabase *db,
+                                            nsMsgSearchBoolExpression ** aExpressionTree,
+                                            bool *pResult)
+{
+
+    return MatchTerms(msgToMatch, termList, defaultCharset, scope, db, nullptr, 0, false, aExpressionTree, pResult);
+}
+
+nsresult nsMsgSearchOfflineMail::ConstructExpressionTree(nsISupportsArray * termList,
+                                            uint32_t termCount,
+                                            uint32_t &aStartPosInList,
+                                            nsMsgSearchBoolExpression ** aExpressionTree)
+{
+  nsMsgSearchBoolExpression * finalExpression = *aExpressionTree;
+
+  if (!finalExpression)
+    finalExpression = new nsMsgSearchBoolExpression();
+
+  while (aStartPosInList < termCount)
+  {
+      nsCOMPtr<nsIMsgSearchTerm> pTerm;
+      termList->QueryElementAt(aStartPosInList, NS_GET_IID(nsIMsgSearchTerm), (void **)getter_AddRefs(pTerm));
+      NS_ASSERTION (pTerm, "couldn't get term to match");
+
+      bool beginsGrouping;
+      bool endsGrouping;
+      pTerm->GetBeginsGrouping(&beginsGrouping);
+      pTerm->GetEndsGrouping(&endsGrouping);
+
+      if (beginsGrouping)
+      {
+          //temporarily turn off the grouping for our recursive call
+          pTerm->SetBeginsGrouping(false);
+          nsMsgSearchBoolExpression * innerExpression = new nsMsgSearchBoolExpression();
+
+          // the first search term in the grouping is the one that holds the operator for how this search term
+          // should be joined with the expressions to it's left.
+          bool booleanAnd;
+          pTerm->GetBooleanAnd(&booleanAnd);
+
+          // now add this expression tree to our overall expression tree...
+          finalExpression = nsMsgSearchBoolExpression::AddExpressionTree(finalExpression, innerExpression, booleanAnd);
+
+          // recursively process this inner expression
+          ConstructExpressionTree(termList, termCount, aStartPosInList,
+            &finalExpression->m_rightChild);
+
+          // undo our damage
+          pTerm->SetBeginsGrouping(true);
+
+      }
+      else
+      {
+        finalExpression = nsMsgSearchBoolExpression::AddSearchTerm(finalExpression, pTerm, nullptr);    // add the term to the expression tree
+
+        if (endsGrouping)
+          break;
+      }
+
+      aStartPosInList++;
+  } // while we still have terms to process in this group
+
+  *aExpressionTree = finalExpression;
+
+  return NS_OK;
+}
+
+nsresult nsMsgSearchOfflineMail::ProcessSearchTerm(nsIMsgDBHdr *msgToMatch,
+                                            nsIMsgSearchTerm * aTerm,
+                                            const char *defaultCharset,
+                                            nsIMsgSearchScopeTerm * scope,
+                                            nsIMsgDatabase * db,
+                                            const char * headers,
+                                            uint32_t headerSize,
+                                            bool Filtering,
+                                            bool *pResult)
+{
+    nsresult err = NS_OK;
+    nsCString  recipients;
+    nsCString  ccList;
+    nsCString  matchString;
+    nsCString  msgCharset;
+    const char *charset;
+    bool charsetOverride = false; /* XXX BUG 68706 */
+    uint32_t msgFlags;
+    bool result;
+    bool matchAll;
+
+    NS_ENSURE_ARG_POINTER(pResult);
+
+    aTerm->GetMatchAll(&matchAll);
+    if (matchAll)
+    {
+      *pResult = true;
+      return NS_OK;
+    }
+    *pResult = false;
+
+    nsMsgSearchAttribValue attrib;
+    aTerm->GetAttrib(&attrib);
+    msgToMatch->GetCharset(getter_Copies(msgCharset));
+    charset = msgCharset.get();
+    if (!charset || !*charset)
+      charset = (const char*)defaultCharset;
+    msgToMatch->GetFlags(&msgFlags);
+
+    switch (attrib)
+    {
+      case nsMsgSearchAttrib::Sender:
+        msgToMatch->GetAuthor(getter_Copies(matchString));
+        err = aTerm->MatchRfc822String(matchString, charset, &result);
+        break;
+      case nsMsgSearchAttrib::Subject:
+      {
+        msgToMatch->GetSubject(getter_Copies(matchString) /* , true */);
+        if (msgFlags & nsMsgMessageFlags::HasRe)
+        {
+          // Make sure we pass along the "Re: " part of the subject if this is a reply.
+          nsCString reString;
+          reString.Assign("Re: ");
+          reString.Append(matchString);
+          err = aTerm->MatchRfc2047String(reString, charset, charsetOverride, &result);
+        }
+        else
+          err = aTerm->MatchRfc2047String(matchString, charset, charsetOverride, &result);
+        break;
+      }
+      case nsMsgSearchAttrib::ToOrCC:
+      {
+        bool boolKeepGoing;
+        aTerm->GetMatchAllBeforeDeciding(&boolKeepGoing);
+        msgToMatch->GetRecipients(getter_Copies(recipients));
+        err = aTerm->MatchRfc822String(recipients, charset, &result);
+        if (boolKeepGoing == result)
+        {
+          msgToMatch->GetCcList(getter_Copies(ccList));
+          err = aTerm->MatchRfc822String(ccList, charset, &result);
+        }
+        break;
+      }
+      case nsMsgSearchAttrib::AllAddresses:
+      {
+        bool boolKeepGoing;
+        aTerm->GetMatchAllBeforeDeciding(&boolKeepGoing);
+        msgToMatch->GetRecipients(getter_Copies(recipients));
+        err = aTerm->MatchRfc822String(recipients, charset, &result);
+        if (boolKeepGoing == result)
+        {
+          msgToMatch->GetCcList(getter_Copies(ccList));
+          err = aTerm->MatchRfc822String(ccList, charset, &result);
+        }
+        if (boolKeepGoing == result)
+        {
+          msgToMatch->GetAuthor(getter_Copies(matchString));
+          err = aTerm->MatchRfc822String(matchString, charset, &result);
+        }
+        if (boolKeepGoing == result)
+        {
+          nsCString bccList;
+          msgToMatch->GetBccList(getter_Copies(bccList));
+          err = aTerm->MatchRfc822String(bccList, charset, &result);
+        }
+        break;
+      }
+      case nsMsgSearchAttrib::Body:
+       {
+         uint64_t messageOffset;
+         uint32_t lineCount;
+         msgToMatch->GetMessageOffset(&messageOffset);
+         msgToMatch->GetLineCount(&lineCount);
+         err = aTerm->MatchBody (scope, messageOffset, lineCount, charset, msgToMatch, db, &result);
+         break;
+       }
+      case nsMsgSearchAttrib::Date:
+      {
+        PRTime date;
+        msgToMatch->GetDate(&date);
+        err = aTerm->MatchDate (date, &result);
+
+        break;
+      }
+      case nsMsgSearchAttrib::HasAttachmentStatus:
+      case nsMsgSearchAttrib::MsgStatus:
+        err = aTerm->MatchStatus (msgFlags, &result);
+        break;
+      case nsMsgSearchAttrib::Priority:
+      {
+        nsMsgPriorityValue msgPriority;
+        msgToMatch->GetPriority(&msgPriority);
+        err = aTerm->MatchPriority (msgPriority, &result);
+        break;
+      }
+      case nsMsgSearchAttrib::Size:
+      {
+        uint32_t messageSize;
+        msgToMatch->GetMessageSize(&messageSize);
+        err = aTerm->MatchSize (messageSize, &result);
+        break;
+      }
+      case nsMsgSearchAttrib::To:
+        msgToMatch->GetRecipients(getter_Copies(recipients));
+        err = aTerm->MatchRfc822String(recipients, charset, &result);
+        break;
+      case nsMsgSearchAttrib::CC:
+        msgToMatch->GetCcList(getter_Copies(ccList));
+        err = aTerm->MatchRfc822String(ccList, charset, &result);
+        break;
+      case nsMsgSearchAttrib::AgeInDays:
+      {
+        PRTime date;
+        msgToMatch->GetDate(&date);
+        err = aTerm->MatchAge (date, &result);
+        break;
+       }
+      case nsMsgSearchAttrib::Label:
+      {
+         nsMsgLabelValue label;
+         msgToMatch->GetLabel(&label);
+         err = aTerm->MatchLabel(label, &result);
+         break;
+      }
+      case nsMsgSearchAttrib::Keywords:
+      {
+          nsCString keywords;
+          nsMsgLabelValue label;
+          msgToMatch->GetStringProperty("keywords", getter_Copies(keywords));
+          msgToMatch->GetLabel(&label);
+          if (label >= 1)
+          {
+            if (!keywords.IsEmpty())
+              keywords.Append(' ');
+            keywords.Append("$label");
+            keywords.Append(label + '0');
+          }
+          err = aTerm->MatchKeyword(keywords, &result);
+          break;
+      }
+      case nsMsgSearchAttrib::JunkStatus:
+      {
+         nsCString junkScoreStr;
+         msgToMatch->GetStringProperty("junkscore", getter_Copies(junkScoreStr));
+         err = aTerm->MatchJunkStatus(junkScoreStr.get(), &result);
+         break;
+      }
+      case nsMsgSearchAttrib::JunkPercent:
+      {
+        // When the junk status is set by the plugin, use junkpercent (if available)
+        // Otherwise, use the limits (0 or 100) depending on the junkscore.
+        uint32_t junkPercent;
+        nsresult rv;
+        nsCString junkScoreOriginStr;
+        nsCString junkPercentStr;
+        msgToMatch->GetStringProperty("junkscoreorigin", getter_Copies(junkScoreOriginStr));
+        msgToMatch->GetStringProperty("junkpercent", getter_Copies(junkPercentStr));
+        if ( junkScoreOriginStr.EqualsLiteral("plugin") &&
+            !junkPercentStr.IsEmpty())
+        {
+          junkPercent = junkPercentStr.ToInteger(&rv);
+          NS_ENSURE_SUCCESS(rv, rv);
+        }
+        else
+        {
+          nsCString junkScoreStr;
+          msgToMatch->GetStringProperty("junkscore", getter_Copies(junkScoreStr));
+          // When junk status is not set (uncertain) we'll set the value to ham.
+          if (junkScoreStr.IsEmpty())
+            junkPercent = nsIJunkMailPlugin::IS_HAM_SCORE;
+          else
+          {
+            junkPercent = junkScoreStr.ToInteger(&rv);
+            NS_ENSURE_SUCCESS(rv, rv);
+          }
+        }
+        err = aTerm->MatchJunkPercent(junkPercent, &result);
+        break;
+      }
+      case nsMsgSearchAttrib::JunkScoreOrigin:
+      {
+         nsCString junkScoreOriginStr;
+         msgToMatch->GetStringProperty("junkscoreorigin", getter_Copies(junkScoreOriginStr));
+         err = aTerm->MatchJunkScoreOrigin(junkScoreOriginStr.get(), &result);
+         break;
+      }
+      case nsMsgSearchAttrib::HdrProperty:
+      {
+        err = aTerm->MatchHdrProperty(msgToMatch, &result);
+        break;
+      }
+      case nsMsgSearchAttrib::Uint32HdrProperty:
+      {
+        err = aTerm->MatchUint32HdrProperty(msgToMatch, &result);
+        break;
+      }
+      case nsMsgSearchAttrib::Custom:
+      {
+        err = aTerm->MatchCustom(msgToMatch, &result);
+        break;
+      }
+      case nsMsgSearchAttrib::FolderFlag:
+      {
+        err = aTerm->MatchFolderFlag(msgToMatch, &result);
+        break;
+      }
+      default:
+          // XXX todo
+          // for the temporary return receipts filters, we use a custom header for Content-Type
+          // but unlike the other custom headers, this one doesn't show up in the search / filter
+          // UI.  we set the attrib to be nsMsgSearchAttrib::OtherHeader, where as for user
+          // defined custom headers start at nsMsgSearchAttrib::OtherHeader + 1
+          // Not sure if there is a better way to do this yet.  Maybe reserve the last
+          // custom header for ::Content-Type?  But if we do, make sure that change
+          // doesn't cause nsMsgFilter::GetTerm() to change, and start making us
+          // ask IMAP servers for the Content-Type header on all messages.
+          if (attrib >= nsMsgSearchAttrib::OtherHeader &&
+              attrib < nsMsgSearchAttrib::kNumMsgSearchAttributes)
+          {
+            uint32_t lineCount;
+            msgToMatch->GetLineCount(&lineCount);
+            uint64_t messageOffset;
+            msgToMatch->GetMessageOffset(&messageOffset);
+            err = aTerm->MatchArbitraryHeader(scope, lineCount,charset,
+                                              charsetOverride, msgToMatch, db,
+                                              headers, headerSize, Filtering,
+                                              &result);
+          }
+          else {
+            err = NS_ERROR_INVALID_ARG; // ### was SearchError_InvalidAttribute
+            result = false;
+          }
+    }
+
+    *pResult = result;
+    return err;
+}
+
+nsresult nsMsgSearchOfflineMail::MatchTerms(nsIMsgDBHdr *msgToMatch,
+                                            nsISupportsArray * termList,
+                                            const char *defaultCharset,
+                                            nsIMsgSearchScopeTerm * scope,
+                                            nsIMsgDatabase * db,
+                                            const char * headers,
+                                            uint32_t headerSize,
+                                            bool Filtering,
+                                            nsMsgSearchBoolExpression ** aExpressionTree,
+                                            bool *pResult)
+{
+  NS_ENSURE_ARG(aExpressionTree);
+  nsresult err;
+
+  if (!*aExpressionTree)
+  {
+    uint32_t initialPos = 0;
+    uint32_t count;
+    termList->Count(&count);
+    err = ConstructExpressionTree(termList, count, initialPos, aExpressionTree);
+    if (NS_FAILED(err))
+      return err;
+  }
+
+  // evaluate the expression tree and return the result
+  *pResult = (*aExpressionTree)
+    ?  (*aExpressionTree)->OfflineEvaluate(msgToMatch,
+                 defaultCharset, scope, db, headers, headerSize, Filtering)
+    :true;  // vacuously true...
+
+  return NS_OK;
+}
+
+nsresult nsMsgSearchOfflineMail::Search(bool *aDone)
+{
+  nsresult err = NS_OK;
+
+  NS_ENSURE_ARG(aDone);
+  nsresult dbErr = NS_OK;
+  nsCOMPtr<nsIMsgDBHdr> msgDBHdr;
+  nsMsgSearchBoolExpression *expressionTree = nullptr;
+
+  const uint32_t kTimeSliceInMS = 200;
+
+  *aDone = false;
+  // Try to open the DB lazily. This will set up a parser if one is required
+  if (!m_db)
+    err = OpenSummaryFile ();
+  if (!m_db)  // must be reparsing.
+    return err;
+
+  // Reparsing is unnecessary or completed
+  if (NS_SUCCEEDED(err))
+  {
+    if (!m_listContext)
+      dbErr = m_db->ReverseEnumerateMessages(getter_AddRefs(m_listContext));
+    if (NS_SUCCEEDED(dbErr) && m_listContext)
+    {
+      PRIntervalTime startTime = PR_IntervalNow();
+      while (!*aDone)  // we'll break out of the loop after kTimeSliceInMS milliseconds
+      {
+        nsCOMPtr<nsISupports> currentItem;
+
+        dbErr = m_listContext->GetNext(getter_AddRefs(currentItem));
+        if(NS_SUCCEEDED(dbErr))
+        {
+          msgDBHdr = do_QueryInterface(currentItem, &dbErr);
+        }
+        if (NS_FAILED(dbErr))
+          *aDone = true; //###phil dbErr is dropped on the floor. just note that we did have an error so we'll clean up later
+        else
+        {
+          bool match = false;
+          nsAutoString nullCharset, folderCharset;
+          GetSearchCharsets(nullCharset, folderCharset);
+          NS_ConvertUTF16toUTF8 charset(folderCharset);
+          // Is this message a hit?
+          err = MatchTermsForSearch (msgDBHdr, m_searchTerms, charset.get(), m_scope, m_db, &expressionTree, &match);
+          // Add search hits to the results list
+          if (NS_SUCCEEDED(err) && match)
+          {
+            AddResultElement (msgDBHdr);
+          }
+          PRIntervalTime elapsedTime = PR_IntervalNow() - startTime;
+          // check if more than kTimeSliceInMS milliseconds have elapsed in this time slice started
+          if (PR_IntervalToMilliseconds(elapsedTime) > kTimeSliceInMS)
+            break;
+        }
+      }
+    }
+  }
+  else
+    *aDone = true; // we couldn't open up the DB. This is an unrecoverable error so mark the scope as done.
+
+  delete expressionTree;
+
+  // in the past an error here would cause an "infinite" search because the url would continue to run...
+  // i.e. if we couldn't open the database, it returns an error code but the caller of this function says, oh,
+  // we did not finish so continue...what we really want is to treat this current scope as done
+  if (*aDone)
+    CleanUpScope(); // Do clean up for end-of-scope processing
+  return err;
+}
+
+void nsMsgSearchOfflineMail::CleanUpScope()
+{
+  // Let go of the DB when we're done with it so we don't kill the db cache
+  if (m_db)
+  {
+    m_listContext = nullptr;
+    m_db->Close(false);
+  }
+  m_db = nullptr;
+
+  if (m_scope)
+    m_scope->CloseInputStream();
+}
+
+NS_IMETHODIMP nsMsgSearchOfflineMail::AddResultElement (nsIMsgDBHdr *pHeaders)
+{
+    nsresult err = NS_OK;
+
+    nsCOMPtr<nsIMsgSearchSession> searchSession;
+    m_scope->GetSearchSession(getter_AddRefs(searchSession));
+    if (searchSession)
+    {
+      nsCOMPtr <nsIMsgFolder> scopeFolder;
+      err = m_scope->GetFolder(getter_AddRefs(scopeFolder));
+      searchSession->AddSearchHit(pHeaders, scopeFolder);
+    }
+    return err;
+}
+
+NS_IMETHODIMP
+nsMsgSearchOfflineMail::Abort ()
+{
+    // Let go of the DB when we're done with it so we don't kill the db cache
+    if (m_db)
+        m_db->Close(true /* commit in case we downloaded new headers */);
+    m_db = nullptr;
+    return nsMsgSearchAdapter::Abort ();
+}
+
+/* void OnStartRunningUrl (in nsIURI url); */
+NS_IMETHODIMP nsMsgSearchOfflineMail::OnStartRunningUrl(nsIURI *url)
+{
+    return NS_OK;
+}
+
+/* void OnStopRunningUrl (in nsIURI url, in nsresult aExitCode); */
+NS_IMETHODIMP nsMsgSearchOfflineMail::OnStopRunningUrl(nsIURI *url, nsresult aExitCode)
+{
+  nsCOMPtr<nsIMsgSearchSession> searchSession;
+  if (m_scope)
+    m_scope->GetSearchSession(getter_AddRefs(searchSession));
+  if (searchSession)
+    searchSession->ResumeSearch();
+
+  return NS_OK;
+}
+
+nsMsgSearchOfflineNews::nsMsgSearchOfflineNews (nsIMsgSearchScopeTerm *scope, nsISupportsArray *termList) : nsMsgSearchOfflineMail (scope, termList)
+{
+}
+
+
+nsMsgSearchOfflineNews::~nsMsgSearchOfflineNews ()
+{
+}
+
+
+nsresult nsMsgSearchOfflineNews::OpenSummaryFile ()
+{
+  nsresult err = NS_OK;
+  nsCOMPtr <nsIDBFolderInfo>  folderInfo;
+  nsCOMPtr <nsIMsgFolder> scopeFolder;
+  err = m_scope->GetFolder(getter_AddRefs(scopeFolder));
+  // code here used to check if offline store existed, which breaks offline news.
+  if (NS_SUCCEEDED(err) && scopeFolder)
+    err = scopeFolder->GetMsgDatabase(getter_AddRefs(m_db));
+  return err;
+}
+
+nsresult nsMsgSearchOfflineNews::ValidateTerms ()
+{
+  return nsMsgSearchOfflineMail::ValidateTerms ();
+}
+
+// local helper functions to set subsets of the validity table
+
+nsresult SetJunk(nsIMsgSearchValidityTable* aTable)
+{
+  NS_ENSURE_ARG_POINTER(aTable);
+
+  aTable->SetAvailable(nsMsgSearchAttrib::JunkStatus, nsMsgSearchOp::Is, 1);
+  aTable->SetEnabled(nsMsgSearchAttrib::JunkStatus, nsMsgSearchOp::Is, 1);
+  aTable->SetAvailable(nsMsgSearchAttrib::JunkStatus, nsMsgSearchOp::Isnt, 1);
+  aTable->SetEnabled(nsMsgSearchAttrib::JunkStatus, nsMsgSearchOp::Isnt, 1);
+  aTable->SetAvailable(nsMsgSearchAttrib::JunkStatus, nsMsgSearchOp::IsEmpty, 1);
+  aTable->SetEnabled(nsMsgSearchAttrib::JunkStatus, nsMsgSearchOp::IsEmpty, 1);
+  aTable->SetAvailable(nsMsgSearchAttrib::JunkStatus, nsMsgSearchOp::IsntEmpty, 1);
+  aTable->SetEnabled(nsMsgSearchAttrib::JunkStatus, nsMsgSearchOp::IsntEmpty, 1);
+
+  aTable->SetAvailable(nsMsgSearchAttrib::JunkPercent, nsMsgSearchOp::IsGreaterThan, 1);
+  aTable->SetEnabled(nsMsgSearchAttrib::JunkPercent, nsMsgSearchOp::IsGreaterThan, 1);
+  aTable->SetAvailable(nsMsgSearchAttrib::JunkPercent, nsMsgSearchOp::IsLessThan, 1);
+  aTable->SetEnabled(nsMsgSearchAttrib::JunkPercent, nsMsgSearchOp::IsLessThan, 1);
+  aTable->SetAvailable(nsMsgSearchAttrib::JunkPercent, nsMsgSearchOp::Is, 1);
+  aTable->SetEnabled(nsMsgSearchAttrib::JunkPercent, nsMsgSearchOp::Is, 1);
+
+  aTable->SetAvailable(nsMsgSearchAttrib::JunkScoreOrigin, nsMsgSearchOp::Is, 1);
+  aTable->SetEnabled(nsMsgSearchAttrib::JunkScoreOrigin, nsMsgSearchOp::Is, 1);
+  aTable->SetAvailable(nsMsgSearchAttrib::JunkScoreOrigin, nsMsgSearchOp::Isnt, 1);
+  aTable->SetEnabled(nsMsgSearchAttrib::JunkScoreOrigin, nsMsgSearchOp::Isnt, 1);
+
+  return NS_OK;
+}
+
+nsresult SetBody(nsIMsgSearchValidityTable* aTable)
+{
+  NS_ENSURE_ARG_POINTER(aTable);
+  aTable->SetAvailable (nsMsgSearchAttrib::Body, nsMsgSearchOp::Contains, 1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::Body, nsMsgSearchOp::Contains, 1);
+  aTable->SetAvailable (nsMsgSearchAttrib::Body, nsMsgSearchOp::DoesntContain, 1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::Body, nsMsgSearchOp::DoesntContain, 1);
+  aTable->SetAvailable (nsMsgSearchAttrib::Body, nsMsgSearchOp::Is, 1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::Body, nsMsgSearchOp::Is, 1);
+  aTable->SetAvailable (nsMsgSearchAttrib::Body, nsMsgSearchOp::Isnt, 1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::Body, nsMsgSearchOp::Isnt, 1);
+
+  return NS_OK;
+}
+
+// set the base validity table values for local news
+nsresult SetLocalNews(nsIMsgSearchValidityTable* aTable)
+{
+  NS_ENSURE_ARG_POINTER(aTable);
+
+  aTable->SetAvailable (nsMsgSearchAttrib::Sender, nsMsgSearchOp::Contains, 1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::Sender, nsMsgSearchOp::Contains, 1);
+  aTable->SetAvailable (nsMsgSearchAttrib::Sender, nsMsgSearchOp::DoesntContain, 1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::Sender, nsMsgSearchOp::DoesntContain, 1);
+  aTable->SetAvailable (nsMsgSearchAttrib::Sender, nsMsgSearchOp::Is, 1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::Sender, nsMsgSearchOp::Is, 1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::Sender, nsMsgSearchOp::Isnt, 1);
+  aTable->SetAvailable (nsMsgSearchAttrib::Sender, nsMsgSearchOp::Isnt, 1);
+  aTable->SetAvailable (nsMsgSearchAttrib::Sender, nsMsgSearchOp::BeginsWith, 1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::Sender, nsMsgSearchOp::BeginsWith, 1);
+  aTable->SetAvailable (nsMsgSearchAttrib::Sender, nsMsgSearchOp::EndsWith, 1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::Sender, nsMsgSearchOp::EndsWith, 1);
+  aTable->SetAvailable (nsMsgSearchAttrib::Sender, nsMsgSearchOp::IsInAB, 1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::Sender, nsMsgSearchOp::IsInAB, 1);
+  aTable->SetAvailable (nsMsgSearchAttrib::Sender, nsMsgSearchOp::IsntInAB, 1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::Sender, nsMsgSearchOp::IsntInAB, 1);
+
+  aTable->SetAvailable (nsMsgSearchAttrib::Subject, nsMsgSearchOp::Contains, 1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::Subject, nsMsgSearchOp::Contains, 1);
+  aTable->SetAvailable (nsMsgSearchAttrib::Subject, nsMsgSearchOp::DoesntContain, 1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::Subject, nsMsgSearchOp::DoesntContain, 1);
+  aTable->SetAvailable (nsMsgSearchAttrib::Subject, nsMsgSearchOp::Is, 1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::Subject, nsMsgSearchOp::Is, 1);
+  aTable->SetAvailable (nsMsgSearchAttrib::Subject, nsMsgSearchOp::Isnt, 1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::Subject, nsMsgSearchOp::Isnt, 1);
+  aTable->SetAvailable (nsMsgSearchAttrib::Subject, nsMsgSearchOp::BeginsWith, 1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::Subject, nsMsgSearchOp::BeginsWith, 1);
+  aTable->SetAvailable (nsMsgSearchAttrib::Subject, nsMsgSearchOp::EndsWith, 1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::Subject, nsMsgSearchOp::EndsWith, 1);
+
+  aTable->SetAvailable (nsMsgSearchAttrib::Date, nsMsgSearchOp::IsBefore, 1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::Date, nsMsgSearchOp::IsBefore, 1);
+  aTable->SetAvailable (nsMsgSearchAttrib::Date, nsMsgSearchOp::IsAfter, 1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::Date, nsMsgSearchOp::IsAfter, 1);
+  aTable->SetAvailable (nsMsgSearchAttrib::Date, nsMsgSearchOp::Is, 1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::Date, nsMsgSearchOp::Is, 1);
+  aTable->SetAvailable (nsMsgSearchAttrib::Date, nsMsgSearchOp::Isnt, 1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::Date, nsMsgSearchOp::Isnt, 1);
+
+  aTable->SetAvailable (nsMsgSearchAttrib::AgeInDays, nsMsgSearchOp::IsGreaterThan, 1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::AgeInDays, nsMsgSearchOp::IsGreaterThan, 1);
+  aTable->SetAvailable (nsMsgSearchAttrib::AgeInDays, nsMsgSearchOp::IsLessThan,  1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::AgeInDays, nsMsgSearchOp::IsLessThan, 1);
+  aTable->SetAvailable (nsMsgSearchAttrib::AgeInDays, nsMsgSearchOp::Is,  1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::AgeInDays, nsMsgSearchOp::Is, 1);
+
+  aTable->SetAvailable (nsMsgSearchAttrib::MsgStatus, nsMsgSearchOp::Is, 1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::MsgStatus, nsMsgSearchOp::Is, 1);
+  aTable->SetAvailable (nsMsgSearchAttrib::MsgStatus, nsMsgSearchOp::Isnt, 1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::MsgStatus, nsMsgSearchOp::Isnt, 1);
+
+  aTable->SetAvailable (nsMsgSearchAttrib::Keywords, nsMsgSearchOp::Contains, 1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::Keywords, nsMsgSearchOp::Contains, 1);
+  aTable->SetAvailable (nsMsgSearchAttrib::Keywords, nsMsgSearchOp::DoesntContain, 1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::Keywords, nsMsgSearchOp::DoesntContain, 1);
+  aTable->SetAvailable (nsMsgSearchAttrib::Keywords, nsMsgSearchOp::Is, 1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::Keywords, nsMsgSearchOp::Is, 1);
+  aTable->SetAvailable (nsMsgSearchAttrib::Keywords, nsMsgSearchOp::Isnt, 1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::Keywords, nsMsgSearchOp::Isnt, 1);
+  aTable->SetAvailable (nsMsgSearchAttrib::Keywords, nsMsgSearchOp::IsEmpty, 1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::Keywords, nsMsgSearchOp::IsEmpty, 1);
+  aTable->SetAvailable (nsMsgSearchAttrib::Keywords, nsMsgSearchOp::IsntEmpty, 1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::Keywords, nsMsgSearchOp::IsntEmpty, 1);
+
+  aTable->SetAvailable (nsMsgSearchAttrib::OtherHeader, nsMsgSearchOp::Contains, 1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::OtherHeader, nsMsgSearchOp::Contains, 1);
+  aTable->SetAvailable (nsMsgSearchAttrib::OtherHeader, nsMsgSearchOp::DoesntContain, 1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::OtherHeader, nsMsgSearchOp::DoesntContain, 1);
+  aTable->SetAvailable (nsMsgSearchAttrib::OtherHeader, nsMsgSearchOp::Is, 1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::OtherHeader, nsMsgSearchOp::Is, 1);
+  aTable->SetAvailable (nsMsgSearchAttrib::OtherHeader, nsMsgSearchOp::Isnt, 1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::OtherHeader, nsMsgSearchOp::Isnt, 1);
+  aTable->SetAvailable (nsMsgSearchAttrib::OtherHeader, nsMsgSearchOp::BeginsWith, 1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::OtherHeader, nsMsgSearchOp::BeginsWith, 1);
+  aTable->SetAvailable (nsMsgSearchAttrib::OtherHeader, nsMsgSearchOp::EndsWith, 1);
+  aTable->SetEnabled   (nsMsgSearchAttrib::OtherHeader, nsMsgSearchOp::EndsWith, 1);
+  return NS_OK;
+
+}
+
+nsresult nsMsgSearchValidityManager::InitLocalNewsTable()
+{
+  NS_ASSERTION (nullptr == m_localNewsTable, "already have local news validity table");
+  nsresult rv = NewTable(getter_AddRefs(m_localNewsTable));
+  NS_ENSURE_SUCCESS(rv, rv);
+  return SetLocalNews(m_localNewsTable);
+}
+
+nsresult nsMsgSearchValidityManager::InitLocalNewsBodyTable()
+{
+  NS_ASSERTION (nullptr == m_localNewsBodyTable, "already have local news+body validity table");
+  nsresult rv = NewTable(getter_AddRefs(m_localNewsBodyTable));
+  NS_ENSURE_SUCCESS(rv, rv);
+  rv = SetLocalNews(m_localNewsBodyTable);
+  NS_ENSURE_SUCCESS(rv, rv);
+  return SetBody(m_localNewsBodyTable);
+}
+
+nsresult nsMsgSearchValidityManager::InitLocalNewsJunkTable()
+{
+  NS_ASSERTION (nullptr == m_localNewsJunkTable, "already have local news+junk validity table");
+  nsresult rv = NewTable(getter_AddRefs(m_localNewsJunkTable));
+  NS_ENSURE_SUCCESS(rv, rv);
+  rv = SetLocalNews(m_localNewsJunkTable);
+  NS_ENSURE_SUCCESS(rv, rv);
+  return SetJunk(m_localNewsJunkTable);
+}
+
+nsresult nsMsgSearchValidityManager::InitLocalNewsJunkBodyTable()
+{
+  NS_ASSERTION (nullptr == m_localNewsJunkBodyTable, "already have local news+junk+body validity table");
+  nsresult rv = NewTable(getter_AddRefs(m_localNewsJunkBodyTable));
+  NS_ENSURE_SUCCESS(rv, rv);
+  rv = SetLocalNews(m_localNewsJunkBodyTable);
+  NS_ENSURE_SUCCESS(rv, rv);
+  rv = SetJunk(m_localNewsJunkBodyTable);
+  NS_ENSURE_SUCCESS(rv, rv);
+  return SetBody(m_localNewsJunkBodyTable);
+}