Bug 1427732 - fix newline handling when copying messages and compacting folders.

Tag #1273

6 files changed, 63 insertions, 30 deletions

diff --git a/mailnews/base/public/msgCore.h b/mailnews/base/public/msgCore.h
index fc18fb16f..d14ff6140 100644
--- a/mailnews/base/public/msgCore.h
+++ b/mailnews/base/public/msgCore.h
@@ -178,6 +178,19 @@ NS_ERROR_GENERATE_FAILURE(NS_ERROR_MODULE_MAILNEWS, value)
 #define MSG_LINEBREAK_LEN 1
 #endif
 
+/*
+ * On Windows, we use \r\n as the line terminator in mbox files. On
+ * other platforms, we use \n. However, we need to be able to
+ * recognize line terminators produced on any platform, because we
+ * allow profiles (including the mbox files they contain) to be shared
+ * between platforms.
+ *
+ * Returns 0 (i.e., false) if the line is not blank, or otherwise the
+ * length of the line terminator, i.e., 1 for \n or 2 for \r\n.
+ */
+#define IS_MSG_LINEBREAK(line) \
+  (line[0] == '\012' ? 1 : ((line[0] == '\015' && line[1] == '\012') ? 2 : 0))
+
 #define NS_MSG_BASE
 #define NS_MSG_BASE_STATIC_MEMBER_(type) type
 
diff --git a/mailnews/base/src/nsMsgFolderCompactor.cpp b/mailnews/base/src/nsMsgFolderCompactor.cpp
index 4b0dc3ad5..5224aef51 100644
--- a/mailnews/base/src/nsMsgFolderCompactor.cpp
+++ b/mailnews/base/src/nsMsgFolderCompactor.cpp
@@ -734,6 +734,7 @@ nsFolderCompactState::OnStopRequest(nsIRequest *request, nsISupports *ctxt,
   }
   else
   {
+    // XXX TODO: Error checking and handling missing here.
     EndCopy(nullptr, status);
     if (m_curIndex >= m_size)
     {
@@ -754,6 +755,8 @@ nsFolderCompactState::OnStopRequest(nsIRequest *request, nsISupports *ctxt,
   return status;
 }
 
+// XXX TODO: This function is sadly lacking all status checking, it always
+// returns NS_OK and moves onto the next message.
 NS_IMETHODIMP
 nsFolderCompactState::OnDataAvailable(nsIRequest *request, nsISupports *ctxt,
                                       nsIInputStream *inStr,
@@ -784,7 +787,7 @@ nsFolderCompactState::OnDataAvailable(nsIRequest *request, nsISupports *ctxt,
       {
         (void) m_curSrcHdr->GetFlags(&msgFlags);
         (void) m_curSrcHdr->GetStatusOffset(&statusOffset);
-        
+
         if (statusOffset == 0)
           m_needStatusLine = true;
         // x-mozilla-status lines should be at the start of the headers, and the code
@@ -794,7 +797,7 @@ nsFolderCompactState::OnDataAvailable(nsIRequest *request, nsISupports *ctxt,
         {
           checkForKeyword = false;
           NS_ASSERTION(false, "status offset past end of read buffer size");
-          
+
         }
       }
     }
@@ -802,18 +805,18 @@ nsFolderCompactState::OnDataAvailable(nsIRequest *request, nsISupports *ctxt,
   }
   uint32_t maxReadCount, readCount, writeCount;
   uint32_t bytesWritten;
-  
+
   while (NS_SUCCEEDED(rv) && (int32_t) count > 0)
   {
     maxReadCount = count > sizeof(m_dataBuffer) - 1 ? sizeof(m_dataBuffer) - 1 : count;
     writeCount = 0;
     rv = inStr->Read(m_dataBuffer, maxReadCount, &readCount);
-    
+
     // if status offset is past the number of bytes we read, it's probably bogus,
     // and we shouldn't do any of the keyword stuff.
     if (statusOffset + X_MOZILLA_STATUS_LEN > readCount)
       checkForKeyword = false;
-    
+
     if (NS_SUCCEEDED(rv))
     {
       if (checkForKeyword)
@@ -967,7 +970,7 @@ nsFolderCompactState::OnDataAvailable(nsIRequest *request, nsISupports *ctxt,
       {
         NS_ASSERTION(false, "bad block offset");
         // not sure what to do to handle this.
-      
+
       }
       m_fileStream->Write(m_dataBuffer + blockOffset, readCount - blockOffset, &bytesWritten);
       writeCount += bytesWritten;
@@ -1131,7 +1134,6 @@ done:
   }
   return rv;
 }
- 
 
 nsresult
 nsOfflineStoreCompactState::FinishCompact()
@@ -1226,6 +1228,10 @@ nsFolderCompactState::EndCopy(nsISupports *url, nsresult aStatus)
     return NS_OK;
   }
 
+  /* Messages need to have trailing blank lines */
+  uint32_t bytesWritten;
+  (void) m_fileStream->Write(MSG_LINEBREAK, MSG_LINEBREAK_LEN, &bytesWritten);
+
   /**
    * Done with the current message; copying the existing message header
    * to the new database.
@@ -1255,7 +1261,7 @@ nsFolderCompactState::EndCopy(nsISupports *url, nsresult aStatus)
       msgSize += m_addedHeaderSize;
       newMsgHdr->SetMessageSize(msgSize);
     }
-    m_totalMsgSize += msgSize;
+    m_totalMsgSize += msgSize + MSG_LINEBREAK_LEN;
   }
 
 //  m_db->Commit(nsMsgDBCommitType::kLargeCommit);  // no sense commiting until the end
@@ -1345,4 +1351,3 @@ nsOfflineStoreCompactState::OnDataAvailable(nsIRequest *request, nsISupports *ct
   }
   return rv;
 }
-
diff --git a/mailnews/local/src/nsLocalMailFolder.cpp b/mailnews/local/src/nsLocalMailFolder.cpp
index 14135fe46..00244ca87 100644
--- a/mailnews/local/src/nsLocalMailFolder.cpp
+++ b/mailnews/local/src/nsLocalMailFolder.cpp
@@ -2117,7 +2117,6 @@ nsresult nsMsgLocalMailFolder::WriteStartOfNewMessage()
 
 nsresult nsMsgLocalMailFolder::InitCopyMsgHdrAndFileStream()
 {
-  nsCOMPtr<nsIMsgPluggableStore> msgStore;
   nsresult rv = GetMsgStore(getter_AddRefs(mCopyState->m_msgStore));
   NS_ENSURE_SUCCESS(rv, rv);
   bool reusable;
@@ -2387,16 +2386,10 @@ NS_IMETHODIMP nsMsgLocalMailFolder::EndCopy(bool aCopySucceeded)
   nsCOMPtr <nsISeekableStream> seekableStream(do_QueryInterface(mCopyState->m_fileStream));
   if (seekableStream)
   {
-    if (mCopyState->m_dummyEnvelopeNeeded)
-    {
-      uint32_t bytesWritten;
-      seekableStream->Seek(nsISeekableStream::NS_SEEK_END, 0);
-      mCopyState->m_fileStream->Write(MSG_LINEBREAK, MSG_LINEBREAK_LEN, &bytesWritten);
-      if (mCopyState->m_parseMsgState)
-        mCopyState->m_parseMsgState->ParseAFolderLine(CRLF, MSG_LINEBREAK_LEN);
-    }
-    rv = mCopyState->m_msgStore->FinishNewMessage(mCopyState->m_fileStream,
-                                                  mCopyState->m_newHdr);
+    seekableStream->Seek(nsISeekableStream::NS_SEEK_END, 0);
+    rv = FinishNewLocalMessage(mCopyState->m_fileStream, mCopyState->m_newHdr,
+                               mCopyState->m_msgStore,
+                               mCopyState->m_parseMsgState);
     if (NS_SUCCEEDED(rv) && mCopyState->m_newHdr)
       mCopyState->m_newHdr->GetMessageKey(&mCopyState->m_curDstKey);
     if (multipleCopiesFinished)
@@ -2714,13 +2707,9 @@ NS_IMETHODIMP nsMsgLocalMailFolder::EndMessage(nsMsgKey key)
   nsCOMPtr <nsISeekableStream> seekableStream = do_QueryInterface(mCopyState->m_fileStream, &rv);
   NS_ENSURE_SUCCESS(rv, rv);
   seekableStream->Seek(nsISeekableStream::NS_SEEK_END, 0);
-  uint32_t bytesWritten;
-  mCopyState->m_fileStream->Write(MSG_LINEBREAK, MSG_LINEBREAK_LEN, &bytesWritten);
-  if (mCopyState->m_parseMsgState)
-    mCopyState->m_parseMsgState->ParseAFolderLine(CRLF, MSG_LINEBREAK_LEN);
-
-  rv = mCopyState->m_msgStore->FinishNewMessage(mCopyState->m_fileStream,
-                                                mCopyState->m_newHdr);
+  rv = FinishNewLocalMessage(mCopyState->m_fileStream, mCopyState->m_newHdr,
+                             mCopyState->m_msgStore,
+                             mCopyState->m_parseMsgState);
   mCopyState->m_fileStream->Close();
   mCopyState->m_fileStream = nullptr; // all done with the file stream
 
@@ -3739,7 +3728,7 @@ nsMsgLocalMailFolder::AddMessageBatch(uint32_t aMessageCount,
       outFileStream->Write(aMessages[i], messageLen, &bytesWritten);
       newMailParser->BufferInput(aMessages[i], messageLen);
 
-      msgStore->FinishNewMessage(outFileStream, newHdr);
+      FinishNewLocalMessage(outFileStream, newHdr, msgStore, newMailParser);
       outFileStream->Close();
       outFileStream = nullptr;
       newMailParser->OnStopRequest(nullptr, nullptr, NS_OK);
@@ -3752,6 +3741,19 @@ nsMsgLocalMailFolder::AddMessageBatch(uint32_t aMessageCount,
   return rv;
 }
 
+nsresult
+nsMsgLocalMailFolder::FinishNewLocalMessage(nsIOutputStream *aOutputStream,
+                                            nsIMsgDBHdr *aNewHdr,
+                                            nsIMsgPluggableStore *aMsgStore,
+                                            nsParseMailMessageState *aParseMsgState)
+{
+  uint32_t bytesWritten;
+  aOutputStream->Write(MSG_LINEBREAK, MSG_LINEBREAK_LEN, &bytesWritten);
+  if (aParseMsgState)
+    aParseMsgState->ParseAFolderLine(MSG_LINEBREAK, MSG_LINEBREAK_LEN);
+  return aMsgStore->FinishNewMessage(aOutputStream, aNewHdr);
+}
+
 NS_IMETHODIMP
 nsMsgLocalMailFolder::WarnIfLocalFileTooBig(nsIMsgWindow *aWindow,
                                             int64_t aSpaceRequested,
diff --git a/mailnews/local/src/nsLocalMailFolder.h b/mailnews/local/src/nsLocalMailFolder.h
index e8c395342..b2503a157 100644
--- a/mailnews/local/src/nsLocalMailFolder.h
+++ b/mailnews/local/src/nsLocalMailFolder.h
@@ -234,6 +234,10 @@ protected:
   void CopyHdrPropertiesWithSkipList(nsIMsgDBHdr *destHdr,
                                      nsIMsgDBHdr *srcHdr,
                                      const nsCString &skipList);
+  nsresult FinishNewLocalMessage(nsIOutputStream *outputStream,
+                                 nsIMsgDBHdr *newHdr,
+                                 nsIMsgPluggableStore *msgStore,
+                                 nsParseMailMessageState *parseMsgState);
 
 protected:
   nsLocalMailCopyState *mCopyState; //We only allow one of these at a time
diff --git a/mailnews/local/src/nsParseMailbox.cpp b/mailnews/local/src/nsParseMailbox.cpp
index 9d68e5cd1..da51c0322 100644
--- a/mailnews/local/src/nsParseMailbox.cpp
+++ b/mailnews/local/src/nsParseMailbox.cpp
@@ -598,6 +598,7 @@ NS_IMETHODIMP nsParseMailMessageState::Clear()
   m_mdn_original_recipient.length = 0;
   m_bccList.length = 0;
   m_body_lines = 0;
+  m_lastLineBlank = 0;
   m_newMsgHdr = nullptr;
   m_envelope_pos = 0;
   m_new_key = nsMsgKey_None;
@@ -694,6 +695,9 @@ nsresult nsParseMailMessageState::ParseFolderLine(const char *line, uint32_t lin
   else if ( m_state == nsIMsgParseMailMsgState::ParseBodyState)
   {
     m_body_lines++;
+    // See comment in msgCore.h for why we use `IS_MSG_LINEBREAK` rather than
+    // just comparing `line` to `MSG_LINEBREAK`.
+    m_lastLineBlank = IS_MSG_LINEBREAK(line);
   }
 
   m_position += lineLength;
@@ -824,7 +828,9 @@ NS_IMETHODIMP nsParseMailMessageState::FinishHeader()
   if (m_newMsgHdr)
   {
     m_newMsgHdr->SetMessageOffset(m_envelope_pos);
-    m_newMsgHdr->SetMessageSize(m_position - m_envelope_pos);
+    if (m_lastLineBlank)
+      m_body_lines--;
+    m_newMsgHdr->SetMessageSize(m_position - m_envelope_pos - m_lastLineBlank);
     m_newMsgHdr->SetLineCount(m_body_lines);
   }
   return NS_OK;
@@ -1888,7 +1894,9 @@ NS_IMETHODIMP nsParseNewMailState::FinishHeader()
 {
   if (m_newMsgHdr)
   {
-    m_newMsgHdr->SetMessageSize(m_position - m_envelope_pos);
+    if (m_lastLineBlank)
+      m_body_lines--;
+    m_newMsgHdr->SetMessageSize(m_position - m_envelope_pos - m_lastLineBlank);
     m_newMsgHdr->SetLineCount(m_body_lines);
   }
   return NS_OK;
diff --git a/mailnews/local/src/nsParseMailbox.h b/mailnews/local/src/nsParseMailbox.h
index 388f1e1b2..aac40611a 100644
--- a/mailnews/local/src/nsParseMailbox.h
+++ b/mailnews/local/src/nsParseMailbox.h
@@ -115,6 +115,7 @@ public:
 
   PRTime m_receivedTime;
   uint16_t              m_body_lines;
+  uint16_t              m_lastLineBlank;
 
   bool                  m_IgnoreXMozillaStatus;