From e60090bc9c7e14bb8253eeb64658aedaa0f863c7 Mon Sep 17 00:00:00 2001 From: "Matt A. Tobin" Date: Sun, 10 Nov 2019 18:43:43 -0500 Subject: Bug 1427732 - fix newline handling when copying messages and compacting folders. Tag #1273 --- mailnews/base/public/msgCore.h | 13 ++++++++++ mailnews/base/src/nsMsgFolderCompactor.cpp | 23 ++++++++++------- mailnews/local/src/nsLocalMailFolder.cpp | 40 ++++++++++++++++-------------- mailnews/local/src/nsLocalMailFolder.h | 4 +++ mailnews/local/src/nsParseMailbox.cpp | 12 +++++++-- mailnews/local/src/nsParseMailbox.h | 1 + 6 files changed, 63 insertions(+), 30 deletions(-) diff --git a/mailnews/base/public/msgCore.h b/mailnews/base/public/msgCore.h index fc18fb16f..d14ff6140 100644 --- a/mailnews/base/public/msgCore.h +++ b/mailnews/base/public/msgCore.h @@ -178,6 +178,19 @@ NS_ERROR_GENERATE_FAILURE(NS_ERROR_MODULE_MAILNEWS, value) #define MSG_LINEBREAK_LEN 1 #endif +/* + * On Windows, we use \r\n as the line terminator in mbox files. On + * other platforms, we use \n. However, we need to be able to + * recognize line terminators produced on any platform, because we + * allow profiles (including the mbox files they contain) to be shared + * between platforms. + * + * Returns 0 (i.e., false) if the line is not blank, or otherwise the + * length of the line terminator, i.e., 1 for \n or 2 for \r\n. + */ +#define IS_MSG_LINEBREAK(line) \ + (line[0] == '\012' ? 1 : ((line[0] == '\015' && line[1] == '\012') ? 2 : 0)) + #define NS_MSG_BASE #define NS_MSG_BASE_STATIC_MEMBER_(type) type diff --git a/mailnews/base/src/nsMsgFolderCompactor.cpp b/mailnews/base/src/nsMsgFolderCompactor.cpp index 4b0dc3ad5..5224aef51 100644 --- a/mailnews/base/src/nsMsgFolderCompactor.cpp +++ b/mailnews/base/src/nsMsgFolderCompactor.cpp @@ -734,6 +734,7 @@ nsFolderCompactState::OnStopRequest(nsIRequest *request, nsISupports *ctxt, } else { + // XXX TODO: Error checking and handling missing here. EndCopy(nullptr, status); if (m_curIndex >= m_size) { @@ -754,6 +755,8 @@ nsFolderCompactState::OnStopRequest(nsIRequest *request, nsISupports *ctxt, return status; } +// XXX TODO: This function is sadly lacking all status checking, it always +// returns NS_OK and moves onto the next message. NS_IMETHODIMP nsFolderCompactState::OnDataAvailable(nsIRequest *request, nsISupports *ctxt, nsIInputStream *inStr, @@ -784,7 +787,7 @@ nsFolderCompactState::OnDataAvailable(nsIRequest *request, nsISupports *ctxt, { (void) m_curSrcHdr->GetFlags(&msgFlags); (void) m_curSrcHdr->GetStatusOffset(&statusOffset); - + if (statusOffset == 0) m_needStatusLine = true; // x-mozilla-status lines should be at the start of the headers, and the code @@ -794,7 +797,7 @@ nsFolderCompactState::OnDataAvailable(nsIRequest *request, nsISupports *ctxt, { checkForKeyword = false; NS_ASSERTION(false, "status offset past end of read buffer size"); - + } } } @@ -802,18 +805,18 @@ nsFolderCompactState::OnDataAvailable(nsIRequest *request, nsISupports *ctxt, } uint32_t maxReadCount, readCount, writeCount; uint32_t bytesWritten; - + while (NS_SUCCEEDED(rv) && (int32_t) count > 0) { maxReadCount = count > sizeof(m_dataBuffer) - 1 ? sizeof(m_dataBuffer) - 1 : count; writeCount = 0; rv = inStr->Read(m_dataBuffer, maxReadCount, &readCount); - + // if status offset is past the number of bytes we read, it's probably bogus, // and we shouldn't do any of the keyword stuff. if (statusOffset + X_MOZILLA_STATUS_LEN > readCount) checkForKeyword = false; - + if (NS_SUCCEEDED(rv)) { if (checkForKeyword) @@ -967,7 +970,7 @@ nsFolderCompactState::OnDataAvailable(nsIRequest *request, nsISupports *ctxt, { NS_ASSERTION(false, "bad block offset"); // not sure what to do to handle this. - + } m_fileStream->Write(m_dataBuffer + blockOffset, readCount - blockOffset, &bytesWritten); writeCount += bytesWritten; @@ -1131,7 +1134,6 @@ done: } return rv; } - nsresult nsOfflineStoreCompactState::FinishCompact() @@ -1226,6 +1228,10 @@ nsFolderCompactState::EndCopy(nsISupports *url, nsresult aStatus) return NS_OK; } + /* Messages need to have trailing blank lines */ + uint32_t bytesWritten; + (void) m_fileStream->Write(MSG_LINEBREAK, MSG_LINEBREAK_LEN, &bytesWritten); + /** * Done with the current message; copying the existing message header * to the new database. @@ -1255,7 +1261,7 @@ nsFolderCompactState::EndCopy(nsISupports *url, nsresult aStatus) msgSize += m_addedHeaderSize; newMsgHdr->SetMessageSize(msgSize); } - m_totalMsgSize += msgSize; + m_totalMsgSize += msgSize + MSG_LINEBREAK_LEN; } // m_db->Commit(nsMsgDBCommitType::kLargeCommit); // no sense commiting until the end @@ -1345,4 +1351,3 @@ nsOfflineStoreCompactState::OnDataAvailable(nsIRequest *request, nsISupports *ct } return rv; } - diff --git a/mailnews/local/src/nsLocalMailFolder.cpp b/mailnews/local/src/nsLocalMailFolder.cpp index 14135fe46..00244ca87 100644 --- a/mailnews/local/src/nsLocalMailFolder.cpp +++ b/mailnews/local/src/nsLocalMailFolder.cpp @@ -2117,7 +2117,6 @@ nsresult nsMsgLocalMailFolder::WriteStartOfNewMessage() nsresult nsMsgLocalMailFolder::InitCopyMsgHdrAndFileStream() { - nsCOMPtr msgStore; nsresult rv = GetMsgStore(getter_AddRefs(mCopyState->m_msgStore)); NS_ENSURE_SUCCESS(rv, rv); bool reusable; @@ -2387,16 +2386,10 @@ NS_IMETHODIMP nsMsgLocalMailFolder::EndCopy(bool aCopySucceeded) nsCOMPtr seekableStream(do_QueryInterface(mCopyState->m_fileStream)); if (seekableStream) { - if (mCopyState->m_dummyEnvelopeNeeded) - { - uint32_t bytesWritten; - seekableStream->Seek(nsISeekableStream::NS_SEEK_END, 0); - mCopyState->m_fileStream->Write(MSG_LINEBREAK, MSG_LINEBREAK_LEN, &bytesWritten); - if (mCopyState->m_parseMsgState) - mCopyState->m_parseMsgState->ParseAFolderLine(CRLF, MSG_LINEBREAK_LEN); - } - rv = mCopyState->m_msgStore->FinishNewMessage(mCopyState->m_fileStream, - mCopyState->m_newHdr); + seekableStream->Seek(nsISeekableStream::NS_SEEK_END, 0); + rv = FinishNewLocalMessage(mCopyState->m_fileStream, mCopyState->m_newHdr, + mCopyState->m_msgStore, + mCopyState->m_parseMsgState); if (NS_SUCCEEDED(rv) && mCopyState->m_newHdr) mCopyState->m_newHdr->GetMessageKey(&mCopyState->m_curDstKey); if (multipleCopiesFinished) @@ -2714,13 +2707,9 @@ NS_IMETHODIMP nsMsgLocalMailFolder::EndMessage(nsMsgKey key) nsCOMPtr seekableStream = do_QueryInterface(mCopyState->m_fileStream, &rv); NS_ENSURE_SUCCESS(rv, rv); seekableStream->Seek(nsISeekableStream::NS_SEEK_END, 0); - uint32_t bytesWritten; - mCopyState->m_fileStream->Write(MSG_LINEBREAK, MSG_LINEBREAK_LEN, &bytesWritten); - if (mCopyState->m_parseMsgState) - mCopyState->m_parseMsgState->ParseAFolderLine(CRLF, MSG_LINEBREAK_LEN); - - rv = mCopyState->m_msgStore->FinishNewMessage(mCopyState->m_fileStream, - mCopyState->m_newHdr); + rv = FinishNewLocalMessage(mCopyState->m_fileStream, mCopyState->m_newHdr, + mCopyState->m_msgStore, + mCopyState->m_parseMsgState); mCopyState->m_fileStream->Close(); mCopyState->m_fileStream = nullptr; // all done with the file stream @@ -3739,7 +3728,7 @@ nsMsgLocalMailFolder::AddMessageBatch(uint32_t aMessageCount, outFileStream->Write(aMessages[i], messageLen, &bytesWritten); newMailParser->BufferInput(aMessages[i], messageLen); - msgStore->FinishNewMessage(outFileStream, newHdr); + FinishNewLocalMessage(outFileStream, newHdr, msgStore, newMailParser); outFileStream->Close(); outFileStream = nullptr; newMailParser->OnStopRequest(nullptr, nullptr, NS_OK); @@ -3752,6 +3741,19 @@ nsMsgLocalMailFolder::AddMessageBatch(uint32_t aMessageCount, return rv; } +nsresult +nsMsgLocalMailFolder::FinishNewLocalMessage(nsIOutputStream *aOutputStream, + nsIMsgDBHdr *aNewHdr, + nsIMsgPluggableStore *aMsgStore, + nsParseMailMessageState *aParseMsgState) +{ + uint32_t bytesWritten; + aOutputStream->Write(MSG_LINEBREAK, MSG_LINEBREAK_LEN, &bytesWritten); + if (aParseMsgState) + aParseMsgState->ParseAFolderLine(MSG_LINEBREAK, MSG_LINEBREAK_LEN); + return aMsgStore->FinishNewMessage(aOutputStream, aNewHdr); +} + NS_IMETHODIMP nsMsgLocalMailFolder::WarnIfLocalFileTooBig(nsIMsgWindow *aWindow, int64_t aSpaceRequested, diff --git a/mailnews/local/src/nsLocalMailFolder.h b/mailnews/local/src/nsLocalMailFolder.h index e8c395342..b2503a157 100644 --- a/mailnews/local/src/nsLocalMailFolder.h +++ b/mailnews/local/src/nsLocalMailFolder.h @@ -234,6 +234,10 @@ protected: void CopyHdrPropertiesWithSkipList(nsIMsgDBHdr *destHdr, nsIMsgDBHdr *srcHdr, const nsCString &skipList); + nsresult FinishNewLocalMessage(nsIOutputStream *outputStream, + nsIMsgDBHdr *newHdr, + nsIMsgPluggableStore *msgStore, + nsParseMailMessageState *parseMsgState); protected: nsLocalMailCopyState *mCopyState; //We only allow one of these at a time diff --git a/mailnews/local/src/nsParseMailbox.cpp b/mailnews/local/src/nsParseMailbox.cpp index 9d68e5cd1..da51c0322 100644 --- a/mailnews/local/src/nsParseMailbox.cpp +++ b/mailnews/local/src/nsParseMailbox.cpp @@ -598,6 +598,7 @@ NS_IMETHODIMP nsParseMailMessageState::Clear() m_mdn_original_recipient.length = 0; m_bccList.length = 0; m_body_lines = 0; + m_lastLineBlank = 0; m_newMsgHdr = nullptr; m_envelope_pos = 0; m_new_key = nsMsgKey_None; @@ -694,6 +695,9 @@ nsresult nsParseMailMessageState::ParseFolderLine(const char *line, uint32_t lin else if ( m_state == nsIMsgParseMailMsgState::ParseBodyState) { m_body_lines++; + // See comment in msgCore.h for why we use `IS_MSG_LINEBREAK` rather than + // just comparing `line` to `MSG_LINEBREAK`. + m_lastLineBlank = IS_MSG_LINEBREAK(line); } m_position += lineLength; @@ -824,7 +828,9 @@ NS_IMETHODIMP nsParseMailMessageState::FinishHeader() if (m_newMsgHdr) { m_newMsgHdr->SetMessageOffset(m_envelope_pos); - m_newMsgHdr->SetMessageSize(m_position - m_envelope_pos); + if (m_lastLineBlank) + m_body_lines--; + m_newMsgHdr->SetMessageSize(m_position - m_envelope_pos - m_lastLineBlank); m_newMsgHdr->SetLineCount(m_body_lines); } return NS_OK; @@ -1888,7 +1894,9 @@ NS_IMETHODIMP nsParseNewMailState::FinishHeader() { if (m_newMsgHdr) { - m_newMsgHdr->SetMessageSize(m_position - m_envelope_pos); + if (m_lastLineBlank) + m_body_lines--; + m_newMsgHdr->SetMessageSize(m_position - m_envelope_pos - m_lastLineBlank); m_newMsgHdr->SetLineCount(m_body_lines); } return NS_OK; diff --git a/mailnews/local/src/nsParseMailbox.h b/mailnews/local/src/nsParseMailbox.h index 388f1e1b2..aac40611a 100644 --- a/mailnews/local/src/nsParseMailbox.h +++ b/mailnews/local/src/nsParseMailbox.h @@ -115,6 +115,7 @@ public: PRTime m_receivedTime; uint16_t m_body_lines; + uint16_t m_lastLineBlank; bool m_IgnoreXMozillaStatus; -- cgit v1.2.3