diff options
Diffstat (limited to 'mailnews/import/oexpress/nsOEMailbox.cpp')
| -rw-r--r-- | mailnews/import/oexpress/nsOEMailbox.cpp | 673 |
1 files changed, 673 insertions, 0 deletions
diff --git a/mailnews/import/oexpress/nsOEMailbox.cpp b/mailnews/import/oexpress/nsOEMailbox.cpp new file mode 100644 index 000000000..7a1fb0be7 --- /dev/null +++ b/mailnews/import/oexpress/nsOEMailbox.cpp @@ -0,0 +1,673 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsOEMailbox.h" + +#include "OEDebugLog.h" +#include "msgCore.h" +#include "prprf.h" +#include "nsMsgLocalFolderHdrs.h" +#include "nsCRT.h" +#include "nsNetUtil.h" +#include "nsIInputStream.h" +#include "nsIOutputStream.h" +#include "nsIMsgFolder.h" +#include "nsIMsgHdr.h" +#include "nsIMsgPluggableStore.h" +#include "nsISeekableStream.h" +#include "nsMsgUtils.h" + +class CMbxScanner { +public: + CMbxScanner(nsString& name, nsIFile * mbxFile, nsIMsgFolder *dstFolder); + ~CMbxScanner(); + + virtual bool Initialize(void); + virtual bool DoWork(bool *pAbort, uint32_t *pDone, uint32_t *pCount); + + bool WasErrorFatal(void) { return m_fatalError;} + uint32_t BytesProcessed(void) { return m_didBytes;} + +protected: + bool WriteMailItem(uint32_t flags, uint32_t offset, uint32_t size, uint32_t *pTotalMsgSize = nullptr); + virtual void CleanUp(void); + +private: + void ReportWriteError(nsIMsgFolder *folder, bool fatal = true); + void ReportReadError(nsIFile * file, bool fatal = true); + bool CopyMbxFileBytes(uint32_t flags, uint32_t numBytes); + bool IsFromLineKey(uint8_t *pBuf, uint32_t max); + +public: + uint32_t m_msgCount; + +protected: + uint32_t * m_pDone; + nsString m_name; + nsCOMPtr<nsIFile> m_mbxFile; + nsCOMPtr<nsIMsgFolder> m_dstFolder; + nsCOMPtr<nsIInputStream> m_mbxFileInputStream; + nsCOMPtr<nsIOutputStream> m_dstOutputStream; + nsCOMPtr<nsIMsgPluggableStore> m_msgStore; + uint8_t * m_pInBuffer; + uint8_t * m_pOutBuffer; + uint32_t m_bufSz; + uint32_t m_didBytes; + bool m_fatalError; + int64_t m_mbxFileSize; + uint32_t m_mbxOffset; + + static const char * m_pFromLine; + +}; + + +class CIndexScanner : public CMbxScanner { +public: + CIndexScanner(nsString& name, nsIFile * idxFile, nsIFile * mbxFile, nsIMsgFolder *dstFolder); + ~CIndexScanner(); + + virtual bool Initialize(void); + virtual bool DoWork(bool *pAbort, uint32_t *pDone, uint32_t *pCount); + +protected: + virtual void CleanUp(void); + +private: + bool ValidateIdxFile(void); + bool GetMailItem(uint32_t *pFlags, uint32_t *pOffset, uint32_t *pSize); + + +private: + nsCOMPtr <nsIFile> m_idxFile; + nsCOMPtr <nsIInputStream> m_idxFileInputStream; + uint32_t m_numMessages; + uint32_t m_idxOffset; + uint32_t m_curItemIndex; +}; + + +bool CImportMailbox::ImportMailbox(uint32_t *pDone, bool *pAbort, + nsString& name, nsIFile * inFile, + nsIMsgFolder *outFolder, uint32_t *pCount) +{ + bool done = false; + nsresult rv; + nsCOMPtr <nsIFile> idxFile = do_CreateInstance(NS_LOCAL_FILE_CONTRACTID, &rv); + if (NS_SUCCEEDED(rv)) + rv = idxFile->InitWithFile(inFile); + if (NS_FAILED(rv)) { + IMPORT_LOG0("New file spec failed!\n"); + return false; + } + + if (GetIndexFile(idxFile)) { + + IMPORT_LOG1("Using index file for: %S\n", name.get()); + + CIndexScanner *pIdxScanner = new CIndexScanner(name, idxFile, inFile, outFolder); + if (pIdxScanner->Initialize()) { + if (pIdxScanner->DoWork(pAbort, pDone, pCount)) { + done = true; + } + else { + IMPORT_LOG0("CIndexScanner::DoWork() failed\n"); + } + } + else { + IMPORT_LOG0("CIndexScanner::Initialize() failed\n"); + } + + delete pIdxScanner; + } + + if (done) + return done; + + /* + something went wrong with the index file, just scan the mailbox + file itself. + */ + CMbxScanner *pMbx = new CMbxScanner(name, inFile, outFolder); + if (pMbx->Initialize()) { + if (pMbx->DoWork(pAbort, pDone, pCount)) { + done = true; + } + else { + IMPORT_LOG0("CMbxScanner::DoWork() failed\n"); + } + } + else { + IMPORT_LOG0("CMbxScanner::Initialize() failed\n"); + } + + delete pMbx; + return done; +} + + +bool CImportMailbox::GetIndexFile(nsIFile* file) +{ + nsCString pLeaf; + if (NS_FAILED(file->GetNativeLeafName(pLeaf))) + return false; + int32_t len = pLeaf.Length(); + if (len < 5) + return false; + + pLeaf.Replace(len - 3, 3, NS_LITERAL_CSTRING("idx")); + + IMPORT_LOG1("Looking for index leaf name: %s\n", pLeaf); + + nsresult rv; + rv = file->SetNativeLeafName(pLeaf); + + bool isFile = false; + bool exists = false; + if (NS_SUCCEEDED(rv)) rv = file->IsFile(&isFile); + if (NS_SUCCEEDED(rv)) rv = file->Exists(&exists); + + return (isFile && exists); +} + + +const char *CMbxScanner::m_pFromLine = "From - Mon Jan 1 00:00:00 1965\x0D\x0A"; +// let's try a 16K buffer and see how well that works? +#define kBufferKB 16 + + +CMbxScanner::CMbxScanner(nsString& name, nsIFile* mbxFile, + nsIMsgFolder* dstFolder) +{ + m_msgCount = 0; + m_name = name; + m_mbxFile = mbxFile; + m_dstFolder = dstFolder; + m_pInBuffer = nullptr; + m_pOutBuffer = nullptr; + m_bufSz = 0; + m_fatalError = false; + m_didBytes = 0; + m_mbxFileSize = 0; + m_mbxOffset = 0; +} + +CMbxScanner::~CMbxScanner() +{ + CleanUp(); +} + +void CMbxScanner::ReportWriteError(nsIMsgFolder * folder, bool fatal) +{ + m_fatalError = fatal; +} + +void CMbxScanner::ReportReadError(nsIFile * file, bool fatal) +{ + m_fatalError = fatal; +} + +bool CMbxScanner::Initialize(void) +{ + m_bufSz = (kBufferKB * 1024); + m_pInBuffer = new uint8_t[m_bufSz]; + m_pOutBuffer = new uint8_t[m_bufSz]; + if (!m_pInBuffer || !m_pOutBuffer) { + return false; + } + + m_mbxFile->GetFileSize(&m_mbxFileSize); + // open the mailbox file... + if (NS_FAILED(NS_NewLocalFileInputStream(getter_AddRefs(m_mbxFileInputStream), m_mbxFile))) { + CleanUp(); + return false; + } + + if (NS_FAILED(m_dstFolder->GetMsgStore(getter_AddRefs(m_msgStore)))) { + CleanUp(); + return false; + } + + return true; +} + + +#define kMbxHeaderSize 0x0054 +#define kMbxMessageHeaderSz 16 + +bool CMbxScanner::DoWork(bool *pAbort, uint32_t *pDone, uint32_t *pCount) +{ + m_mbxOffset = kMbxHeaderSize; + m_didBytes = kMbxHeaderSize; + + while (!(*pAbort) && ((m_mbxOffset + kMbxMessageHeaderSz) < m_mbxFileSize)) { + uint32_t msgSz; + if (!WriteMailItem(0, m_mbxOffset, 0, &msgSz)) { + if (!WasErrorFatal()) + ReportReadError(m_mbxFile); + return false; + } + m_mbxOffset += msgSz; + m_didBytes += msgSz; + m_msgCount++; + if (pDone) + *pDone = m_didBytes; + if (pCount) + *pCount = m_msgCount; + } + + CleanUp(); + + return true; +} + + +void CMbxScanner::CleanUp(void) +{ + if (m_mbxFileInputStream) + m_mbxFileInputStream->Close(); + if (m_dstOutputStream) + m_dstOutputStream->Close(); + + delete [] m_pInBuffer; + m_pInBuffer = nullptr; + + delete [] m_pOutBuffer; + m_pOutBuffer = nullptr; +} + + +#define kNumMbxLongsToRead 4 + +bool CMbxScanner::WriteMailItem(uint32_t flags, uint32_t offset, uint32_t size, + uint32_t *pTotalMsgSize) +{ + uint32_t values[kNumMbxLongsToRead]; + int32_t cnt = kNumMbxLongsToRead * sizeof(uint32_t); + nsresult rv; + uint32_t cntRead; + int8_t * pChar = (int8_t *) values; + + nsCOMPtr <nsISeekableStream> seekableStream = do_QueryInterface(m_mbxFileInputStream); + rv = seekableStream->Seek(nsISeekableStream::NS_SEEK_SET, offset); + + if (NS_FAILED(rv)) { + IMPORT_LOG1("Mbx seek error: 0x%lx\n", offset); + return false; + } + rv = m_mbxFileInputStream->Read((char *) pChar, cnt, &cntRead); + if (NS_FAILED(rv) || (cntRead != cnt)) { + IMPORT_LOG1("Mbx read error at: 0x%lx\n", offset); + return false; + } + if (values[0] != 0x7F007F00) { + IMPORT_LOG2("Mbx tag field doesn't match: 0x%lx, at offset: 0x%lx\n", values[0], offset); + return false; + } + if (size && (values[2] != size)) { + IMPORT_LOG3("Mbx size doesn't match idx, mbx: %ld, idx: %ld, at offset: 0x%lx\n", values[2], size, offset); + return false; + } + + if (pTotalMsgSize != nullptr) + *pTotalMsgSize = values[2]; + + nsCOMPtr<nsIMsgDBHdr> msgHdr; + bool reusable; + + rv = m_msgStore->GetNewMsgOutputStream(m_dstFolder, getter_AddRefs(msgHdr), &reusable, + getter_AddRefs(m_dstOutputStream)); + if (NS_FAILED(rv)) + { + IMPORT_LOG1( "Mbx getting outputstream error: 0x%lx\n", rv); + return false; + } + + // everything looks kosher... + // the actual message text follows and is values[3] bytes long... + bool copyOK = CopyMbxFileBytes(flags, values[3]); + if (copyOK) + m_msgStore->FinishNewMessage(m_dstOutputStream, msgHdr); + else { + m_msgStore->DiscardNewMessage(m_dstOutputStream, msgHdr); + IMPORT_LOG0( "Mbx CopyMbxFileBytes failed\n"); + } + if (!reusable) + { + m_dstOutputStream->Close(); + m_dstOutputStream = nullptr; + } + return copyOK; +} + +bool CMbxScanner::IsFromLineKey(uint8_t * pBuf, uint32_t max) +{ + return (max > 5 && (pBuf[0] == 'F') && (pBuf[1] == 'r') && (pBuf[2] == 'o') && (pBuf[3] == 'm') && (pBuf[4] == ' ')); +} + + +#define IS_ANY_SPACE(_ch) ((_ch == ' ') || (_ch == '\t') || (_ch == 10) || (_ch == 13)) + + +bool CMbxScanner::CopyMbxFileBytes(uint32_t flags, uint32_t numBytes) +{ + if (!numBytes) + return true; + + uint32_t cnt; + uint8_t last[2] = {0, 0}; + uint32_t inIdx = 0; + bool first = true; + uint8_t * pIn; + uint8_t * pStart; + int32_t fromLen = strlen(m_pFromLine); + nsresult rv; + uint32_t cntRead; + uint8_t * pChar; + + while (numBytes) { + if (numBytes > (m_bufSz - inIdx)) + cnt = m_bufSz - inIdx; + else + cnt = numBytes; + // Read some of the message from the file... + pChar = m_pInBuffer + inIdx; + rv = m_mbxFileInputStream->Read((char *) pChar, (int32_t)cnt, &cntRead); + if (NS_FAILED(rv) || (cntRead != (int32_t)cnt)) { + ReportReadError(m_mbxFile); + return false; + } + // Keep track of the last 2 bytes of the message for terminating EOL logic + if (cnt < 2) { + last[0] = last[1]; + last[1] = m_pInBuffer[cnt - 1]; + } + else { + last[0] = m_pInBuffer[cnt - 2]; + last[1] = m_pInBuffer[cnt - 1]; + } + + inIdx = 0; + // Handle the beginning line, don't duplicate an existing From separator + if (first) { + // check the first buffer to see if it already starts with a From line + // If it does, throw it away and use our own + if (IsFromLineKey(m_pInBuffer, cnt)) { + // skip past the first line + while ((inIdx < cnt) && (m_pInBuffer[inIdx] != nsCRT::CR)) + inIdx++; + while ((inIdx < cnt) && (IS_ANY_SPACE(m_pInBuffer[inIdx]))) + inIdx++; + if (inIdx >= cnt) { + // This should not occurr - it means the message starts + // with a From separator line that is longer than our + // file buffer! In this bizarre case, just skip this message + // since it is probably bogus anyway. + return true; + } + + } + // Begin every message with a From separator + rv = m_dstOutputStream->Write(m_pFromLine, fromLen, &cntRead); + if (NS_FAILED(rv) || (cntRead != fromLen)) { + ReportWriteError(m_dstFolder); + return false; + } + char statusLine[50]; + uint32_t msgFlags = flags; // need to convert from OE flags to mozilla flags + PR_snprintf(statusLine, sizeof(statusLine), X_MOZILLA_STATUS_FORMAT MSG_LINEBREAK, msgFlags & 0xFFFF); + rv = m_dstOutputStream->Write(statusLine, strlen(statusLine), &cntRead); + if (NS_SUCCEEDED(rv) && cntRead == fromLen) + { + PR_snprintf(statusLine, sizeof(statusLine), X_MOZILLA_STATUS2_FORMAT MSG_LINEBREAK, msgFlags & 0xFFFF0000); + rv = m_dstOutputStream->Write(statusLine, strlen(statusLine), &cntRead); + } + if (NS_FAILED(rv) || (cntRead != fromLen)) { + ReportWriteError(m_dstFolder); + return false; + } + first = false; + } + + // Handle generic data, escape any lines that begin with "From " + pIn = m_pInBuffer + inIdx; + numBytes -= cnt; + m_didBytes += cnt; + pStart = pIn; + cnt -= inIdx; + inIdx = 0; + while (cnt) { + if (*pIn == nsCRT::CR) { + // need more in buffer? + if ((cnt < 7) && numBytes) + break; + + if (cnt > 6) { + if ((pIn[1] == nsCRT::LF) && IsFromLineKey(pIn + 2, cnt)) { + inIdx += 2; + // Match, escape it + rv = m_dstOutputStream->Write((const char *)pStart, (int32_t)inIdx, &cntRead); + if (NS_SUCCEEDED(rv) && (cntRead == (int32_t)inIdx)) + rv = m_dstOutputStream->Write(">", 1, &cntRead); + if (NS_FAILED(rv) || (cntRead != 1)) { + ReportWriteError(m_dstFolder); + return false; + } + + cnt -= 2; + pIn += 2; + inIdx = 0; + pStart = pIn; + continue; + } + } + } // == nsCRT::CR + + cnt--; + inIdx++; + pIn++; + } + rv = m_dstOutputStream->Write((const char *)pStart, (int32_t)inIdx, &cntRead); + if (NS_FAILED(rv) || (cntRead != (int32_t)inIdx)) { + ReportWriteError(m_dstFolder); + return false; + } + + if (cnt) { + inIdx = cnt; + memcpy(m_pInBuffer, pIn, cnt); + } + else + inIdx = 0; + } + + // I used to check for an eol before writing one but + // it turns out that adding a proper EOL before the next + // separator never really hurts so better to be safe + // and always do it. + // if ((last[0] != nsCRT::CR) || (last[1] != nsCRT::LF)) { + rv = m_dstOutputStream->Write("\x0D\x0A", 2, &cntRead); + if (NS_FAILED(rv) || (cntRead != 2)) { + ReportWriteError(m_dstFolder); + return false; + } + // } // != nsCRT::CR || != nsCRT::LF + + return true; +} + +CIndexScanner::CIndexScanner(nsString& name, nsIFile * idxFile, + nsIFile * mbxFile, nsIMsgFolder * dstFolder) + : CMbxScanner( name, mbxFile, dstFolder) +{ + m_idxFile = idxFile; + m_curItemIndex = 0; + m_idxOffset = 0; +} + +CIndexScanner::~CIndexScanner() +{ + CleanUp(); +} + +bool CIndexScanner::Initialize(void) +{ + if (!CMbxScanner::Initialize()) + return false; + + + nsresult rv = NS_NewLocalFileInputStream(getter_AddRefs(m_idxFileInputStream), m_idxFile); + if (NS_FAILED(rv)) { + CleanUp(); + return false; + } + + return true; +} + +bool CIndexScanner::ValidateIdxFile(void) +{ + int8_t id[4]; + int32_t cnt = 4; + nsresult rv; + uint32_t cntRead; + int8_t * pReadTo; + + pReadTo = id; + rv = m_idxFileInputStream->Read((char *) pReadTo, cnt, &cntRead); + if (NS_FAILED(rv) || (cntRead != cnt)) + return false; + if ((id[0] != 'J') || (id[1] != 'M') || (id[2] != 'F') || (id[3] != '9')) + return false; + cnt = 4; + uint32_t subId; + pReadTo = (int8_t *) &subId; + rv = m_idxFileInputStream->Read((char *) pReadTo, cnt, &cntRead); + if (NS_FAILED(rv) || (cntRead != cnt)) + return false; + if (subId != 0x00010004) { + IMPORT_LOG1("Idx file subid doesn't match: 0x%lx\n", subId); + return false; + } + + pReadTo = (int8_t *) &m_numMessages; + rv = m_idxFileInputStream->Read((char *) pReadTo, cnt, &cntRead); + if (NS_FAILED(rv) || (cntRead != cnt)) + return false; + + IMPORT_LOG1("Idx file num messages: %ld\n", m_numMessages); + + m_didBytes += 80; + m_idxOffset = 80; + return true; +} + +/* +Idx file... +Header is 80 bytes, JMF9, subId? 0x00010004, numMessages, fileSize, 1, 0x00010010 +Entries start at byte 80 +4 byte numbers +Flags? maybe +?? who knows +index +start of this entry in the file +length of this record +msg offset in mbx +msg length in mbx + +*/ + +// #define DEBUG_SUBJECT_AND_FLAGS 1 +#define kNumIdxLongsToRead 7 + +bool CIndexScanner::GetMailItem(uint32_t *pFlags, uint32_t *pOffset, uint32_t *pSize) +{ + uint32_t values[kNumIdxLongsToRead]; + int32_t cnt = kNumIdxLongsToRead * sizeof(uint32_t); + int8_t * pReadTo = (int8_t *) values; + uint32_t cntRead; + nsresult rv; + + nsCOMPtr <nsISeekableStream> seekableStream = do_QueryInterface(m_idxFileInputStream); + rv = seekableStream->Seek(nsISeekableStream::NS_SEEK_SET, m_idxOffset); + if (NS_FAILED(rv)) + return false; + + rv = m_idxFileInputStream->Read((char *) pReadTo, cnt, &cntRead); + if (NS_FAILED(rv) || (cntRead != cnt)) + return false; + + if (values[3] != m_idxOffset) { + IMPORT_LOG2("Self pointer invalid: m_idxOffset=0x%lx, self=0x%lx\n", m_idxOffset, values[3]); + return false; + } + + // So... what do we have here??? +#ifdef DEBUG_SUBJECT_AND_FLAGS + IMPORT_LOG2("Number: %ld, msg offset: 0x%lx, ", values[2], values[5]); + IMPORT_LOG2("msg length: %ld, Flags: 0x%lx\n", values[6], values[0]); + seekableStream->Seek(nsISeekableStream::NS_SEEK_SET, m_idxOffset + 212); + uint32_t subSz = 0; + cnt = 4; + pReadTo = (int8_t *) &subSz; + m_idxFileInputStream->Read((char *) pReadTo, cnt, &cntRead); + if ((subSz >= 0) && (subSz < 1024)) { + char *pSub = new char[subSz + 1]; + m_idxFileInputStream->Read(pSub, subSz, &cntRead); + pSub[subSz] = 0; + IMPORT_LOG1(" Subject: %s\n", pSub); + delete [] pSub; + } +#endif + + m_idxOffset += values[4]; + m_didBytes += values[4]; + + *pFlags = values[0]; + *pOffset = values[5]; + *pSize = values[6]; + return true; +} + +#define kOEDeletedFlag 0x0001 + +bool CIndexScanner::DoWork(bool *pAbort, uint32_t *pDone, uint32_t *pCount) +{ + m_didBytes = 0; + if (!ValidateIdxFile()) + return false; + + bool failed = false; + while ((m_curItemIndex < m_numMessages) && !failed && !(*pAbort)) { + uint32_t flags, offset, size; + if (!GetMailItem(&flags, &offset, &size)) { + CleanUp(); + return false; + } + m_curItemIndex++; + if (!(flags & kOEDeletedFlag)) { + if (!WriteMailItem(flags, offset, size)) + failed = true; + else { + m_msgCount++; + } + } + m_didBytes += size; + if (pDone) + *pDone = m_didBytes; + if (pCount) + *pCount = m_msgCount; + } + + CleanUp(); + return !failed; +} + + +void CIndexScanner::CleanUp(void) +{ + CMbxScanner::CleanUp(); + m_idxFileInputStream->Close(); +} |