Issue #1258 - Part 1: Import mailnews, ldap, and mork from comm-esr52.9.1

1 files changed, 471 insertions, 0 deletions

diff --git a/mailnews/import/text/src/nsTextAddress.cpp b/mailnews/import/text/src/nsTextAddress.cpp
new file mode 100644
index 000000000..6b0b82ed1
--- /dev/null
+++ b/mailnews/import/text/src/nsTextAddress.cpp
@@ -0,0 +1,471 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "nsTextAddress.h"
+#include "nsIAddrDatabase.h"
+#include "nsNativeCharsetUtils.h"
+#include "nsIFile.h"
+#include "nsIInputStream.h"
+#include "nsILineInputStream.h"
+#include "nsNetUtil.h"
+#include "nsMsgI18N.h"
+#include "nsMsgUtils.h"
+#include "mdb.h"
+#include "nsIConverterInputStream.h"
+#include "nsIUnicharLineInputStream.h"
+#include "nsMsgUtils.h"
+
+#include "TextDebugLog.h"
+#include "plstr.h"
+#include "msgCore.h"
+#include <algorithm>
+
+#ifndef MOZILLA_INTERNAL_API
+#include "nsMsgI18N.h"
+#define NS_CopyNativeToUnicode(source, dest) \
+        nsMsgI18NConvertToUnicode(nsMsgI18NFileSystemCharset(), source, dest)
+#endif
+
+#define kWhitespace    " \t\b\r\n"
+
+nsTextAddress::nsTextAddress()
+{
+    m_database = nullptr;
+    m_fieldMap = nullptr;
+    m_LFCount = 0;
+    m_CRCount = 0;
+}
+
+nsTextAddress::~nsTextAddress()
+{
+    NS_IF_RELEASE(m_database);
+    NS_IF_RELEASE(m_fieldMap);
+}
+
+nsresult nsTextAddress::GetUnicharLineStreamForFile(nsIFile *aFile,
+                                                    nsIInputStream *aInputStream,
+                                                    nsIUnicharLineInputStream **aStream)
+{
+  nsAutoCString charset;
+  nsresult rv = MsgDetectCharsetFromFile(aFile, charset);
+  if (NS_FAILED(rv)) {
+    charset.Assign(nsMsgI18NFileSystemCharset());
+  }
+
+  nsCOMPtr<nsIConverterInputStream> converterStream =
+    do_CreateInstance("@mozilla.org/intl/converter-input-stream;1", &rv);
+  if (NS_SUCCEEDED(rv)) {
+    rv = converterStream->Init(aInputStream,
+                               charset.get(),
+                               8192,
+                               nsIConverterInputStream::DEFAULT_REPLACEMENT_CHARACTER);
+  }
+
+  return CallQueryInterface(converterStream, aStream);
+}
+
+nsresult nsTextAddress::ImportAddresses(bool *pAbort, const char16_t *pName, nsIFile *pSrc, nsIAddrDatabase *pDb, nsIImportFieldMap *fieldMap, nsString& errors, uint32_t *pProgress)
+{
+  // Open the source file for reading, read each line and process it!
+  NS_IF_RELEASE(m_database);
+  NS_IF_RELEASE(m_fieldMap);
+  m_database = pDb;
+  m_fieldMap = fieldMap;
+  NS_ADDREF(m_fieldMap);
+  NS_ADDREF(m_database);
+
+  nsCOMPtr<nsIInputStream> inputStream;
+  nsresult rv = NS_NewLocalFileInputStream(getter_AddRefs(inputStream), pSrc);
+  if (NS_FAILED(rv)) {
+    IMPORT_LOG0("*** Error opening address file for reading\n");
+    return rv;
+  }
+
+  // Here we use this to work out the size of the file, so we can update
+  // an integer as we go through the file which will update a progress
+  // bar if required by the caller.
+  uint64_t bytesLeft = 0;
+  rv = inputStream->Available(&bytesLeft);
+  if (NS_FAILED(rv)) {
+    IMPORT_LOG0("*** Error checking address file for size\n");
+    inputStream->Close();
+    return rv;
+  }
+
+  uint64_t totalBytes = bytesLeft;
+  bool skipRecord = false;
+
+  rv = m_fieldMap->GetSkipFirstRecord(&skipRecord);
+  if (NS_FAILED(rv)) {
+    IMPORT_LOG0("*** Error checking to see if we should skip the first record\n");
+    return rv;
+  }
+
+  nsCOMPtr<nsIUnicharLineInputStream> lineStream;
+  rv = GetUnicharLineStreamForFile(pSrc, inputStream, getter_AddRefs(lineStream));
+  if (NS_FAILED(rv)) {
+    IMPORT_LOG0("*** Error opening converter stream for importer\n");
+    return rv;
+  }
+
+  bool more = true;
+  nsAutoString line;
+
+  // Skip the first record if the user has requested it.
+  if (skipRecord)
+    rv = ReadRecord(lineStream, line, &more);
+
+  while (!(*pAbort) && more && NS_SUCCEEDED(rv)) {
+    // Read the line in
+    rv = ReadRecord(lineStream, line, &more);
+    if (NS_SUCCEEDED(rv)) {
+      // Now proces it to add it to the database
+      rv = ProcessLine(line, errors);
+
+      if (NS_FAILED(rv)) {
+        IMPORT_LOG0("*** Error processing text record.\n");
+      }
+    }
+    if (NS_SUCCEEDED(rv) && pProgress) {
+      // This won't be totally accurate, but its the best we can do
+      // considering that lineStream won't give us how many bytes
+      // are actually left.
+      bytesLeft -= line.Length();
+      *pProgress = std::min(totalBytes - bytesLeft, uint64_t(PR_UINT32_MAX));
+    }
+  }
+
+  inputStream->Close();
+
+  if (NS_FAILED(rv)) {
+    IMPORT_LOG0("*** Error reading the address book - probably incorrect ending\n");
+    return NS_ERROR_FAILURE;
+  }
+
+  return pDb->Commit(nsAddrDBCommitType::kLargeCommit);
+}
+
+nsresult nsTextAddress::ReadRecord(nsIUnicharLineInputStream *aLineStream,
+                                   nsAString &aLine,
+                                   bool *aMore)
+{
+  bool more = true;
+  uint32_t numQuotes = 0;
+  nsresult rv;
+  nsAutoString line;
+
+  // ensure aLine is empty
+  aLine.Truncate();
+
+  do {
+    if (!more) {
+      // No more, so we must have an incorrect file.
+      rv = NS_ERROR_FAILURE;
+    }
+    else {
+      // Read the line and append it
+      rv = aLineStream->ReadLine(line, &more);
+      if (NS_SUCCEEDED(rv)) {
+        if (!aLine.IsEmpty())
+          aLine.AppendLiteral(MSG_LINEBREAK);
+        aLine.Append(line);
+
+        numQuotes += MsgCountChar(line, char16_t('"'));
+      }
+    }
+    // Continue whilst everything is ok, and we have an odd number of quotes.
+  } while (NS_SUCCEEDED(rv) && (numQuotes % 2 != 0));
+
+  *aMore = more;
+  return rv;
+}
+
+nsresult nsTextAddress::ReadRecordNumber(nsIFile *aSrc, nsAString &aLine, int32_t rNum)
+{
+  nsCOMPtr<nsIInputStream> inputStream;
+  nsresult rv = NS_NewLocalFileInputStream(getter_AddRefs(inputStream), aSrc);
+  if (NS_FAILED(rv)) {
+    IMPORT_LOG0("*** Error opening address file for reading\n");
+    return rv;
+  }
+
+  int32_t rIndex = 0;
+  uint64_t bytesLeft = 0;
+
+  rv = inputStream->Available(&bytesLeft);
+  if (NS_FAILED(rv)) {
+    IMPORT_LOG0("*** Error checking address file for eof\n");
+    inputStream->Close();
+    return rv;
+  }
+
+  nsCOMPtr<nsIUnicharLineInputStream> lineStream;
+  rv = GetUnicharLineStreamForFile(aSrc, inputStream, getter_AddRefs(lineStream));
+  if (NS_FAILED(rv)) {
+    IMPORT_LOG0("*** Error opening converter stream for importer\n");
+    return rv;
+  }
+
+  bool more = true;
+
+  while (more && (rIndex <= rNum)) {
+    rv = ReadRecord(lineStream, aLine, &more);
+    if (NS_FAILED(rv)) {
+      inputStream->Close();
+      return rv;
+    }
+    if (rIndex == rNum) {
+      inputStream->Close();
+      return NS_OK;
+    }
+
+    rIndex++;
+  }
+
+  return NS_ERROR_FAILURE;
+}
+
+int32_t nsTextAddress::CountFields(const nsAString &aLine, char16_t delim)
+{
+    int32_t pos = 0;
+    int32_t maxLen = aLine.Length();
+    int32_t count = 0;
+    char16_t tab = char16_t('\t');
+    char16_t doubleQuote = char16_t('"');
+
+    if (delim == tab)
+        tab = char16_t('\0');
+
+    while (pos < maxLen) {
+        while (((aLine[pos] == char16_t(' ')) || (aLine[pos] == tab)) &&
+               (pos < maxLen)) {
+            pos++;
+        }
+        if ((pos < maxLen) && (aLine[pos] == doubleQuote)) {
+            pos++;
+            while ((pos < maxLen) && (aLine[pos] != doubleQuote)) {
+                pos++;
+                if (((pos + 1) < maxLen) &&
+                    (aLine[pos] == doubleQuote) &&
+                    (aLine[pos + 1] == doubleQuote)) {
+                    pos += 2;
+                }
+            }
+            if (pos < maxLen)
+                pos++;
+        }
+        while ((pos < maxLen) && (aLine[pos] != delim))
+            pos++;
+
+        count++;
+        pos++;
+    }
+
+    return count;
+}
+
+bool nsTextAddress::GetField(const nsAString &aLine,
+                             int32_t index,
+                             nsString &field,
+                             char16_t delim)
+{
+    bool result = false;
+    int32_t pos = 0;
+    int32_t maxLen = aLine.Length();
+    char16_t tab = char16_t('\t');
+    char16_t doubleQuote = char16_t('"');
+
+    field.Truncate();
+
+    if (delim == tab)
+        tab = 0;
+
+    while (index && (pos < maxLen)) {
+        while (((aLine[pos] == char16_t(' ')) || (aLine[pos] == tab)) &&
+               (pos < maxLen)) {
+            pos++;
+        }
+        if (pos >= maxLen)
+            break;
+        if (aLine[pos] == doubleQuote) {
+            do {
+                pos++;
+                if (((pos + 1) < maxLen) &&
+                    (aLine[pos] == doubleQuote) &&
+                    (aLine[pos + 1] == doubleQuote)) {
+                    pos += 2;
+                }
+            } while ((pos < maxLen) && (aLine[pos] != doubleQuote));
+            if (pos < maxLen)
+                pos++;
+        }
+        if (pos >= maxLen)
+            break;
+
+        while ((pos < maxLen) && (aLine[pos] != delim))
+            pos++;
+
+        if (pos >= maxLen)
+            break;
+
+        index--;
+        pos++;
+    }
+
+    if (pos >= maxLen)
+        return result;
+
+    result = true;
+
+    while ((pos < maxLen) && ((aLine[pos] == ' ') || (aLine[pos] == tab)))
+        pos++;
+
+    int32_t fLen = 0;
+    int32_t startPos = pos;
+    bool    quoted = false;
+    if (aLine[pos] == '"') {
+        startPos++;
+        fLen = -1;
+        do {
+            pos++;
+            fLen++;
+            if (((pos + 1) < maxLen) &&
+                (aLine[pos] == doubleQuote) &&
+                (aLine[pos + 1] == doubleQuote)) {
+                quoted = true;
+                pos += 2;
+                fLen += 2;
+            }
+        } while ((pos < maxLen) && (aLine[pos] != doubleQuote));
+    }
+    else {
+        while ((pos < maxLen) && (aLine[pos] != delim)) {
+            pos++;
+            fLen++;
+        }
+    }
+
+    if (!fLen) {
+        return result;
+    }
+
+    field.Append(nsDependentSubstring(aLine, startPos, fLen));
+    field.Trim(kWhitespace);
+
+    if (quoted) {
+      int32_t offset = field.Find("\"\"");
+      while (offset != -1) {
+        field.Cut(offset, 1);
+        offset = MsgFind(field, "\"\"", false, offset + 1);
+      }
+    }
+
+    return result;
+}
+
+nsresult nsTextAddress::DetermineDelim(nsIFile *aSrc)
+{
+  nsCOMPtr<nsIInputStream> inputStream;
+  nsresult rv = NS_NewLocalFileInputStream(getter_AddRefs(inputStream), aSrc);
+  if (NS_FAILED(rv)) {
+    IMPORT_LOG0("*** Error opening address file for reading\n");
+    return rv;
+  }
+
+  int32_t lineCount = 0;
+  int32_t tabCount = 0;
+  int32_t commaCount = 0;
+  int32_t tabLines = 0;
+  int32_t commaLines = 0;
+  nsAutoString line;
+  bool more = true;
+
+  nsCOMPtr<nsIUnicharLineInputStream> lineStream;
+  rv = GetUnicharLineStreamForFile(aSrc, inputStream, getter_AddRefs(lineStream));
+  if (NS_FAILED(rv)) {
+    IMPORT_LOG0("*** Error opening converter stream for importer\n");
+    return rv;
+  }
+
+  while (more && NS_SUCCEEDED(rv) && (lineCount < 100)) {
+    rv = lineStream->ReadLine(line, &more);
+    if (NS_SUCCEEDED(rv)) {
+      tabCount = CountFields(line, char16_t('\t'));
+      commaCount = CountFields(line, char16_t(','));
+      if (tabCount > commaCount)
+        tabLines++;
+      else if (commaCount)
+        commaLines++;
+    }
+    lineCount++;
+  }
+
+  rv = inputStream->Close();
+
+  if (tabLines > commaLines)
+    m_delim = char16_t('\t');
+  else
+    m_delim = char16_t(',');
+
+  IMPORT_LOG2( "Tab count = %d, Comma count = %d\n", tabLines, commaLines);
+
+  return rv;
+}
+
+/*
+    This is where the real work happens!
+    Go through the field map and set the data in a new database row
+*/
+nsresult nsTextAddress::ProcessLine(const nsAString &aLine, nsString& errors)
+{
+    if (!m_fieldMap) {
+        IMPORT_LOG0("*** Error, text import needs a field map\n");
+        return NS_ERROR_FAILURE;
+    }
+
+    nsresult rv;
+
+    // Wait until we get our first non-empty field, then create a new row,
+    // fill in the data, then add the row to the database.
+    nsCOMPtr<nsIMdbRow> newRow;
+    nsAutoString   fieldVal;
+    int32_t        fieldNum;
+    int32_t        numFields = 0;
+    bool           active;
+    rv = m_fieldMap->GetMapSize(&numFields);
+    for (int32_t i = 0; (i < numFields) && NS_SUCCEEDED(rv); i++) {
+        active = false;
+        rv = m_fieldMap->GetFieldMap(i, &fieldNum);
+        if (NS_SUCCEEDED(rv))
+            rv = m_fieldMap->GetFieldActive(i, &active);
+        if (NS_SUCCEEDED(rv) && active) {
+            if (GetField(aLine, i, fieldVal, m_delim)) {
+                if (!fieldVal.IsEmpty()) {
+                    if (!newRow) {
+                        rv = m_database->GetNewRow(getter_AddRefs(newRow));
+                        if (NS_FAILED(rv)) {
+                            IMPORT_LOG0("*** Error getting new address database row\n");
+                        }
+                    }
+                    if (newRow) {
+                        rv = m_fieldMap->SetFieldValue(m_database, newRow, fieldNum, fieldVal.get());
+                    }
+                }
+            }
+            else
+                break;
+        }
+        else if (active) {
+          IMPORT_LOG1("*** Error getting field map for index %ld\n", (long) i);
+        }
+    }
+
+    if (NS_SUCCEEDED(rv) && newRow)
+      rv = m_database->AddCardRowToDB(newRow);
+
+    return rv;
+}
+