diff options
Diffstat (limited to 'mailnews/extensions')
109 files changed, 33641 insertions, 0 deletions
diff --git a/mailnews/extensions/bayesian-spam-filter/moz.build b/mailnews/extensions/bayesian-spam-filter/moz.build new file mode 100644 index 000000000..8755860cb --- /dev/null +++ b/mailnews/extensions/bayesian-spam-filter/moz.build @@ -0,0 +1,6 @@ +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +DIRS += ['src'] diff --git a/mailnews/extensions/bayesian-spam-filter/src/moz.build b/mailnews/extensions/bayesian-spam-filter/src/moz.build new file mode 100644 index 000000000..5819766a3 --- /dev/null +++ b/mailnews/extensions/bayesian-spam-filter/src/moz.build @@ -0,0 +1,11 @@ +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +SOURCES += [ + 'nsBayesianFilter.cpp', +] + +FINAL_LIBRARY = 'mail' + diff --git a/mailnews/extensions/bayesian-spam-filter/src/nsBayesianFilter.cpp b/mailnews/extensions/bayesian-spam-filter/src/nsBayesianFilter.cpp new file mode 100644 index 000000000..0fa5aa1e2 --- /dev/null +++ b/mailnews/extensions/bayesian-spam-filter/src/nsBayesianFilter.cpp @@ -0,0 +1,2758 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsBayesianFilter.h" +#include "nsIInputStream.h" +#include "nsIStreamListener.h" +#include "nsNetUtil.h" +#include "nsQuickSort.h" +#include "nsIMsgMessageService.h" +#include "nsMsgUtils.h" // for GetMessageServiceFromURI +#include "prnetdb.h" +#include "nsIMsgWindow.h" +#include "mozilla/Logging.h" +#include "nsAppDirectoryServiceDefs.h" +#include "nsUnicharUtils.h" +#include "nsDirectoryServiceUtils.h" +#include "nsIMIMEHeaderParam.h" +#include "nsNetCID.h" +#include "nsIMimeHeaders.h" +#include "nsMsgMimeCID.h" +#include "nsIMsgMailNewsUrl.h" +#include "nsIMimeMiscStatus.h" +#include "nsIPrefService.h" +#include "nsIPrefBranch.h" +#include "nsIStringEnumerator.h" +#include "nsIObserverService.h" +#include "nsIChannel.h" + +using namespace mozilla; + +// needed to mark attachment flag on the db hdr +#include "nsIMsgHdr.h" + +// needed to strip html out of the body +#include "nsIContentSerializer.h" +#include "nsLayoutCID.h" +#include "nsIParserUtils.h" +#include "nsIDocumentEncoder.h" + +#include "nsIncompleteGamma.h" +#include <math.h> +#include <prmem.h> +#include "nsIMsgTraitService.h" +#include "mozilla/Services.h" +#include "mozilla/Attributes.h" +#include <cstdlib> // for std::abs(int/long) +#include <cmath> // for std::abs(float/double) + +static PRLogModuleInfo *BayesianFilterLogModule = nullptr; + +#define kDefaultJunkThreshold .99 // we override this value via a pref +static const char* kBayesianFilterTokenDelimiters = " \t\n\r\f."; +static unsigned int kMinLengthForToken = 3; // lower bound on the number of characters in a word before we treat it as a token +static unsigned int kMaxLengthForToken = 12; // upper bound on the number of characters in a word to be declared as a token + +#define FORGED_RECEIVED_HEADER_HINT NS_LITERAL_CSTRING("may be forged") + +#ifndef M_LN2 +#define M_LN2 0.69314718055994530942 +#endif + +#ifndef M_E +#define M_E 2.7182818284590452354 +#endif + +// provide base implementation of hash lookup of a string +struct BaseToken : public PLDHashEntryHdr +{ + const char* mWord; +}; + +// token for a particular message +// mCount, mAnalysisLink are initialized to zero by the hash code +struct Token : public BaseToken { + uint32_t mCount; + uint32_t mAnalysisLink; // index in mAnalysisStore of the AnalysisPerToken + // object for the first trait for this token +}; + +// token stored in a training file for a group of messages +// mTraitLink is initialized to 0 by the hash code +struct CorpusToken : public BaseToken +{ + uint32_t mTraitLink; // index in mTraitStore of the TraitPerToken + // object for the first trait for this token +}; + +// set the value of a TraitPerToken object +TraitPerToken::TraitPerToken(uint32_t aTraitId, uint32_t aCount) + : mId(aTraitId), mCount(aCount), mNextLink(0) +{ +} + +// shorthand representations of trait ids for junk and good +static const uint32_t kJunkTrait = nsIJunkMailPlugin::JUNK_TRAIT; +static const uint32_t kGoodTrait = nsIJunkMailPlugin::GOOD_TRAIT; + +// set the value of an AnalysisPerToken object +AnalysisPerToken::AnalysisPerToken( + uint32_t aTraitIndex, double aDistance, double aProbability) : + mTraitIndex(aTraitIndex), + mDistance(aDistance), + mProbability(aProbability), + mNextLink(0) +{ +} + +// the initial size of the AnalysisPerToken linked list storage +const uint32_t kAnalysisStoreCapacity = 2048; + +// the initial size of the TraitPerToken linked list storage +const uint32_t kTraitStoreCapacity = 16384; + +// Size of Auto arrays representing per trait information +const uint32_t kTraitAutoCapacity = 10; + +TokenEnumeration::TokenEnumeration(PLDHashTable* table) + : mIterator(table->Iter()) +{ +} + +inline bool TokenEnumeration::hasMoreTokens() +{ + return !mIterator.Done(); +} + +inline BaseToken* TokenEnumeration::nextToken() +{ + auto token = static_cast<BaseToken*>(mIterator.Get()); + mIterator.Next(); + return token; +} + +// member variables +static const PLDHashTableOps gTokenTableOps = { + PLDHashTable::HashStringKey, + PLDHashTable::MatchStringKey, + PLDHashTable::MoveEntryStub, + PLDHashTable::ClearEntryStub, + nullptr +}; + +TokenHash::TokenHash(uint32_t aEntrySize) + : mTokenTable(&gTokenTableOps, aEntrySize, 128) +{ + mEntrySize = aEntrySize; + PL_INIT_ARENA_POOL(&mWordPool, "Words Arena", 16384); +} + +TokenHash::~TokenHash() +{ + PL_FinishArenaPool(&mWordPool); +} + +nsresult TokenHash::clearTokens() +{ + // we re-use the tokenizer when classifying multiple messages, + // so this gets called after every message classification. + mTokenTable.ClearAndPrepareForLength(128); + PL_FreeArenaPool(&mWordPool); + return NS_OK; +} + +char* TokenHash::copyWord(const char* word, uint32_t len) +{ + void* result; + uint32_t size = 1 + len; + PL_ARENA_ALLOCATE(result, &mWordPool, size); + if (result) + memcpy(result, word, size); + return reinterpret_cast<char*>(result); +} + +inline BaseToken* TokenHash::get(const char* word) +{ + PLDHashEntryHdr* entry = mTokenTable.Search(word); + if (entry) + return static_cast<BaseToken*>(entry); + return NULL; +} + +BaseToken* TokenHash::add(const char* word) +{ + if (!word || !*word) + { + NS_ERROR("Trying to add a null word"); + return nullptr; + } + + MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, ("add word: %s", word)); + + PLDHashEntryHdr* entry = mTokenTable.Add(word, mozilla::fallible); + BaseToken* token = static_cast<BaseToken*>(entry); + if (token) { + if (token->mWord == NULL) { + uint32_t len = strlen(word); + NS_ASSERTION(len != 0, "adding zero length word to tokenizer"); + if (!len) + MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, ("adding zero length word to tokenizer")); + token->mWord = copyWord(word, len); + NS_ASSERTION(token->mWord, "copyWord failed"); + if (!token->mWord) { + MOZ_LOG(BayesianFilterLogModule, LogLevel::Error, ("copyWord failed: %s (%d)", word, len)); + mTokenTable.RawRemove(entry); + return NULL; + } + } + } + return token; +} + +inline uint32_t TokenHash::countTokens() +{ + return mTokenTable.EntryCount(); +} + +inline TokenEnumeration TokenHash::getTokens() +{ + return TokenEnumeration(&mTokenTable); +} + +Tokenizer::Tokenizer() : + TokenHash(sizeof(Token)), + mBodyDelimiters(kBayesianFilterTokenDelimiters), + mHeaderDelimiters(kBayesianFilterTokenDelimiters), + mCustomHeaderTokenization(false), + mMaxLengthForToken(kMaxLengthForToken), + mIframeToDiv(false) +{ + nsresult rv; + nsCOMPtr<nsIPrefService> prefs = do_GetService(NS_PREFSERVICE_CONTRACTID, &rv); + NS_ENSURE_SUCCESS_VOID(rv); + + nsCOMPtr<nsIPrefBranch> prefBranch; + rv = prefs->GetBranch("mailnews.bayesian_spam_filter.", getter_AddRefs(prefBranch)); + NS_ENSURE_SUCCESS_VOID(rv); // no branch defined, just use defaults + + /* + * RSS feeds store their summary as alternate content of an iframe. But due + * to bug 365953, this is not seen by the serializer. As a workaround, allow + * the tokenizer to replace the iframe with div for tokenization. + */ + rv = prefBranch->GetBoolPref("iframe_to_div", &mIframeToDiv); + if (NS_FAILED(rv)) + mIframeToDiv = false; + + /* + * the list of delimiters used to tokenize the message and body + * defaults to the value in kBayesianFilterTokenDelimiters, but may be + * set with the following preferences for the body and header + * separately. + * + * \t, \n, \v, \f, \r, and \\ will be escaped to their normal + * C-library values, all other two-letter combinations beginning with \ + * will be ignored. + */ + + prefBranch->GetCharPref("body_delimiters", getter_Copies(mBodyDelimiters)); + if (!mBodyDelimiters.IsEmpty()) + UnescapeCString(mBodyDelimiters); + else // prefBranch empties the result when it fails :( + mBodyDelimiters.Assign(kBayesianFilterTokenDelimiters); + + prefBranch->GetCharPref("header_delimiters", getter_Copies(mHeaderDelimiters)); + if (!mHeaderDelimiters.IsEmpty()) + UnescapeCString(mHeaderDelimiters); + else + mHeaderDelimiters.Assign(kBayesianFilterTokenDelimiters); + + /* + * Extensions may wish to enable or disable tokenization of certain headers. + * Define any headers to enable/disable in a string preference like this: + * "mailnews.bayesian_spam_filter.tokenizeheader.headername" + * + * where "headername" is the header to tokenize. For example, to tokenize the + * header "x-spam-status" use the preference: + * + * "mailnews.bayesian_spam_filter.tokenizeheader.x-spam-status" + * + * The value of the string preference will be interpreted in one of + * four ways, depending on the value: + * + * If "false" then do not tokenize that header + * If "full" then add the entire header value as a token, + * without breaking up into subtokens using delimiters + * If "standard" then tokenize the header using as delimiters the current + * value of the generic header delimiters + * Any other string is interpreted as a list of delimiters to use to parse + * the header. \t, \n, \v, \f, \r, and \\ will be escaped to their normal + * C-library values, all other two-letter combinations beginning with \ + * will be ignored. + * + * Header names in the preference should be all lower case + * + * Extensions may also set the maximum length of a token (default is + * kMaxLengthForToken) by setting the int preference: + * "mailnews.bayesian_spam_filter.maxlengthfortoken" + */ + + char** headers; + uint32_t count; + + // get customized maximum token length + int32_t maxLengthForToken; + rv = prefBranch->GetIntPref("maxlengthfortoken", &maxLengthForToken); + mMaxLengthForToken = NS_SUCCEEDED(rv) ? uint32_t(maxLengthForToken) : kMaxLengthForToken; + + rv = prefs->GetBranch("mailnews.bayesian_spam_filter.tokenizeheader.", getter_AddRefs(prefBranch)); + if (NS_SUCCEEDED(rv)) + rv = prefBranch->GetChildList("", &count, &headers); + + if (NS_SUCCEEDED(rv)) + { + mCustomHeaderTokenization = true; + for (uint32_t i = 0; i < count; i++) + { + nsCString value; + prefBranch->GetCharPref(headers[i], getter_Copies(value)); + if (value.EqualsLiteral("false")) + { + mDisabledHeaders.AppendElement(headers[i]); + continue; + } + mEnabledHeaders.AppendElement(headers[i]); + if (value.EqualsLiteral("standard")) + value.SetIsVoid(true); // Void means use default delimiter + else if (value.EqualsLiteral("full")) + value.Truncate(); // Empty means add full header + else + UnescapeCString(value); + mEnabledHeadersDelimiters.AppendElement(value); + } + NS_FREE_XPCOM_ALLOCATED_POINTER_ARRAY(count, headers); + } +} + +Tokenizer::~Tokenizer() +{ +} + +inline Token* Tokenizer::get(const char* word) +{ + return static_cast<Token*>(TokenHash::get(word)); +} + +Token* Tokenizer::add(const char* word, uint32_t count) +{ + MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, ("add word: %s (count=%d)", + word, count)); + + Token* token = static_cast<Token*>(TokenHash::add(word)); + if (token) + { + token->mCount += count; // hash code initializes this to zero + MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, + ("adding word to tokenizer: %s (count=%d) (mCount=%d)", + word, count, token->mCount)); + } + return token; +} + +static bool isDecimalNumber(const char* word) +{ + const char* p = word; + if (*p == '-') ++p; + char c; + while ((c = *p++)) { + if (!isdigit((unsigned char) c)) + return false; + } + return true; +} + +static bool isASCII(const char* word) +{ + const unsigned char* p = (const unsigned char*)word; + unsigned char c; + while ((c = *p++)) { + if (c > 127) + return false; + } + return true; +} + +inline bool isUpperCase(char c) { return ('A' <= c) && (c <= 'Z'); } + +static char* toLowerCase(char* str) +{ + char c, *p = str; + while ((c = *p++)) { + if (isUpperCase(c)) + p[-1] = c + ('a' - 'A'); + } + return str; +} + +void Tokenizer::addTokenForHeader(const char * aTokenPrefix, nsACString& aValue, + bool aTokenizeValue, const char* aDelimiters) +{ + if (aValue.Length()) + { + ToLowerCase(aValue); + if (!aTokenizeValue) + { + nsCString tmpStr; + tmpStr.Assign(aTokenPrefix); + tmpStr.Append(':'); + tmpStr.Append(aValue); + + add(tmpStr.get()); + } + else + { + char* word; + nsCString str(aValue); + char *next = str.BeginWriting(); + const char* delimiters = !aDelimiters ? + mHeaderDelimiters.get() : aDelimiters; + while ((word = NS_strtok(delimiters, &next)) != NULL) + { + if (strlen(word) < kMinLengthForToken) + continue; + if (isDecimalNumber(word)) + continue; + if (isASCII(word)) + { + nsCString tmpStr; + tmpStr.Assign(aTokenPrefix); + tmpStr.Append(':'); + tmpStr.Append(word); + add(tmpStr.get()); + } + } + } + } +} + +void Tokenizer::tokenizeAttachment(const char * aContentType, const char * aFileName) +{ + nsAutoCString contentType; + nsAutoCString fileName; + fileName.Assign(aFileName); + contentType.Assign(aContentType); + + // normalize the content type and the file name + ToLowerCase(fileName); + ToLowerCase(contentType); + addTokenForHeader("attachment/filename", fileName); + + addTokenForHeader("attachment/content-type", contentType); +} + +void Tokenizer::tokenizeHeaders(nsIUTF8StringEnumerator * aHeaderNames, nsIUTF8StringEnumerator * aHeaderValues) +{ + nsCString headerValue; + nsAutoCString headerName; // we'll be normalizing all header names to lower case + bool hasMore; + + while (aHeaderNames->HasMore(&hasMore), hasMore) + { + aHeaderNames->GetNext(headerName); + ToLowerCase(headerName); + aHeaderValues->GetNext(headerValue); + + bool headerProcessed = false; + if (mCustomHeaderTokenization) + { + // Process any exceptions set from preferences + for (uint32_t i = 0; i < mEnabledHeaders.Length(); i++) + if (headerName.Equals(mEnabledHeaders[i])) + { + if (mEnabledHeadersDelimiters[i].IsVoid()) + // tokenize with standard delimiters for all headers + addTokenForHeader(headerName.get(), headerValue, true); + else if (mEnabledHeadersDelimiters[i].IsEmpty()) + // do not break the header into tokens + addTokenForHeader(headerName.get(), headerValue); + else + // use the delimiter in mEnabledHeadersDelimiters + addTokenForHeader(headerName.get(), headerValue, true, + mEnabledHeadersDelimiters[i].get()); + headerProcessed = true; + break; // we found the header, no need to look for more custom values + } + + for (uint32_t i = 0; i < mDisabledHeaders.Length(); i++) + { + if (headerName.Equals(mDisabledHeaders[i])) + { + headerProcessed = true; + break; + } + } + + if (headerProcessed) + continue; + } + + switch (headerName.First()) + { + case 'c': + if (headerName.Equals("content-type")) + { + nsresult rv; + nsCOMPtr<nsIMIMEHeaderParam> mimehdrpar = do_GetService(NS_MIMEHEADERPARAM_CONTRACTID, &rv); + if (NS_FAILED(rv)) + break; + + // extract the charset parameter + nsCString parameterValue; + mimehdrpar->GetParameterInternal(headerValue.get(), "charset", nullptr, nullptr, getter_Copies(parameterValue)); + addTokenForHeader("charset", parameterValue); + + // create a token containing just the content type + mimehdrpar->GetParameterInternal(headerValue.get(), "type", nullptr, nullptr, getter_Copies(parameterValue)); + if (!parameterValue.Length()) + mimehdrpar->GetParameterInternal(headerValue.get(), nullptr /* use first unnamed param */, nullptr, nullptr, getter_Copies(parameterValue)); + addTokenForHeader("content-type/type", parameterValue); + + // XXX: should we add a token for the entire content-type header as well or just these parts we have extracted? + } + break; + case 'r': + if (headerName.Equals("received")) + { + // look for the string "may be forged" in the received headers. sendmail sometimes adds this hint + // This does not compile on linux yet. Need to figure out why. Commenting out for now + // if (FindInReadable(FORGED_RECEIVED_HEADER_HINT, headerValue)) + // addTokenForHeader(headerName.get(), FORGED_RECEIVED_HEADER_HINT); + } + + // leave out reply-to + break; + case 's': + if (headerName.Equals("subject")) + { + // we want to tokenize the subject + addTokenForHeader(headerName.get(), headerValue, true); + } + + // important: leave out sender field. Too strong of an indicator + break; + case 'x': // (2) X-Mailer / user-agent works best if it is untokenized, just fold the case and any leading/trailing white space + // all headers beginning with x-mozilla are being changed by us, so ignore + if (Substring(headerName, 0, 9).Equals("x-mozilla")) + break; + // fall through + MOZ_FALLTHROUGH; + case 'u': + addTokenForHeader(headerName.get(), headerValue); + break; + default: + addTokenForHeader(headerName.get(), headerValue); + break; + } // end switch + + } +} + +void Tokenizer::tokenize_ascii_word(char * aWord) +{ + // always deal with normalized lower case strings + toLowerCase(aWord); + uint32_t wordLength = strlen(aWord); + + // if the wordLength is within our accepted token limit, then add it + if (wordLength >= kMinLengthForToken && wordLength <= mMaxLengthForToken) + add(aWord); + else if (wordLength > mMaxLengthForToken) + { + // don't skip over the word if it looks like an email address, + // there is value in adding tokens for addresses + nsDependentCString word (aWord, wordLength); // CHEAP, no allocation occurs here... + + // XXX: i think the 40 byte check is just for perf reasons...if the email address is longer than that then forget about it. + const char *atSign = strchr(aWord, '@'); + if (wordLength < 40 && strchr(aWord, '.') && atSign && !strchr(atSign + 1, '@')) + { + uint32_t numBytesToSep = atSign - aWord; + if (numBytesToSep < wordLength - 1) // if the @ sign is the last character, it must not be an email address + { + // split the john@foo.com into john and foo.com, treat them as separate tokens + nsCString emailNameToken; + emailNameToken.AssignLiteral("email name:"); + emailNameToken.Append(Substring(word, 0, numBytesToSep++)); + add(emailNameToken.get()); + nsCString emailAddrToken; + emailAddrToken.AssignLiteral("email addr:"); + emailAddrToken.Append(Substring(word, numBytesToSep, wordLength - numBytesToSep)); + add(emailAddrToken.get()); + return; + } + } + + // there is value in generating a token indicating the number + // of characters we are skipping. We'll round to the nearest 10 + nsCString skipToken; + skipToken.AssignLiteral("skip:"); + skipToken.Append(word[0]); + skipToken.Append(' '); + skipToken.AppendInt((wordLength/10) * 10); + add(skipToken.get()); + } +} + +// one substract and one conditional jump should be faster than two conditional jump on most recent system. +#define IN_RANGE(x, low, high) ((uint16_t)((x)-(low)) <= (high)-(low)) + +#define IS_JA_HIRAGANA(x) IN_RANGE(x, 0x3040, 0x309F) +// swapping the range using xor operation to reduce conditional jump. +#define IS_JA_KATAKANA(x) (IN_RANGE(x^0x0004, 0x30A0, 0x30FE)||(IN_RANGE(x, 0xFF66, 0xFF9F))) +#define IS_JA_KANJI(x) (IN_RANGE(x, 0x2E80, 0x2FDF)||IN_RANGE(x, 0x4E00, 0x9FAF)) +#define IS_JA_KUTEN(x) (((x)==0x3001)||((x)==0xFF64)||((x)==0xFF0E)) +#define IS_JA_TOUTEN(x) (((x)==0x3002)||((x)==0xFF61)||((x)==0xFF0C)) +#define IS_JA_SPACE(x) ((x)==0x3000) +#define IS_JA_FWLATAIN(x) IN_RANGE(x, 0xFF01, 0xFF5E) +#define IS_JA_FWNUMERAL(x) IN_RANGE(x, 0xFF10, 0xFF19) + +#define IS_JAPANESE_SPECIFIC(x) (IN_RANGE(x, 0x3040, 0x30FF)||IN_RANGE(x, 0xFF01, 0xFF9F)) + +enum char_class{ + others = 0, + space, + hiragana, + katakana, + kanji, + kuten, + touten, + kigou, + fwlatain, + ascii +}; + +static char_class getCharClass(char16_t c) +{ + char_class charClass = others; + + if(IS_JA_HIRAGANA(c)) + charClass = hiragana; + else if(IS_JA_KATAKANA(c)) + charClass = katakana; + else if(IS_JA_KANJI(c)) + charClass = kanji; + else if(IS_JA_KUTEN(c)) + charClass = kuten; + else if(IS_JA_TOUTEN(c)) + charClass = touten; + else if(IS_JA_FWLATAIN(c)) + charClass = fwlatain; + + return charClass; +} + +static bool isJapanese(const char* word) +{ + nsString text = NS_ConvertUTF8toUTF16(word); + char16_t* p = (char16_t*)text.get(); + char16_t c; + + // it is japanese chunk if it contains any hiragana or katakana. + while((c = *p++)) + if( IS_JAPANESE_SPECIFIC(c)) + return true; + + return false; +} + +static bool isFWNumeral(const char16_t* p1, const char16_t* p2) +{ + for(;p1<p2;p1++) + if(!IS_JA_FWNUMERAL(*p1)) + return false; + + return true; +} + +// The japanese tokenizer was added as part of Bug #277354 +void Tokenizer::tokenize_japanese_word(char* chunk) +{ + MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, ("entering tokenize_japanese_word(%s)", chunk)); + + nsString srcStr = NS_ConvertUTF8toUTF16(chunk); + const char16_t* p1 = srcStr.get(); + const char16_t* p2 = p1; + if(!*p2) return; + + char_class cc = getCharClass(*p2); + while(*(++p2)) + { + if(cc == getCharClass(*p2)) + continue; + + nsCString token = NS_ConvertUTF16toUTF8(p1, p2-p1); + if( (!isDecimalNumber(token.get())) && (!isFWNumeral(p1, p2))) + { + nsCString tmpStr; + tmpStr.AppendLiteral("JA:"); + tmpStr.Append(token); + add(tmpStr.get()); + } + + cc = getCharClass(*p2); + p1 = p2; + } +} + +nsresult Tokenizer::stripHTML(const nsAString& inString, nsAString& outString) +{ + uint32_t flags = nsIDocumentEncoder::OutputLFLineBreak + | nsIDocumentEncoder::OutputNoScriptContent + | nsIDocumentEncoder::OutputNoFramesContent + | nsIDocumentEncoder::OutputBodyOnly; + nsCOMPtr<nsIParserUtils> utils = + do_GetService(NS_PARSERUTILS_CONTRACTID); + return utils->ConvertToPlainText(inString, flags, 80, outString); +} + +void Tokenizer::tokenize(const char* aText) +{ + MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, ("tokenize: %s", aText)); + + // strip out HTML tags before we begin processing + // uggh but first we have to blow up our string into UCS2 + // since that's what the document encoder wants. UTF8/UCS2, I wish we all + // spoke the same language here.. + nsString text = NS_ConvertUTF8toUTF16(aText); + nsString strippedUCS2; + + // RSS feeds store their summary information as an iframe. But due to + // bug 365953, we can't see those in the plaintext serializer. As a + // workaround, allow an option to replace iframe with div in the message + // text. We disable by default, since most people won't be applying bayes + // to RSS + + if (mIframeToDiv) + { + MsgReplaceSubstring(text, NS_LITERAL_STRING("<iframe"), + NS_LITERAL_STRING("<div")); + MsgReplaceSubstring(text, NS_LITERAL_STRING("/iframe>"), + NS_LITERAL_STRING("/div>")); + } + + stripHTML(text, strippedUCS2); + + // convert 0x3000(full width space) into 0x0020 + char16_t * substr_start = strippedUCS2.BeginWriting(); + char16_t * substr_end = strippedUCS2.EndWriting(); + while (substr_start != substr_end) { + if (*substr_start == 0x3000) + *substr_start = 0x0020; + ++substr_start; + } + + nsCString strippedStr = NS_ConvertUTF16toUTF8(strippedUCS2); + char * strippedText = strippedStr.BeginWriting(); + MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, ("tokenize stripped html: %s", strippedText)); + + char* word; + char* next = strippedText; + while ((word = NS_strtok(mBodyDelimiters.get(), &next)) != NULL) { + if (!*word) continue; + if (isDecimalNumber(word)) continue; + if (isASCII(word)) + tokenize_ascii_word(word); + else if (isJapanese(word)) + tokenize_japanese_word(word); + else { + nsresult rv; + // use I18N scanner to break this word into meaningful semantic units. + if (!mScanner) { + mScanner = do_CreateInstance(NS_SEMANTICUNITSCANNER_CONTRACTID, &rv); + NS_ASSERTION(NS_SUCCEEDED(rv), "couldn't create semantic unit scanner!"); + if (NS_FAILED(rv)) { + return; + } + } + if (mScanner) { + mScanner->Start("UTF-8"); + // convert this word from UTF-8 into UCS2. + NS_ConvertUTF8toUTF16 uword(word); + ToLowerCase(uword); + const char16_t* utext = uword.get(); + int32_t len = uword.Length(), pos = 0, begin, end; + bool gotUnit; + while (pos < len) { + rv = mScanner->Next(utext, len, pos, true, &begin, &end, &gotUnit); + if (NS_SUCCEEDED(rv) && gotUnit) { + NS_ConvertUTF16toUTF8 utfUnit(utext + begin, end - begin); + add(utfUnit.get()); + // advance to end of current unit. + pos = end; + } else { + break; + } + } + } + } + } +} + +// helper function to escape \n, \t, etc from a CString +void Tokenizer::UnescapeCString(nsCString& aCString) +{ + nsAutoCString result; + + const char* readEnd = aCString.EndReading(); + char* writeStart = result.BeginWriting(); + char* writeIter = writeStart; + + bool inEscape = false; + for (const char* readIter = aCString.BeginReading(); readIter != readEnd; readIter++) + { + if (!inEscape) + { + if (*readIter == '\\') + inEscape = true; + else + *(writeIter++) = *readIter; + } + else + { + inEscape = false; + switch (*readIter) + { + case '\\': + *(writeIter++) = '\\'; + break; + case 't': + *(writeIter++) = '\t'; + break; + case 'n': + *(writeIter++) = '\n'; + break; + case 'v': + *(writeIter++) = '\v'; + break; + case 'f': + *(writeIter++) = '\f'; + break; + case 'r': + *(writeIter++) = '\r'; + break; + default: + // all other escapes are ignored + break; + } + } + } + result.SetLength(writeIter - writeStart); + aCString.Assign(result); +} + +Token* Tokenizer::copyTokens() +{ + uint32_t count = countTokens(); + if (count > 0) { + Token* tokens = new Token[count]; + if (tokens) { + Token* tp = tokens; + TokenEnumeration e(&mTokenTable); + while (e.hasMoreTokens()) + *tp++ = *(static_cast<Token*>(e.nextToken())); + } + return tokens; + } + return NULL; +} + +class TokenAnalyzer { +public: + virtual ~TokenAnalyzer() {} + + virtual void analyzeTokens(Tokenizer& tokenizer) = 0; + void setTokenListener(nsIStreamListener *aTokenListener) + { + mTokenListener = aTokenListener; + } + + void setSource(const char *sourceURI) {mTokenSource = sourceURI;} + + nsCOMPtr<nsIStreamListener> mTokenListener; + nsCString mTokenSource; + +}; + +/** + * This class downloads the raw content of an email message, buffering until + * complete segments are seen, that is until a linefeed is seen, although + * any of the valid token separators would do. This could be a further + * refinement. + */ +class TokenStreamListener : public nsIStreamListener, nsIMsgHeaderSink { +public: + NS_DECL_ISUPPORTS + NS_DECL_NSIREQUESTOBSERVER + NS_DECL_NSISTREAMLISTENER + NS_DECL_NSIMSGHEADERSINK + + TokenStreamListener(TokenAnalyzer* analyzer); +protected: + virtual ~TokenStreamListener(); + TokenAnalyzer* mAnalyzer; + char* mBuffer; + uint32_t mBufferSize; + uint32_t mLeftOverCount; + Tokenizer mTokenizer; + bool mSetAttachmentFlag; +}; + +const uint32_t kBufferSize = 16384; + +TokenStreamListener::TokenStreamListener(TokenAnalyzer* analyzer) + : mAnalyzer(analyzer), + mBuffer(NULL), mBufferSize(kBufferSize), mLeftOverCount(0), + mSetAttachmentFlag(false) +{ +} + +TokenStreamListener::~TokenStreamListener() +{ + delete[] mBuffer; + delete mAnalyzer; +} + +NS_IMPL_ISUPPORTS(TokenStreamListener, nsIRequestObserver, nsIStreamListener, nsIMsgHeaderSink) + +NS_IMETHODIMP TokenStreamListener::ProcessHeaders(nsIUTF8StringEnumerator *aHeaderNames, nsIUTF8StringEnumerator *aHeaderValues, bool dontCollectAddress) +{ + mTokenizer.tokenizeHeaders(aHeaderNames, aHeaderValues); + return NS_OK; +} + +NS_IMETHODIMP TokenStreamListener::HandleAttachment(const char *contentType, const char *url, const char16_t *displayName, const char *uri, bool aIsExternalAttachment) +{ + mTokenizer.tokenizeAttachment(contentType, NS_ConvertUTF16toUTF8(displayName).get()); + return NS_OK; +} + +NS_IMETHODIMP TokenStreamListener::AddAttachmentField(const char *field, const char *value) +{ + return NS_OK; +} + +NS_IMETHODIMP TokenStreamListener::OnEndAllAttachments() +{ + return NS_OK; +} + +NS_IMETHODIMP TokenStreamListener::OnEndMsgDownload(nsIMsgMailNewsUrl *url) +{ + return NS_OK; +} + + +NS_IMETHODIMP TokenStreamListener::OnMsgHasRemoteContent(nsIMsgDBHdr *aMsgHdr, + nsIURI *aContentURI, + bool aCanOverride) +{ + return NS_OK; +} + +NS_IMETHODIMP TokenStreamListener::OnEndMsgHeaders(nsIMsgMailNewsUrl *url) +{ + return NS_OK; +} + + +NS_IMETHODIMP TokenStreamListener::GetSecurityInfo(nsISupports * *aSecurityInfo) +{ + return NS_OK; +} +NS_IMETHODIMP TokenStreamListener::SetSecurityInfo(nsISupports * aSecurityInfo) +{ + return NS_OK; +} + +NS_IMETHODIMP TokenStreamListener::GetDummyMsgHeader(nsIMsgDBHdr **aMsgDBHdr) +{ + return NS_ERROR_NOT_IMPLEMENTED; +} + +NS_IMETHODIMP TokenStreamListener::ResetProperties() +{ + return NS_OK; +} + +NS_IMETHODIMP TokenStreamListener::GetProperties(nsIWritablePropertyBag2 * *aProperties) +{ + return NS_ERROR_NOT_IMPLEMENTED; +} + +/* void onStartRequest (in nsIRequest aRequest, in nsISupports aContext); */ +NS_IMETHODIMP TokenStreamListener::OnStartRequest(nsIRequest *aRequest, nsISupports *aContext) +{ + mLeftOverCount = 0; + if (!mBuffer) + { + mBuffer = new char[mBufferSize]; + NS_ENSURE_TRUE(mBuffer, NS_ERROR_OUT_OF_MEMORY); + } + + // get the url for the channel and set our nsIMsgHeaderSink on it so we get notified + // about the headers and attachments + + nsCOMPtr<nsIChannel> channel (do_QueryInterface(aRequest)); + if (channel) + { + nsCOMPtr<nsIURI> uri; + channel->GetURI(getter_AddRefs(uri)); + nsCOMPtr<nsIMsgMailNewsUrl> mailUrl = do_QueryInterface(uri); + if (mailUrl) + mailUrl->SetMsgHeaderSink(static_cast<nsIMsgHeaderSink*>(this)); + } + + return NS_OK; +} + +/* void onDataAvailable (in nsIRequest aRequest, in nsISupports aContext, in nsIInputStream aInputStream, in unsigned long long aOffset, in unsigned long aCount); */ +NS_IMETHODIMP TokenStreamListener::OnDataAvailable(nsIRequest *aRequest, nsISupports *aContext, nsIInputStream *aInputStream, uint64_t aOffset, uint32_t aCount) +{ + nsresult rv = NS_OK; + + while (aCount > 0) { + uint32_t readCount, totalCount = (aCount + mLeftOverCount); + if (totalCount >= mBufferSize) { + readCount = mBufferSize - mLeftOverCount - 1; + } else { + readCount = aCount; + } + + // mBuffer is supposed to be allocated in onStartRequest. But something + // is causing that to not happen, so as a last-ditch attempt we'll + // do it here. + if (!mBuffer) + { + mBuffer = new char[mBufferSize]; + NS_ENSURE_TRUE(mBuffer, NS_ERROR_OUT_OF_MEMORY); + } + + char* buffer = mBuffer; + rv = aInputStream->Read(buffer + mLeftOverCount, readCount, &readCount); + if (NS_FAILED(rv)) + break; + + if (readCount == 0) { + rv = NS_ERROR_UNEXPECTED; + NS_WARNING("failed to tokenize"); + break; + } + + aCount -= readCount; + + /* consume the tokens up to the last legal token delimiter in the buffer. */ + totalCount = (readCount + mLeftOverCount); + buffer[totalCount] = '\0'; + char* lastDelimiter = NULL; + char* scan = buffer + totalCount; + while (scan > buffer) { + if (strchr(mTokenizer.mBodyDelimiters.get(), *--scan)) { + lastDelimiter = scan; + break; + } + } + + if (lastDelimiter) { + *lastDelimiter = '\0'; + mTokenizer.tokenize(buffer); + + uint32_t consumedCount = 1 + (lastDelimiter - buffer); + mLeftOverCount = totalCount - consumedCount; + if (mLeftOverCount) + memmove(buffer, buffer + consumedCount, mLeftOverCount); + } else { + /* didn't find a delimiter, keep the whole buffer around. */ + mLeftOverCount = totalCount; + if (totalCount >= (mBufferSize / 2)) { + uint32_t newBufferSize = mBufferSize * 2; + char* newBuffer = new char[newBufferSize]; + NS_ENSURE_TRUE(newBuffer, NS_ERROR_OUT_OF_MEMORY); + memcpy(newBuffer, mBuffer, mLeftOverCount); + delete[] mBuffer; + mBuffer = newBuffer; + mBufferSize = newBufferSize; + } + } + } + + return rv; +} + +/* void onStopRequest (in nsIRequest aRequest, in nsISupports aContext, in nsresult aStatusCode); */ +NS_IMETHODIMP TokenStreamListener::OnStopRequest(nsIRequest *aRequest, nsISupports *aContext, nsresult aStatusCode) +{ + if (mLeftOverCount) { + /* assume final buffer is complete. */ + mBuffer[mLeftOverCount] = '\0'; + mTokenizer.tokenize(mBuffer); + } + + /* finally, analyze the tokenized message. */ + MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, ("analyze the tokenized message")); + if (mAnalyzer) + mAnalyzer->analyzeTokens(mTokenizer); + + return NS_OK; +} + +/* Implementation file */ + +NS_IMPL_ISUPPORTS(nsBayesianFilter, nsIMsgFilterPlugin, + nsIJunkMailPlugin, nsIMsgCorpus, nsISupportsWeakReference, + nsIObserver) + +nsBayesianFilter::nsBayesianFilter() + : mTrainingDataDirty(false) +{ + if (!BayesianFilterLogModule) + BayesianFilterLogModule = PR_NewLogModule("BayesianFilter"); + + int32_t junkThreshold = 0; + nsresult rv; + nsCOMPtr<nsIPrefBranch> pPrefBranch(do_GetService(NS_PREFSERVICE_CONTRACTID, &rv)); + if (pPrefBranch) + pPrefBranch->GetIntPref("mail.adaptivefilters.junk_threshold", &junkThreshold); + + mJunkProbabilityThreshold = (static_cast<double>(junkThreshold)) / 100.0; + if (mJunkProbabilityThreshold == 0 || mJunkProbabilityThreshold >= 1) + mJunkProbabilityThreshold = kDefaultJunkThreshold; + + MOZ_LOG(BayesianFilterLogModule, LogLevel::Warning, ("junk probability threshold: %f", mJunkProbabilityThreshold)); + + mCorpus.readTrainingData(); + + // get parameters for training data flushing, from the prefs + + nsCOMPtr<nsIPrefBranch> prefBranch; + + nsCOMPtr<nsIPrefService> prefs = do_GetService(NS_PREFSERVICE_CONTRACTID, &rv); + NS_ASSERTION(NS_SUCCEEDED(rv),"failed accessing preferences service"); + rv = prefs->GetBranch(nullptr, getter_AddRefs(prefBranch)); + NS_ASSERTION(NS_SUCCEEDED(rv),"failed getting preferences branch"); + + rv = prefBranch->GetIntPref("mailnews.bayesian_spam_filter.flush.minimum_interval",&mMinFlushInterval); + // it is not a good idea to allow a minimum interval of under 1 second + if (NS_FAILED(rv) || (mMinFlushInterval <= 1000) ) + mMinFlushInterval = DEFAULT_MIN_INTERVAL_BETWEEN_WRITES; + + rv = prefBranch->GetIntPref("mailnews.bayesian_spam_filter.junk_maxtokens", &mMaximumTokenCount); + if (NS_FAILED(rv)) + mMaximumTokenCount = 0; // which means do not limit token counts + MOZ_LOG(BayesianFilterLogModule, LogLevel::Warning, ("maximum junk tokens: %d", mMaximumTokenCount)); + + mTimer = do_CreateInstance(NS_TIMER_CONTRACTID, &rv); + NS_ASSERTION(NS_SUCCEEDED(rv), "unable to create a timer; training data will only be written on exit"); + + // the timer is not used on object construction, since for + // the time being there are no dirying messages + + // give a default capacity to the memory structure used to store + // per-message/per-trait token data + mAnalysisStore.SetCapacity(kAnalysisStoreCapacity); + + // dummy 0th element. Index 0 means "end of list" so we need to + // start from 1 + AnalysisPerToken analysisPT(0, 0.0, 0.0); + mAnalysisStore.AppendElement(analysisPT); + mNextAnalysisIndex = 1; +} + +nsresult nsBayesianFilter::Init() +{ + nsCOMPtr<nsIObserverService> observerService = + mozilla::services::GetObserverService(); + if (observerService) + observerService->AddObserver(this, "profile-before-change", true); + return NS_OK; +} + +void +nsBayesianFilter::TimerCallback(nsITimer* aTimer, void* aClosure) +{ + // we will flush the training data to disk after enough time has passed + // since the first time a message has been classified after the last flush + + nsBayesianFilter *filter = static_cast<nsBayesianFilter *>(aClosure); + filter->mCorpus.writeTrainingData(filter->mMaximumTokenCount); + filter->mTrainingDataDirty = false; +} + +nsBayesianFilter::~nsBayesianFilter() +{ + if (mTimer) + { + mTimer->Cancel(); + mTimer = nullptr; + } + // call shutdown when we are going away in case we need + // to flush the training set to disk + Shutdown(); +} + +// this object is used for one call to classifyMessage or classifyMessages(). +// So if we're classifying multiple messages, this object will be used for each message. +// It's going to hold a reference to itself, basically, to stay in memory. +class MessageClassifier : public TokenAnalyzer { +public: + // full classifier with arbitrary traits + MessageClassifier(nsBayesianFilter* aFilter, + nsIJunkMailClassificationListener* aJunkListener, + nsIMsgTraitClassificationListener* aTraitListener, + nsIMsgTraitDetailListener* aDetailListener, + nsTArray<uint32_t>& aProTraits, + nsTArray<uint32_t>& aAntiTraits, + nsIMsgWindow *aMsgWindow, + uint32_t aNumMessagesToClassify, + const char **aMessageURIs) + : mFilter(aFilter), + mJunkMailPlugin(aFilter), + mJunkListener(aJunkListener), + mTraitListener(aTraitListener), + mDetailListener(aDetailListener), + mProTraits(aProTraits), + mAntiTraits(aAntiTraits), + mMsgWindow(aMsgWindow) + { + mCurMessageToClassify = 0; + mNumMessagesToClassify = aNumMessagesToClassify; + mMessageURIs = (char **) moz_xmalloc(sizeof(char *) * aNumMessagesToClassify); + for (uint32_t i = 0; i < aNumMessagesToClassify; i++) + mMessageURIs[i] = PL_strdup(aMessageURIs[i]); + + } + + // junk-only classifier + MessageClassifier(nsBayesianFilter* aFilter, + nsIJunkMailClassificationListener* aJunkListener, + nsIMsgWindow *aMsgWindow, + uint32_t aNumMessagesToClassify, + const char **aMessageURIs) + : mFilter(aFilter), + mJunkMailPlugin(aFilter), + mJunkListener(aJunkListener), + mTraitListener(nullptr), + mDetailListener(nullptr), + mMsgWindow(aMsgWindow) + { + mCurMessageToClassify = 0; + mNumMessagesToClassify = aNumMessagesToClassify; + mMessageURIs = (char **) moz_xmalloc(sizeof(char *) * aNumMessagesToClassify); + for (uint32_t i = 0; i < aNumMessagesToClassify; i++) + mMessageURIs[i] = PL_strdup(aMessageURIs[i]); + mProTraits.AppendElement(kJunkTrait); + mAntiTraits.AppendElement(kGoodTrait); + + } + + virtual ~MessageClassifier() + { + if (mMessageURIs) + { + NS_FREE_XPCOM_ALLOCATED_POINTER_ARRAY(mNumMessagesToClassify, mMessageURIs); + } + } + virtual void analyzeTokens(Tokenizer& tokenizer) + { + mFilter->classifyMessage(tokenizer, + mTokenSource.get(), + mProTraits, + mAntiTraits, + mJunkListener, + mTraitListener, + mDetailListener); + tokenizer.clearTokens(); + classifyNextMessage(); + } + + virtual void classifyNextMessage() + { + + if (++mCurMessageToClassify < mNumMessagesToClassify && mMessageURIs[mCurMessageToClassify]) { + MOZ_LOG(BayesianFilterLogModule, LogLevel::Warning, ("classifyNextMessage(%s)", mMessageURIs[mCurMessageToClassify])); + mFilter->tokenizeMessage(mMessageURIs[mCurMessageToClassify], mMsgWindow, this); + } + else + { + // call all listeners with null parameters to signify end of batch + if (mJunkListener) + mJunkListener->OnMessageClassified(nullptr, nsIJunkMailPlugin::UNCLASSIFIED, 0); + if (mTraitListener) + mTraitListener->OnMessageTraitsClassified(nullptr, 0, nullptr, nullptr); + mTokenListener = nullptr; // this breaks the circular ref that keeps this object alive + // so we will be destroyed as a result. + } + } + +private: + nsBayesianFilter* mFilter; + nsCOMPtr<nsIJunkMailPlugin> mJunkMailPlugin; + nsCOMPtr<nsIJunkMailClassificationListener> mJunkListener; + nsCOMPtr<nsIMsgTraitClassificationListener> mTraitListener; + nsCOMPtr<nsIMsgTraitDetailListener> mDetailListener; + nsTArray<uint32_t> mProTraits; + nsTArray<uint32_t> mAntiTraits; + nsCOMPtr<nsIMsgWindow> mMsgWindow; + int32_t mNumMessagesToClassify; + int32_t mCurMessageToClassify; // 0-based index + char **mMessageURIs; +}; + +nsresult nsBayesianFilter::tokenizeMessage(const char* aMessageURI, nsIMsgWindow *aMsgWindow, TokenAnalyzer* aAnalyzer) +{ + NS_ENSURE_ARG_POINTER(aMessageURI); + + nsCOMPtr <nsIMsgMessageService> msgService; + nsresult rv = GetMessageServiceFromURI(nsDependentCString(aMessageURI), getter_AddRefs(msgService)); + NS_ENSURE_SUCCESS(rv, rv); + + aAnalyzer->setSource(aMessageURI); + nsCOMPtr<nsIURI> dummyNull; + return msgService->StreamMessage(aMessageURI, aAnalyzer->mTokenListener, + aMsgWindow, nullptr, true /* convert data */, + NS_LITERAL_CSTRING("filter"), false, getter_AddRefs(dummyNull)); +} + +// a TraitAnalysis is the per-token representation of the statistical +// calculations, basically created to group information that is then +// sorted by mDistance +struct TraitAnalysis +{ + uint32_t mTokenIndex; + double mDistance; + double mProbability; +}; + +// comparator required to sort an nsTArray +class compareTraitAnalysis +{ +public: + bool Equals(const TraitAnalysis& a, const TraitAnalysis& b) const + { + return a.mDistance == b.mDistance; + } + bool LessThan(const TraitAnalysis& a, const TraitAnalysis& b) const + { + return a.mDistance < b.mDistance; + } +}; + +inline double dmax(double x, double y) { return (x > y ? x : y); } +inline double dmin(double x, double y) { return (x < y ? x : y); } + +// Chi square functions are implemented by an incomplete gamma function. +// Note that chi2P's callers multiply the arguments by 2 but chi2P +// divides them by 2 again. Inlining chi2P gives the compiler a +// chance to notice this. + +// Both chi2P and nsIncompleteGammaP set *error negative on domain +// errors and nsIncompleteGammaP sets it posivive on internal errors. +// This may be useful but the chi2P callers treat any error as fatal. + +// Note that converting unsigned ints to floating point can be slow on +// some platforms (like Intel) so use signed quantities for the numeric +// routines. +static inline double chi2P (double chi2, double nu, int32_t *error) +{ + // domain checks; set error and return a dummy value + if (chi2 < 0.0 || nu <= 0.0) + { + *error = -1; + return 0.0; + } + // reversing the arguments is intentional + return nsIncompleteGammaP (nu/2.0, chi2/2.0, error); +} + +void nsBayesianFilter::classifyMessage( + Tokenizer& tokenizer, + const char* messageURI, + nsTArray<uint32_t>& aProTraits, + nsTArray<uint32_t>& aAntiTraits, + nsIJunkMailClassificationListener* listener, + nsIMsgTraitClassificationListener* aTraitListener, + nsIMsgTraitDetailListener* aDetailListener) +{ + Token* tokens = tokenizer.copyTokens(); + uint32_t tokenCount; + if (!tokens) + { + // This can happen with problems with UTF conversion + NS_ERROR("Trying to classify a null or invalid message"); + tokenCount = 0; + // don't return so that we still call the listeners + } + else + { + tokenCount = tokenizer.countTokens(); + } + + if (aProTraits.Length() != aAntiTraits.Length()) + { + NS_ERROR("Each Pro trait needs a matching Anti trait"); + return; + } + + /* this part is similar to the Graham algorithm with some adjustments. */ + uint32_t traitCount = aProTraits.Length(); + + // pro message counts per trait index + AutoTArray<uint32_t, kTraitAutoCapacity> numProMessages; + // anti message counts per trait index + AutoTArray<uint32_t, kTraitAutoCapacity> numAntiMessages; + // array of pro aliases per trait index + AutoTArray<uint32_t*, kTraitAutoCapacity > proAliasArrays; + // number of pro aliases per trait index + AutoTArray<uint32_t, kTraitAutoCapacity > proAliasesLengths; + // array of anti aliases per trait index + AutoTArray<uint32_t*, kTraitAutoCapacity> antiAliasArrays; + // number of anti aliases per trait index + AutoTArray<uint32_t, kTraitAutoCapacity > antiAliasesLengths; + // construct the outgoing listener arrays + AutoTArray<uint32_t, kTraitAutoCapacity> traits; + AutoTArray<uint32_t, kTraitAutoCapacity> percents; + if (traitCount > kTraitAutoCapacity) + { + traits.SetCapacity(traitCount); + percents.SetCapacity(traitCount); + numProMessages.SetCapacity(traitCount); + numAntiMessages.SetCapacity(traitCount); + proAliasesLengths.SetCapacity(traitCount); + antiAliasesLengths.SetCapacity(traitCount); + proAliasArrays.SetCapacity(traitCount); + antiAliasArrays.SetCapacity(traitCount); + } + + nsresult rv; + nsCOMPtr<nsIMsgTraitService> traitService(do_GetService("@mozilla.org/msg-trait-service;1", &rv)); + if (NS_FAILED(rv)) + { + NS_ERROR("Failed to get trait service"); + MOZ_LOG(BayesianFilterLogModule, LogLevel::Error, ("Failed to get trait service")); + } + + // get aliases and message counts for the pro and anti traits + for (uint32_t traitIndex = 0; traitIndex < traitCount; traitIndex++) + { + nsresult rv; + + // pro trait + uint32_t proAliasesLength = 0; + uint32_t* proAliases = nullptr; + uint32_t proTrait = aProTraits[traitIndex]; + if (traitService) + { + rv = traitService->GetAliases(proTrait, &proAliasesLength, &proAliases); + if (NS_FAILED(rv)) + { + NS_ERROR("trait service failed to get aliases"); + MOZ_LOG(BayesianFilterLogModule, LogLevel::Error, ("trait service failed to get aliases")); + } + } + proAliasesLengths.AppendElement(proAliasesLength); + proAliasArrays.AppendElement(proAliases); + uint32_t proMessageCount = mCorpus.getMessageCount(proTrait); + for (uint32_t aliasIndex = 0; aliasIndex < proAliasesLength; aliasIndex++) + proMessageCount += mCorpus.getMessageCount(proAliases[aliasIndex]); + numProMessages.AppendElement(proMessageCount); + + // anti trait + uint32_t antiAliasesLength = 0; + uint32_t* antiAliases = nullptr; + uint32_t antiTrait = aAntiTraits[traitIndex]; + if (traitService) + { + rv = traitService->GetAliases(antiTrait, &antiAliasesLength, &antiAliases); + if (NS_FAILED(rv)) + { + NS_ERROR("trait service failed to get aliases"); + MOZ_LOG(BayesianFilterLogModule, LogLevel::Error, ("trait service failed to get aliases")); + } + } + antiAliasesLengths.AppendElement(antiAliasesLength); + antiAliasArrays.AppendElement(antiAliases); + uint32_t antiMessageCount = mCorpus.getMessageCount(antiTrait); + for (uint32_t aliasIndex = 0; aliasIndex < antiAliasesLength; aliasIndex++) + antiMessageCount += mCorpus.getMessageCount(antiAliases[aliasIndex]); + numAntiMessages.AppendElement(antiMessageCount); + } + + for (uint32_t i = 0; i < tokenCount; ++i) + { + Token& token = tokens[i]; + CorpusToken* t = mCorpus.get(token.mWord); + if (!t) + continue; + for (uint32_t traitIndex = 0; traitIndex < traitCount; traitIndex++) + { + uint32_t iProCount = mCorpus.getTraitCount(t, aProTraits[traitIndex]); + // add in any counts for aliases to proTrait + for (uint32_t aliasIndex = 0; aliasIndex < proAliasesLengths[traitIndex]; aliasIndex++) + iProCount += mCorpus.getTraitCount(t, proAliasArrays[traitIndex][aliasIndex]); + double proCount = static_cast<double>(iProCount); + + uint32_t iAntiCount = mCorpus.getTraitCount(t, aAntiTraits[traitIndex]); + // add in any counts for aliases to antiTrait + for (uint32_t aliasIndex = 0; aliasIndex < antiAliasesLengths[traitIndex]; aliasIndex++) + iAntiCount += mCorpus.getTraitCount(t, antiAliasArrays[traitIndex][aliasIndex]); + double antiCount = static_cast<double>(iAntiCount); + + double prob, denom; + // Prevent a divide by zero error by setting defaults for prob + + // If there are no matching tokens at all, ignore. + if (antiCount == 0.0 && proCount == 0.0) + continue; + // if only anti match, set probability to 0% + if (proCount == 0.0) + prob = 0.0; + // if only pro match, set probability to 100% + else if (antiCount == 0.0) + prob = 1.0; + // not really needed, but just to be sure check the denom as well + else if ((denom = proCount * numAntiMessages[traitIndex] + + antiCount * numProMessages[traitIndex]) == 0.0) + continue; + else + prob = (proCount * numAntiMessages[traitIndex]) / denom; + + double n = proCount + antiCount; + prob = (0.225 + n * prob) / (.45 + n); + double distance = std::abs(prob - 0.5); + if (distance >= .1) + { + mozilla::DebugOnly<nsresult> rv = setAnalysis(token, traitIndex, distance, prob); + NS_ASSERTION(NS_SUCCEEDED(rv), "Problem in setAnalysis"); + } + } + } + + for (uint32_t traitIndex = 0; traitIndex < traitCount; traitIndex++) + { + AutoTArray<TraitAnalysis, 1024> traitAnalyses; + // copy valid tokens into an array to sort + for (uint32_t tokenIndex = 0; tokenIndex < tokenCount; tokenIndex++) + { + uint32_t storeIndex = getAnalysisIndex(tokens[tokenIndex], traitIndex); + if (storeIndex) + { + TraitAnalysis ta = + {tokenIndex, + mAnalysisStore[storeIndex].mDistance, + mAnalysisStore[storeIndex].mProbability}; + traitAnalyses.AppendElement(ta); + } + } + + // sort the array by the distances + traitAnalyses.Sort(compareTraitAnalysis()); + uint32_t count = traitAnalyses.Length(); + uint32_t first, last = count; + const uint32_t kMaxTokens = 150; + first = ( count > kMaxTokens) ? count - kMaxTokens : 0; + + // Setup the arrays to save details if needed + nsTArray<double> sArray; + nsTArray<double> hArray; + uint32_t usedTokenCount = ( count > kMaxTokens) ? kMaxTokens : count; + if (aDetailListener) + { + sArray.SetCapacity(usedTokenCount); + hArray.SetCapacity(usedTokenCount); + } + + double H = 1.0, S = 1.0; + int32_t Hexp = 0, Sexp = 0; + uint32_t goodclues=0; + int e; + + // index from end to analyze most significant first + for (uint32_t ip1 = last; ip1 != first; --ip1) + { + TraitAnalysis& ta = traitAnalyses[ip1 - 1]; + if (ta.mDistance > 0.0) + { + goodclues++; + double value = ta.mProbability; + S *= (1.0 - value); + H *= value; + if ( S < 1e-200 ) + { + S = frexp(S, &e); + Sexp += e; + } + if ( H < 1e-200 ) + { + H = frexp(H, &e); + Hexp += e; + } + MOZ_LOG(BayesianFilterLogModule, LogLevel::Warning, + ("token probability (%s) is %f", + tokens[ta.mTokenIndex].mWord, ta.mProbability)); + } + if (aDetailListener) + { + sArray.AppendElement(log(S) + Sexp * M_LN2); + hArray.AppendElement(log(H) + Hexp * M_LN2); + } + } + + S = log(S) + Sexp * M_LN2; + H = log(H) + Hexp * M_LN2; + + double prob; + if (goodclues > 0) + { + int32_t chi_error; + S = chi2P(-2.0 * S, 2.0 * goodclues, &chi_error); + if (!chi_error) + H = chi2P(-2.0 * H, 2.0 * goodclues, &chi_error); + // if any error toss the entire calculation + if (!chi_error) + prob = (S-H +1.0) / 2.0; + else + prob = 0.5; + } + else + prob = 0.5; + + if (aDetailListener) + { + // Prepare output arrays + nsTArray<uint32_t> tokenPercents(usedTokenCount); + nsTArray<uint32_t> runningPercents(usedTokenCount); + nsTArray<char16_t*> tokenStrings(usedTokenCount); + + double clueCount = 1.0; + for (uint32_t tokenIndex = 0; tokenIndex < usedTokenCount; tokenIndex++) + { + TraitAnalysis& ta = traitAnalyses[last - 1 - tokenIndex]; + int32_t chi_error; + S = chi2P(-2.0 * sArray[tokenIndex], 2.0 * clueCount, &chi_error); + if (!chi_error) + H = chi2P(-2.0 * hArray[tokenIndex], 2.0 * clueCount, &chi_error); + clueCount += 1.0; + double runningProb; + if (!chi_error) + runningProb = (S - H + 1.0) / 2.0; + else + runningProb = 0.5; + runningPercents.AppendElement(static_cast<uint32_t>(runningProb * + 100. + .5)); + tokenPercents.AppendElement(static_cast<uint32_t>(ta.mProbability * + 100. + .5)); + tokenStrings.AppendElement(ToNewUnicode(NS_ConvertUTF8toUTF16( + tokens[ta.mTokenIndex].mWord))); + } + + aDetailListener->OnMessageTraitDetails(messageURI, aProTraits[traitIndex], + usedTokenCount, (const char16_t**)tokenStrings.Elements(), + tokenPercents.Elements(), runningPercents.Elements()); + for (uint32_t tokenIndex = 0; tokenIndex < usedTokenCount; tokenIndex++) + NS_Free(tokenStrings[tokenIndex]); + } + + uint32_t proPercent = static_cast<uint32_t>(prob*100. + .5); + + // directly classify junk to maintain backwards compatibility + if (aProTraits[traitIndex] == kJunkTrait) + { + bool isJunk = (prob >= mJunkProbabilityThreshold); + MOZ_LOG(BayesianFilterLogModule, LogLevel::Info, + ("%s is junk probability = (%f) HAM SCORE:%f SPAM SCORE:%f", + messageURI, prob,H,S)); + + // the algorithm in "A Plan For Spam" assumes that you have a large good + // corpus and a large junk corpus. + // that won't be the case with users who first use the junk mail trait + // so, we do certain things to encourage them to train. + // + // if there are no good tokens, assume the message is junk + // this will "encourage" the user to train + // and if there are no bad tokens, assume the message is not junk + // this will also "encourage" the user to train + // see bug #194238 + + if (listener && !mCorpus.getMessageCount(kGoodTrait)) + isJunk = true; + else if (listener && !mCorpus.getMessageCount(kJunkTrait)) + isJunk = false; + + if (listener) + listener->OnMessageClassified(messageURI, isJunk ? + nsMsgJunkStatus(nsIJunkMailPlugin::JUNK) : + nsMsgJunkStatus(nsIJunkMailPlugin::GOOD), proPercent); + } + + if (aTraitListener) + { + traits.AppendElement(aProTraits[traitIndex]); + percents.AppendElement(proPercent); + } + + // free aliases arrays returned from XPCOM + if (proAliasesLengths[traitIndex]) + NS_Free(proAliasArrays[traitIndex]); + if (antiAliasesLengths[traitIndex]) + NS_Free(antiAliasArrays[traitIndex]); + } + + if (aTraitListener) + aTraitListener->OnMessageTraitsClassified(messageURI, + traits.Length(), traits.Elements(), percents.Elements()); + + delete[] tokens; + // reuse mAnalysisStore without clearing memory + mNextAnalysisIndex = 1; + // but shrink it back to the default size + if (mAnalysisStore.Length() > kAnalysisStoreCapacity) + mAnalysisStore.RemoveElementsAt(kAnalysisStoreCapacity, + mAnalysisStore.Length() - kAnalysisStoreCapacity); + mAnalysisStore.Compact(); +} + +void nsBayesianFilter::classifyMessage( + Tokenizer& tokens, + const char* messageURI, + nsIJunkMailClassificationListener* aJunkListener) +{ + AutoTArray<uint32_t, 1> proTraits; + AutoTArray<uint32_t, 1> antiTraits; + proTraits.AppendElement(kJunkTrait); + antiTraits.AppendElement(kGoodTrait); + classifyMessage(tokens, messageURI, proTraits, antiTraits, + aJunkListener, nullptr, nullptr); +} + +NS_IMETHODIMP +nsBayesianFilter::Observe(nsISupports *aSubject, const char *aTopic, + const char16_t *someData) +{ + if (!strcmp(aTopic, "profile-before-change")) + Shutdown(); + return NS_OK; +} + +/* void shutdown (); */ +NS_IMETHODIMP nsBayesianFilter::Shutdown() +{ + if (mTrainingDataDirty) + mCorpus.writeTrainingData(mMaximumTokenCount); + mTrainingDataDirty = false; + + return NS_OK; +} + +/* readonly attribute boolean shouldDownloadAllHeaders; */ +NS_IMETHODIMP nsBayesianFilter::GetShouldDownloadAllHeaders(bool *aShouldDownloadAllHeaders) +{ + // bayesian filters work on the whole msg body currently. + *aShouldDownloadAllHeaders = false; + return NS_OK; +} + +/* void classifyMessage (in string aMsgURL, in nsIJunkMailClassificationListener aListener); */ +NS_IMETHODIMP nsBayesianFilter::ClassifyMessage(const char *aMessageURL, nsIMsgWindow *aMsgWindow, nsIJunkMailClassificationListener *aListener) +{ + MessageClassifier* analyzer = new MessageClassifier(this, aListener, aMsgWindow, 1, &aMessageURL); + NS_ENSURE_TRUE(analyzer, NS_ERROR_OUT_OF_MEMORY); + TokenStreamListener *tokenListener = new TokenStreamListener(analyzer); + NS_ENSURE_TRUE(tokenListener, NS_ERROR_OUT_OF_MEMORY); + analyzer->setTokenListener(tokenListener); + return tokenizeMessage(aMessageURL, aMsgWindow, analyzer); +} + +/* void classifyMessages (in unsigned long aCount, [array, size_is (aCount)] in string aMsgURLs, in nsIJunkMailClassificationListener aListener); */ +NS_IMETHODIMP nsBayesianFilter::ClassifyMessages(uint32_t aCount, const char **aMsgURLs, nsIMsgWindow *aMsgWindow, nsIJunkMailClassificationListener *aListener) +{ + NS_ENSURE_ARG_POINTER(aMsgURLs); + + TokenAnalyzer* analyzer = new MessageClassifier(this, aListener, aMsgWindow, aCount, aMsgURLs); + NS_ENSURE_TRUE(analyzer, NS_ERROR_OUT_OF_MEMORY); + TokenStreamListener *tokenListener = new TokenStreamListener(analyzer); + NS_ENSURE_TRUE(tokenListener, NS_ERROR_OUT_OF_MEMORY); + analyzer->setTokenListener(tokenListener); + return tokenizeMessage(aMsgURLs[0], aMsgWindow, analyzer); +} + +nsresult nsBayesianFilter::setAnalysis(Token& token, uint32_t aTraitIndex, + double aDistance, double aProbability) +{ + uint32_t nextLink = token.mAnalysisLink; + uint32_t lastLink = 0; + uint32_t linkCount = 0, maxLinks = 100; + + // try to find an existing element. Limit the search to maxLinks + // as a precaution + for (linkCount = 0; nextLink && linkCount < maxLinks; linkCount++) + { + AnalysisPerToken &rAnalysis = mAnalysisStore[nextLink]; + if (rAnalysis.mTraitIndex == aTraitIndex) + { + rAnalysis.mDistance = aDistance; + rAnalysis.mProbability = aProbability; + return NS_OK; + } + lastLink = nextLink; + nextLink = rAnalysis.mNextLink; + } + if (linkCount >= maxLinks) + return NS_ERROR_FAILURE; + + // trait does not exist, so add it + + AnalysisPerToken analysis(aTraitIndex, aDistance, aProbability); + if (mAnalysisStore.Length() == mNextAnalysisIndex) + mAnalysisStore.InsertElementAt(mNextAnalysisIndex, analysis); + else if (mAnalysisStore.Length() > mNextAnalysisIndex) + mAnalysisStore.ReplaceElementsAt(mNextAnalysisIndex, 1, analysis); + else // we can only insert at the end of the array + return NS_ERROR_FAILURE; + + if (lastLink) + // the token had at least one link, so update the last link to point to + // the new item + mAnalysisStore[lastLink].mNextLink = mNextAnalysisIndex; + else + // need to update the token's first link + token.mAnalysisLink = mNextAnalysisIndex; + mNextAnalysisIndex++; + return NS_OK; +} + +uint32_t nsBayesianFilter::getAnalysisIndex(Token& token, uint32_t aTraitIndex) +{ + uint32_t nextLink; + uint32_t linkCount = 0, maxLinks = 100; + for (nextLink = token.mAnalysisLink; nextLink && linkCount < maxLinks; linkCount++) + { + AnalysisPerToken &rAnalysis = mAnalysisStore[nextLink]; + if (rAnalysis.mTraitIndex == aTraitIndex) + return nextLink; + nextLink = rAnalysis.mNextLink; + } + NS_ASSERTION(linkCount < maxLinks, "corrupt analysis store"); + + // Trait not found, indicate by zero + return 0; +} + +NS_IMETHODIMP nsBayesianFilter::ClassifyTraitsInMessage( + const char *aMsgURI, + uint32_t aTraitCount, + uint32_t *aProTraits, + uint32_t *aAntiTraits, + nsIMsgTraitClassificationListener *aTraitListener, + nsIMsgWindow *aMsgWindow, + nsIJunkMailClassificationListener *aJunkListener) +{ + return ClassifyTraitsInMessages(1, &aMsgURI, aTraitCount, aProTraits, + aAntiTraits, aTraitListener, aMsgWindow, aJunkListener); +} + +NS_IMETHODIMP nsBayesianFilter::ClassifyTraitsInMessages( + uint32_t aCount, + const char **aMsgURIs, + uint32_t aTraitCount, + uint32_t *aProTraits, + uint32_t *aAntiTraits, + nsIMsgTraitClassificationListener *aTraitListener, + nsIMsgWindow *aMsgWindow, + nsIJunkMailClassificationListener *aJunkListener) +{ + AutoTArray<uint32_t, kTraitAutoCapacity> proTraits; + AutoTArray<uint32_t, kTraitAutoCapacity> antiTraits; + if (aTraitCount > kTraitAutoCapacity) + { + proTraits.SetCapacity(aTraitCount); + antiTraits.SetCapacity(aTraitCount); + } + proTraits.AppendElements(aProTraits, aTraitCount); + antiTraits.AppendElements(aAntiTraits, aTraitCount); + + MessageClassifier* analyzer = new MessageClassifier(this, aJunkListener, + aTraitListener, nullptr, proTraits, antiTraits, aMsgWindow, aCount, aMsgURIs); + NS_ENSURE_TRUE(analyzer, NS_ERROR_OUT_OF_MEMORY); + + TokenStreamListener *tokenListener = new TokenStreamListener(analyzer); + NS_ENSURE_TRUE(tokenListener, NS_ERROR_OUT_OF_MEMORY); + + analyzer->setTokenListener(tokenListener); + return tokenizeMessage(aMsgURIs[0], aMsgWindow, analyzer); +} + +class MessageObserver : public TokenAnalyzer { +public: + MessageObserver(nsBayesianFilter* filter, + nsTArray<uint32_t>& aOldClassifications, + nsTArray<uint32_t>& aNewClassifications, + nsIJunkMailClassificationListener* aJunkListener, + nsIMsgTraitClassificationListener* aTraitListener) + : mFilter(filter), mJunkMailPlugin(filter), mJunkListener(aJunkListener), + mTraitListener(aTraitListener), + mOldClassifications(aOldClassifications), + mNewClassifications(aNewClassifications) + { + } + + virtual void analyzeTokens(Tokenizer& tokenizer) + { + mFilter->observeMessage(tokenizer, mTokenSource.get(), mOldClassifications, + mNewClassifications, mJunkListener, mTraitListener); + // release reference to listener, which will allow us to go away as well. + mTokenListener = nullptr; + } + +private: + nsBayesianFilter* mFilter; + nsCOMPtr<nsIJunkMailPlugin> mJunkMailPlugin; + nsCOMPtr<nsIJunkMailClassificationListener> mJunkListener; + nsCOMPtr<nsIMsgTraitClassificationListener> mTraitListener; + nsTArray<uint32_t> mOldClassifications; + nsTArray<uint32_t> mNewClassifications; +}; + +NS_IMETHODIMP nsBayesianFilter::SetMsgTraitClassification( + const char *aMsgURI, + uint32_t aOldCount, + uint32_t *aOldTraits, + uint32_t aNewCount, + uint32_t *aNewTraits, + nsIMsgTraitClassificationListener *aTraitListener, + nsIMsgWindow *aMsgWindow, + nsIJunkMailClassificationListener *aJunkListener) +{ + AutoTArray<uint32_t, kTraitAutoCapacity> oldTraits; + AutoTArray<uint32_t, kTraitAutoCapacity> newTraits; + if (aOldCount > kTraitAutoCapacity) + oldTraits.SetCapacity(aOldCount); + if (aNewCount > kTraitAutoCapacity) + newTraits.SetCapacity(aNewCount); + oldTraits.AppendElements(aOldTraits, aOldCount); + newTraits.AppendElements(aNewTraits, aNewCount); + + MessageObserver* analyzer = new MessageObserver(this, oldTraits, + newTraits, aJunkListener, aTraitListener); + NS_ENSURE_TRUE(analyzer, NS_ERROR_OUT_OF_MEMORY); + + TokenStreamListener *tokenListener = new TokenStreamListener(analyzer); + NS_ENSURE_TRUE(tokenListener, NS_ERROR_OUT_OF_MEMORY); + + analyzer->setTokenListener(tokenListener); + return tokenizeMessage(aMsgURI, aMsgWindow, analyzer); +} + +// set new message classifications for a message +void nsBayesianFilter::observeMessage( + Tokenizer& tokenizer, + const char* messageURL, + nsTArray<uint32_t>& oldClassifications, + nsTArray<uint32_t>& newClassifications, + nsIJunkMailClassificationListener* aJunkListener, + nsIMsgTraitClassificationListener* aTraitListener) +{ + + bool trainingDataWasDirty = mTrainingDataDirty; + + // Uhoh...if the user is re-training then the message may already be classified and we are classifying it again with the same classification. + // the old code would have removed the tokens for this message then added them back. But this really hurts the message occurrence + // count for tokens if you just removed training.dat and are re-training. See Bug #237095 for more details. + // What can we do here? Well we can skip the token removal step if the classifications are the same and assume the user is + // just re-training. But this then allows users to re-classify the same message on the same training set over and over again + // leading to data skew. But that's all I can think to do right now to address this..... + uint32_t oldLength = oldClassifications.Length(); + for (uint32_t index = 0; index < oldLength; index++) + { + uint32_t trait = oldClassifications.ElementAt(index); + // skip removing if trait is also in the new set + if (newClassifications.Contains(trait)) + continue; + // remove the tokens from the token set it is currently in + uint32_t messageCount; + messageCount = mCorpus.getMessageCount(trait); + if (messageCount > 0) + { + mCorpus.setMessageCount(trait, messageCount - 1); + mCorpus.forgetTokens(tokenizer, trait, 1); + mTrainingDataDirty = true; + } + } + + nsMsgJunkStatus newClassification = nsIJunkMailPlugin::UNCLASSIFIED; + uint32_t junkPercent = 0; // 0 here is no possibility of meeting the classification + uint32_t newLength = newClassifications.Length(); + for (uint32_t index = 0; index < newLength; index++) + { + uint32_t trait = newClassifications.ElementAt(index); + mCorpus.setMessageCount(trait, mCorpus.getMessageCount(trait) + 1); + mCorpus.rememberTokens(tokenizer, trait, 1); + mTrainingDataDirty = true; + + if (aJunkListener) + { + if (trait == kJunkTrait) + { + junkPercent = nsIJunkMailPlugin::IS_SPAM_SCORE; + newClassification = nsIJunkMailPlugin::JUNK; + } + else if (trait == kGoodTrait) + { + junkPercent = nsIJunkMailPlugin::IS_HAM_SCORE; + newClassification = nsIJunkMailPlugin::GOOD; + } + } + } + + if (aJunkListener) + aJunkListener->OnMessageClassified(messageURL, newClassification, junkPercent); + + if (aTraitListener) + { + // construct the outgoing listener arrays + AutoTArray<uint32_t, kTraitAutoCapacity> traits; + AutoTArray<uint32_t, kTraitAutoCapacity> percents; + uint32_t newLength = newClassifications.Length(); + if (newLength > kTraitAutoCapacity) + { + traits.SetCapacity(newLength); + percents.SetCapacity(newLength); + } + traits.AppendElements(newClassifications); + for (uint32_t index = 0; index < newLength; index++) + percents.AppendElement(100); // This is 100 percent, or certainty + aTraitListener->OnMessageTraitsClassified(messageURL, + traits.Length(), traits.Elements(), percents.Elements()); + } + + if (mTrainingDataDirty && !trainingDataWasDirty && ( mTimer != nullptr )) + { + // if training data became dirty just now, schedule flush + // mMinFlushInterval msec from now + MOZ_LOG( + BayesianFilterLogModule, LogLevel::Debug, + ("starting training data flush timer %i msec", mMinFlushInterval)); + mTimer->InitWithFuncCallback(nsBayesianFilter::TimerCallback, this, mMinFlushInterval, nsITimer::TYPE_ONE_SHOT); + } +} + +NS_IMETHODIMP nsBayesianFilter::GetUserHasClassified(bool *aResult) +{ + *aResult = ( (mCorpus.getMessageCount(kGoodTrait) + + mCorpus.getMessageCount(kJunkTrait)) && + mCorpus.countTokens()); + return NS_OK; +} + +// Set message classification (only allows junk and good) +NS_IMETHODIMP nsBayesianFilter::SetMessageClassification( + const char *aMsgURL, + nsMsgJunkStatus aOldClassification, + nsMsgJunkStatus aNewClassification, + nsIMsgWindow *aMsgWindow, + nsIJunkMailClassificationListener *aListener) +{ + AutoTArray<uint32_t, 1> oldClassifications; + AutoTArray<uint32_t, 1> newClassifications; + + // convert between classifications and trait + if (aOldClassification == nsIJunkMailPlugin::JUNK) + oldClassifications.AppendElement(kJunkTrait); + else if (aOldClassification == nsIJunkMailPlugin::GOOD) + oldClassifications.AppendElement(kGoodTrait); + if (aNewClassification == nsIJunkMailPlugin::JUNK) + newClassifications.AppendElement(kJunkTrait); + else if (aNewClassification == nsIJunkMailPlugin::GOOD) + newClassifications.AppendElement(kGoodTrait); + + MessageObserver* analyzer = new MessageObserver(this, oldClassifications, + newClassifications, aListener, nullptr); + NS_ENSURE_TRUE(analyzer, NS_ERROR_OUT_OF_MEMORY); + + TokenStreamListener *tokenListener = new TokenStreamListener(analyzer); + NS_ENSURE_TRUE(tokenListener, NS_ERROR_OUT_OF_MEMORY); + + analyzer->setTokenListener(tokenListener); + return tokenizeMessage(aMsgURL, aMsgWindow, analyzer); +} + +NS_IMETHODIMP nsBayesianFilter::ResetTrainingData() +{ + return mCorpus.resetTrainingData(); +} + +NS_IMETHODIMP nsBayesianFilter::DetailMessage(const char *aMsgURI, + uint32_t aProTrait, uint32_t aAntiTrait, + nsIMsgTraitDetailListener *aDetailListener, nsIMsgWindow *aMsgWindow) +{ + AutoTArray<uint32_t, 1> proTraits; + AutoTArray<uint32_t, 1> antiTraits; + proTraits.AppendElement(aProTrait); + antiTraits.AppendElement(aAntiTrait); + + MessageClassifier* analyzer = new MessageClassifier(this, nullptr, + nullptr, aDetailListener, proTraits, antiTraits, aMsgWindow, 1, &aMsgURI); + NS_ENSURE_TRUE(analyzer, NS_ERROR_OUT_OF_MEMORY); + + TokenStreamListener *tokenListener = new TokenStreamListener(analyzer); + NS_ENSURE_TRUE(tokenListener, NS_ERROR_OUT_OF_MEMORY); + + analyzer->setTokenListener(tokenListener); + return tokenizeMessage(aMsgURI, aMsgWindow, analyzer); +} + +// nsIMsgCorpus implementation + +NS_IMETHODIMP nsBayesianFilter::CorpusCounts(uint32_t aTrait, + uint32_t *aMessageCount, + uint32_t *aTokenCount) +{ + NS_ENSURE_ARG_POINTER(aTokenCount); + *aTokenCount = mCorpus.countTokens(); + if (aTrait && aMessageCount) + *aMessageCount = mCorpus.getMessageCount(aTrait); + return NS_OK; +} + +NS_IMETHODIMP nsBayesianFilter::ClearTrait(uint32_t aTrait) +{ + return mCorpus.ClearTrait(aTrait); +} + +NS_IMETHODIMP +nsBayesianFilter::UpdateData(nsIFile *aFile, + bool aIsAdd, + uint32_t aRemapCount, + uint32_t *aFromTraits, + uint32_t *aToTraits) +{ + return mCorpus.UpdateData(aFile, aIsAdd, aRemapCount, aFromTraits, aToTraits); +} + +NS_IMETHODIMP +nsBayesianFilter::GetTokenCount(const nsACString &aWord, + uint32_t aTrait, + uint32_t *aCount) +{ + NS_ENSURE_ARG_POINTER(aCount); + CorpusToken* t = mCorpus.get(PromiseFlatCString(aWord).get()); + uint32_t count = mCorpus.getTraitCount(t, aTrait); + *aCount = count; + return NS_OK; +} + +/* Corpus Store */ + +/* + Format of the training file for version 1: + [0xFEEDFACE] + [number good messages][number bad messages] + [number good tokens] + [count][length of word]word + ... + [number bad tokens] + [count][length of word]word + ... + + Format of the trait file for version 1: + [0xFCA93601] (the 01 is the version) + for each trait to write + [id of trait to write] (0 means end of list) + [number of messages per trait] + for each token with non-zero count + [count] + [length of word]word +*/ + +CorpusStore::CorpusStore() : + TokenHash(sizeof(CorpusToken)), + mNextTraitIndex(1) // skip 0 since index=0 will mean end of linked list +{ + getTrainingFile(getter_AddRefs(mTrainingFile)); + mTraitStore.SetCapacity(kTraitStoreCapacity); + TraitPerToken traitPT(0, 0); + mTraitStore.AppendElement(traitPT); // dummy 0th element +} + +CorpusStore::~CorpusStore() +{ +} + +inline int writeUInt32(FILE* stream, uint32_t value) +{ + value = PR_htonl(value); + return fwrite(&value, sizeof(uint32_t), 1, stream); +} + +inline int readUInt32(FILE* stream, uint32_t* value) +{ + int n = fread(value, sizeof(uint32_t), 1, stream); + if (n == 1) { + *value = PR_ntohl(*value); + } + return n; +} + +void CorpusStore::forgetTokens(Tokenizer& aTokenizer, + uint32_t aTraitId, uint32_t aCount) +{ + // if we are forgetting the tokens for a message, should only + // subtract 1 from the occurrence count for that token in the training set + // because we assume we only bumped the training set count once per messages + // containing the token. + TokenEnumeration tokens = aTokenizer.getTokens(); + while (tokens.hasMoreTokens()) + { + CorpusToken* token = static_cast<CorpusToken*>(tokens.nextToken()); + remove(token->mWord, aTraitId, aCount); + } +} + +void CorpusStore::rememberTokens(Tokenizer& aTokenizer, + uint32_t aTraitId, uint32_t aCount) +{ + TokenEnumeration tokens = aTokenizer.getTokens(); + while (tokens.hasMoreTokens()) + { + CorpusToken* token = static_cast<CorpusToken*>(tokens.nextToken()); + if (!token) + { + NS_ERROR("null token"); + continue; + } + add(token->mWord, aTraitId, aCount); + } +} + +bool CorpusStore::writeTokens(FILE* stream, bool shrink, uint32_t aTraitId) +{ + uint32_t tokenCount = countTokens(); + uint32_t newTokenCount = 0; + + // calculate the tokens for this trait to write + + TokenEnumeration tokens = getTokens(); + for (uint32_t i = 0; i < tokenCount; ++i) + { + CorpusToken* token = static_cast<CorpusToken*>(tokens.nextToken()); + uint32_t count = getTraitCount(token, aTraitId); + // Shrinking the token database is accomplished by dividing all token counts by 2. + // If shrinking, we'll ignore counts < 2, otherwise only ignore counts of < 1 + if ((shrink && count > 1) || (!shrink && count)) + newTokenCount++; + } + + if (writeUInt32(stream, newTokenCount) != 1) + return false; + + if (newTokenCount > 0) + { + TokenEnumeration tokens = getTokens(); + for (uint32_t i = 0; i < tokenCount; ++i) + { + CorpusToken* token = static_cast<CorpusToken*>(tokens.nextToken()); + uint32_t wordCount = getTraitCount(token, aTraitId); + if (shrink) + wordCount /= 2; + if (!wordCount) + continue; // Don't output zero count words + if (writeUInt32(stream, wordCount) != 1) + return false; + uint32_t tokenLength = strlen(token->mWord); + if (writeUInt32(stream, tokenLength) != 1) + return false; + if (fwrite(token->mWord, tokenLength, 1, stream) != 1) + return false; + } + } + return true; +} + +bool CorpusStore::readTokens(FILE* stream, int64_t fileSize, + uint32_t aTraitId, bool aIsAdd) +{ + uint32_t tokenCount; + if (readUInt32(stream, &tokenCount) != 1) + return false; + + int64_t fpos = ftell(stream); + if (fpos < 0) + return false; + + uint32_t bufferSize = 4096; + char* buffer = new char[bufferSize]; + if (!buffer) return false; + + for (uint32_t i = 0; i < tokenCount; ++i) { + uint32_t count; + if (readUInt32(stream, &count) != 1) + break; + uint32_t size; + if (readUInt32(stream, &size) != 1) + break; + fpos += 8; + if (fpos + size > fileSize) { + delete[] buffer; + return false; + } + if (size >= bufferSize) { + delete[] buffer; + while (size >= bufferSize) { + bufferSize *= 2; + if (bufferSize == 0) + return false; + } + buffer = new char[bufferSize]; + if (!buffer) return false; + } + if (fread(buffer, size, 1, stream) != 1) + break; + fpos += size; + buffer[size] = '\0'; + if (aIsAdd) + add(buffer, aTraitId, count); + else + remove(buffer, aTraitId, count); + } + + delete[] buffer; + + return true; +} + +nsresult CorpusStore::getTrainingFile(nsIFile ** aTrainingFile) +{ + // should we cache the profile manager's directory? + nsCOMPtr<nsIFile> profileDir; + + nsresult rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(profileDir)); + NS_ENSURE_SUCCESS(rv, rv); + rv = profileDir->Append(NS_LITERAL_STRING("training.dat")); + NS_ENSURE_SUCCESS(rv, rv); + + return profileDir->QueryInterface(NS_GET_IID(nsIFile), (void **) aTrainingFile); +} + +nsresult CorpusStore::getTraitFile(nsIFile ** aTraitFile) +{ + // should we cache the profile manager's directory? + nsCOMPtr<nsIFile> profileDir; + + nsresult rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(profileDir)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = profileDir->Append(NS_LITERAL_STRING("traits.dat")); + NS_ENSURE_SUCCESS(rv, rv); + + return profileDir->QueryInterface(NS_GET_IID(nsIFile), (void **) aTraitFile); +} + +static const char kMagicCookie[] = { '\xFE', '\xED', '\xFA', '\xCE' }; + +// random string used to identify trait file and version (last byte is version) +static const char kTraitCookie[] = { '\xFC', '\xA9', '\x36', '\x01' }; + +void CorpusStore::writeTrainingData(uint32_t aMaximumTokenCount) +{ + MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, ("writeTrainingData() entered")); + if (!mTrainingFile) + return; + + /* + * For backwards compatibility, write the good and junk tokens to + * training.dat; additional traits are added to a different file + */ + + // open the file, and write out training data + FILE* stream; + nsresult rv = mTrainingFile->OpenANSIFileDesc("wb", &stream); + if (NS_FAILED(rv)) + return; + + // If the number of tokens exceeds our limit, set the shrink flag + bool shrink = false; + if ((aMaximumTokenCount > 0) && // if 0, do not limit tokens + (countTokens() > aMaximumTokenCount)) + { + shrink = true; + MOZ_LOG(BayesianFilterLogModule, LogLevel::Warning, ("shrinking token data file")); + } + + // We implement shrink by dividing counts by two + uint32_t shrinkFactor = shrink ? 2 : 1; + + if (!((fwrite(kMagicCookie, sizeof(kMagicCookie), 1, stream) == 1) && + (writeUInt32(stream, getMessageCount(kGoodTrait) / shrinkFactor)) && + (writeUInt32(stream, getMessageCount(kJunkTrait) / shrinkFactor)) && + writeTokens(stream, shrink, kGoodTrait) && + writeTokens(stream, shrink, kJunkTrait))) + { + NS_WARNING("failed to write training data."); + fclose(stream); + // delete the training data file, since it is potentially corrupt. + mTrainingFile->Remove(false); + } + else + { + fclose(stream); + } + + /* + * Write the remaining data to a second file traits.dat + */ + + if (!mTraitFile) + { + getTraitFile(getter_AddRefs(mTraitFile)); + if (!mTraitFile) + return; + } + + // open the file, and write out training data + rv = mTraitFile->OpenANSIFileDesc("wb", &stream); + if (NS_FAILED(rv)) + return; + + uint32_t numberOfTraits = mMessageCounts.Length(); + bool error; + while (1) // break on error or done + { + if ((error = (fwrite(kTraitCookie, sizeof(kTraitCookie), 1, stream) != 1))) + break; + + for (uint32_t index = 0; index < numberOfTraits; index++) + { + uint32_t trait = mMessageCountsId[index]; + if (trait == 1 || trait == 2) + continue; // junk traits are stored in training.dat + if ((error = (writeUInt32(stream, trait) != 1))) + break; + if ((error = (writeUInt32(stream, mMessageCounts[index] / shrinkFactor) != 1))) + break; + if ((error = !writeTokens(stream, shrink, trait))) + break; + } + break; + } + // we add a 0 at the end to represent end of trait list + error = writeUInt32(stream, 0) != 1; + + fclose(stream); + if (error) + { + NS_WARNING("failed to write trait data."); + // delete the trait data file, since it is probably corrupt. + mTraitFile->Remove(false); + } + + if (shrink) + { + // We'll clear the tokens, and read them back in from the file. + // Yes this is slower than in place, but this is a rare event. + + if (countTokens()) + { + clearTokens(); + for (uint32_t index = 0; index < numberOfTraits; index++) + mMessageCounts[index] = 0; + } + + readTrainingData(); + } +} + +void CorpusStore::readTrainingData() +{ + + /* + * To maintain backwards compatibility, good and junk traits + * are stored in a file "training.dat" + */ + if (!mTrainingFile) + return; + + bool exists; + nsresult rv = mTrainingFile->Exists(&exists); + if (NS_FAILED(rv) || !exists) + return; + + FILE* stream; + rv = mTrainingFile->OpenANSIFileDesc("rb", &stream); + if (NS_FAILED(rv)) + return; + + int64_t fileSize; + rv = mTrainingFile->GetFileSize(&fileSize); + if (NS_FAILED(rv)) + return; + + // FIXME: should make sure that the tokenizers are empty. + char cookie[4]; + uint32_t goodMessageCount = 0, junkMessageCount = 0; + if (!((fread(cookie, sizeof(cookie), 1, stream) == 1) && + (memcmp(cookie, kMagicCookie, sizeof(cookie)) == 0) && + (readUInt32(stream, &goodMessageCount) == 1) && + (readUInt32(stream, &junkMessageCount) == 1) && + readTokens(stream, fileSize, kGoodTrait, true) && + readTokens(stream, fileSize, kJunkTrait, true))) { + NS_WARNING("failed to read training data."); + MOZ_LOG(BayesianFilterLogModule, LogLevel::Error, ("failed to read training data.")); + } + setMessageCount(kGoodTrait, goodMessageCount); + setMessageCount(kJunkTrait, junkMessageCount); + + fclose(stream); + + /* + * Additional traits are stored in traits.dat + */ + + if (!mTraitFile) + { + getTraitFile(getter_AddRefs(mTraitFile)); + if (!mTraitFile) + return; + } + + rv = mTraitFile->Exists(&exists); + if (NS_FAILED(rv) || !exists) + return; + + rv = UpdateData(mTraitFile, true, 0, nullptr, nullptr); + + if (NS_FAILED(rv)) + { + NS_WARNING("failed to read training data."); + MOZ_LOG(BayesianFilterLogModule, LogLevel::Error, ("failed to read training data.")); + } + return; +} + +nsresult CorpusStore::resetTrainingData() +{ + // clear out our in memory training tokens... + if (countTokens()) + clearTokens(); + + uint32_t length = mMessageCounts.Length(); + for (uint32_t index = 0 ; index < length; index++) + mMessageCounts[index] = 0; + + if (mTrainingFile) + mTrainingFile->Remove(false); + if (mTraitFile) + mTraitFile->Remove(false); + return NS_OK; +} + +inline CorpusToken* CorpusStore::get(const char* word) +{ + return static_cast<CorpusToken*>(TokenHash::get(word)); +} + +nsresult CorpusStore::updateTrait(CorpusToken* token, uint32_t aTraitId, + int32_t aCountChange) +{ + NS_ENSURE_ARG_POINTER(token); + uint32_t nextLink = token->mTraitLink; + uint32_t lastLink = 0; + + uint32_t linkCount, maxLinks = 100; //sanity check + for (linkCount = 0; nextLink && linkCount < maxLinks; linkCount++) + { + TraitPerToken& traitPT = mTraitStore[nextLink]; + if (traitPT.mId == aTraitId) + { + // be careful with signed versus unsigned issues here + if (static_cast<int32_t>(traitPT.mCount) + aCountChange > 0) + traitPT.mCount += aCountChange; + else + traitPT.mCount = 0; + // we could delete zero count traits here, but let's not. It's rare anyway. + return NS_OK; + } + lastLink = nextLink; + nextLink = traitPT.mNextLink; + } + if (linkCount >= maxLinks) + return NS_ERROR_FAILURE; + + // trait does not exist, so add it + + if (aCountChange > 0) // don't set a negative count + { + TraitPerToken traitPT(aTraitId, aCountChange); + if (mTraitStore.Length() == mNextTraitIndex) + mTraitStore.InsertElementAt(mNextTraitIndex, traitPT); + else if (mTraitStore.Length() > mNextTraitIndex) + mTraitStore.ReplaceElementsAt(mNextTraitIndex, 1, traitPT); + else + return NS_ERROR_FAILURE; + if (lastLink) + // the token had a parent, so update it + mTraitStore[lastLink].mNextLink = mNextTraitIndex; + else + // need to update the token's root link + token->mTraitLink = mNextTraitIndex; + mNextTraitIndex++; + } + return NS_OK; +} + +uint32_t CorpusStore::getTraitCount(CorpusToken* token, uint32_t aTraitId) +{ + uint32_t nextLink; + if (!token || !(nextLink = token->mTraitLink)) + return 0; + + uint32_t linkCount, maxLinks = 100; //sanity check + for (linkCount = 0; nextLink && linkCount < maxLinks; linkCount++) + { + TraitPerToken& traitPT = mTraitStore[nextLink]; + if (traitPT.mId == aTraitId) + return traitPT.mCount; + nextLink = traitPT.mNextLink; + } + NS_ASSERTION(linkCount < maxLinks, "Corrupt trait count store"); + + // trait not found (or error), so count is zero + return 0; +} + +CorpusToken* CorpusStore::add(const char* word, uint32_t aTraitId, uint32_t aCount) +{ + CorpusToken* token = static_cast<CorpusToken*>(TokenHash::add(word)); + if (token) { + MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, + ("adding word to corpus store: %s (Trait=%d) (deltaCount=%d)", + word, aTraitId, aCount)); + updateTrait(token, aTraitId, aCount); + } + return token; + } + +void CorpusStore::remove(const char* word, uint32_t aTraitId, uint32_t aCount) +{ + MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, + ("remove word: %s (TraitId=%d) (Count=%d)", + word, aTraitId, aCount)); + CorpusToken* token = get(word); + if (token) + updateTrait(token, aTraitId, -static_cast<int32_t>(aCount)); +} + +uint32_t CorpusStore::getMessageCount(uint32_t aTraitId) +{ + size_t index = mMessageCountsId.IndexOf(aTraitId); + if (index == mMessageCountsId.NoIndex) + return 0; + return mMessageCounts.ElementAt(index); +} + +void CorpusStore::setMessageCount(uint32_t aTraitId, uint32_t aCount) +{ + size_t index = mMessageCountsId.IndexOf(aTraitId); + if (index == mMessageCountsId.NoIndex) + { + mMessageCounts.AppendElement(aCount); + mMessageCountsId.AppendElement(aTraitId); + } + else + { + mMessageCounts[index] = aCount; + } +} + +nsresult +CorpusStore::UpdateData(nsIFile *aFile, + bool aIsAdd, + uint32_t aRemapCount, + uint32_t *aFromTraits, + uint32_t *aToTraits) +{ + NS_ENSURE_ARG_POINTER(aFile); + if (aRemapCount) + { + NS_ENSURE_ARG_POINTER(aFromTraits); + NS_ENSURE_ARG_POINTER(aToTraits); + } + + int64_t fileSize; + nsresult rv = aFile->GetFileSize(&fileSize); + NS_ENSURE_SUCCESS(rv, rv); + + FILE* stream; + rv = aFile->OpenANSIFileDesc("rb", &stream); + NS_ENSURE_SUCCESS(rv, rv); + + bool error; + do // break on error or done + { + char cookie[4]; + if ((error = (fread(cookie, sizeof(cookie), 1, stream) != 1))) + break; + + if ((error = memcmp(cookie, kTraitCookie, sizeof(cookie)))) + break; + + uint32_t fileTrait; + while ( !(error = (readUInt32(stream, &fileTrait) != 1)) && fileTrait) + { + uint32_t count; + if ((error = (readUInt32(stream, &count) != 1))) + break; + + uint32_t localTrait = fileTrait; + // remap the trait + for (uint32_t i = 0; i < aRemapCount; i++) + { + if (aFromTraits[i] == fileTrait) + localTrait = aToTraits[i]; + } + + uint32_t messageCount = getMessageCount(localTrait); + if (aIsAdd) + messageCount += count; + else if (count > messageCount) + messageCount = 0; + else + messageCount -= count; + setMessageCount(localTrait, messageCount); + + if ((error = !readTokens(stream, fileSize, localTrait, aIsAdd))) + break; + } + break; + } while (0); + + fclose(stream); + + if (error) + return NS_ERROR_FAILURE; + return NS_OK; +} + +nsresult CorpusStore::ClearTrait(uint32_t aTrait) +{ + // clear message counts + setMessageCount(aTrait, 0); + + TokenEnumeration tokens = getTokens(); + while (tokens.hasMoreTokens()) + { + CorpusToken* token = static_cast<CorpusToken*>(tokens.nextToken()); + int32_t wordCount = static_cast<int32_t>(getTraitCount(token, aTrait)); + updateTrait(token, aTrait, -wordCount); + } + return NS_OK; +} diff --git a/mailnews/extensions/bayesian-spam-filter/src/nsBayesianFilter.h b/mailnews/extensions/bayesian-spam-filter/src/nsBayesianFilter.h new file mode 100644 index 000000000..32a9d26d0 --- /dev/null +++ b/mailnews/extensions/bayesian-spam-filter/src/nsBayesianFilter.h @@ -0,0 +1,404 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef nsBayesianFilter_h__ +#define nsBayesianFilter_h__ + +#include <stdio.h> +#include "nsCOMPtr.h" +#include "nsIMsgFilterPlugin.h" +#include "nsISemanticUnitScanner.h" +#include "PLDHashTable.h" +#include "nsITimer.h" +#include "nsTArray.h" +#include "nsStringGlue.h" +#include "nsWeakReference.h" +#include "nsIObserver.h" + +// XXX can't simply byte align arenas, must at least 2-byte align. +#define PL_ARENA_CONST_ALIGN_MASK 1 +#include "plarena.h" + +#define DEFAULT_MIN_INTERVAL_BETWEEN_WRITES 15*60*1000 + +struct Token; +class TokenEnumeration; +class TokenAnalyzer; +class nsIMsgWindow; +class nsIMimeHeaders; +class nsIUTF8StringEnumerator; +struct BaseToken; +struct CorpusToken; + +/** + * Helper class to enumerate Token objects in a PLDHashTable + * safely and without copying (see bugzilla #174859). The + * enumeration is safe to use until an Add() + * or Remove() is performed on the table. + */ +class TokenEnumeration { +public: + TokenEnumeration(PLDHashTable* table); + bool hasMoreTokens(); + BaseToken* nextToken(); + +private: + PLDHashTable::Iterator mIterator; +}; + +// A trait is some aspect of a message, like being junk or tagged as +// Personal, that the statistical classifier should track. The Trait +// structure is a per-token representation of information pertaining to +// a message trait. +// +// Traits per token are maintained as a linked list. +// +struct TraitPerToken +{ + uint32_t mId; // identifying number for a trait + uint32_t mCount; // count of messages with this token and trait + uint32_t mNextLink; // index in mTraitStore for the next trait, or 0 + // for none + TraitPerToken(uint32_t aId, uint32_t aCount); // inititializer +}; + +// An Analysis is the statistical results for a particular message, a +// particular token, and for a particular pair of trait/antitrait, that +// is then used in subsequent analysis to score the message. +// +// Analyses per token are maintained as a linked list. +// +struct AnalysisPerToken +{ + uint32_t mTraitIndex; // index representing a protrait/antitrait pair. + // So if we are analyzing 3 different traits, then + // the first trait is 0, the second 1, etc. + double mDistance; // absolute value of mProbability - 0.5 + double mProbability; // relative indicator of match of trait to token + uint32_t mNextLink; // index in mAnalysisStore for the Analysis object + // for the next trait index, or 0 for none. + // initializer + AnalysisPerToken(uint32_t aTraitIndex, double aDistance, double aProbability); +}; + +class TokenHash { +public: + + virtual ~TokenHash(); + /** + * Clears out the previous message tokens. + */ + nsresult clearTokens(); + uint32_t countTokens(); + TokenEnumeration getTokens(); + BaseToken* add(const char* word); + +protected: + TokenHash(uint32_t entrySize); + PLArenaPool mWordPool; + uint32_t mEntrySize; + PLDHashTable mTokenTable; + char* copyWord(const char* word, uint32_t len); + BaseToken* get(const char* word); +}; + +class Tokenizer: public TokenHash { +public: + Tokenizer(); + ~Tokenizer(); + + Token* get(const char* word); + + // The training set keeps an occurrence count on each word. This count + // is supposed to count the # of messsages it occurs in. + // When add/remove is called while tokenizing a message and NOT the training set, + // + Token* add(const char* word, uint32_t count = 1); + + Token* copyTokens(); + + void tokenize(const char* text); + + /** + * Creates specific tokens based on the mime headers for the message being tokenized + */ + void tokenizeHeaders(nsIUTF8StringEnumerator * aHeaderNames, nsIUTF8StringEnumerator * aHeaderValues); + + void tokenizeAttachment(const char * aContentType, const char * aFileName); + + nsCString mBodyDelimiters; // delimiters for body tokenization + nsCString mHeaderDelimiters; // delimiters for header tokenization + + // arrays of extra headers to tokenize / to not tokenize + nsTArray<nsCString> mEnabledHeaders; + nsTArray<nsCString> mDisabledHeaders; + // Delimiters used in tokenizing a particular header. + // Parallel array to mEnabledHeaders + nsTArray<nsCString> mEnabledHeadersDelimiters; + bool mCustomHeaderTokenization; // Are there any preference-set tokenization customizations? + uint32_t mMaxLengthForToken; // maximum length of a token + // should we convert iframe to div during tokenization? + bool mIframeToDiv; + +private: + + void tokenize_ascii_word(char * word); + void tokenize_japanese_word(char* chunk); + inline void addTokenForHeader(const char * aTokenPrefix, nsACString& aValue, + bool aTokenizeValue = false, const char* aDelimiters = nullptr); + nsresult stripHTML(const nsAString& inString, nsAString& outString); + // helper function to escape \n, \t, etc from a CString + void UnescapeCString(nsCString& aCString); + +private: + nsCOMPtr<nsISemanticUnitScanner> mScanner; +}; + +/** + * Implements storage of a collection of message tokens and counts for + * a corpus of classified messages + */ + +class CorpusStore: public TokenHash { +public: + CorpusStore(); + ~CorpusStore(); + + /** + * retrieve the token structure for a particular string + * + * @param word the character representation of the token + * + * @return token structure containing counts, null if not found + */ + CorpusToken* get(const char* word); + + /** + * add tokens to the storage, or increment counts if already exists. + * + * @param aTokenizer tokenizer for the list of tokens to remember + * @param aTraitId id for the trait whose counts will be remembered + * @param aCount number of new messages represented by the token list + */ + void rememberTokens(Tokenizer& aTokenizer, uint32_t aTraitId, uint32_t aCount); + + /** + * decrement counts for tokens in the storage, removing if all counts + * are zero + * + * @param aTokenizer tokenizer for the list of tokens to forget + * @param aTraitId id for the trait whose counts will be removed + * @param aCount number of messages represented by the token list + */ + void forgetTokens(Tokenizer& aTokenizer, uint32_t aTraitId, uint32_t aCount); + + /** + * write the corpus information to file storage + * + * @param aMaximumTokenCount prune tokens if number of tokens exceeds + * this value. == 0 for no pruning + */ + void writeTrainingData(uint32_t aMaximumTokenCount); + + /** + * read the corpus information from file storage + */ + void readTrainingData(); + + /** + * delete the local corpus storage file and data + */ + nsresult resetTrainingData(); + + /** + * get the count of messages whose tokens are stored that are associated + * with a trait + * + * @param aTraitId identifier for the trait + * @return number of messages for that trait + */ + uint32_t getMessageCount(uint32_t aTraitId); + + /** + * set the count of messages whose tokens are stored that are associated + * with a trait + * + * @param aTraitId identifier for the trait + * @param aCount number of messages for that trait + */ + void setMessageCount(uint32_t aTraitId, uint32_t aCount); + + /** + * get the count of messages associated with a particular token and trait + * + * @param token the token string and associated counts + * @param aTraitId identifier for the trait + */ + uint32_t getTraitCount(CorpusToken *token, uint32_t aTraitId); + + /** + * Add (or remove) data from a particular file to the corpus data. + * + * @param aFile the file with the data, in the format: + * + * Format of the trait file for version 1: + * [0xFCA93601] (the 01 is the version) + * for each trait to write: + * [id of trait to write] (0 means end of list) + * [number of messages per trait] + * for each token with non-zero count + * [count] + * [length of word]word + * + * @param aIsAdd should the data be added, or removed? true if adding, + * else removing. + * + * @param aRemapCount number of items in the parallel arrays aFromTraits, + * aToTraits. These arrays allow conversion of the + * trait id stored in the file (which may be originated + * externally) to the trait id used in the local corpus + * (which is defined locally using nsIMsgTraitService). + * + * @param aFromTraits array of trait ids used in aFile. If aFile contains + * trait ids that are not in this array, they are not + * remapped, but assummed to be local trait ids. + * + * @param aToTraits array of trait ids, corresponding to elements of + * aFromTraits, that represent the local trait ids to be + * used in storing data from aFile into the local corpus. + * + */ + nsresult UpdateData(nsIFile *aFile, bool aIsAdd, + uint32_t aRemapCount, uint32_t *aFromTraits, + uint32_t *aToTraits); + + /** + * remove all counts (message and tokens) for a trait id + * + * @param aTrait trait id for the trait to remove + */ + nsresult ClearTrait(uint32_t aTrait); + +protected: + + /** + * return the local corpus storage file for junk traits + */ + nsresult getTrainingFile(nsIFile ** aFile); + + /** + * return the local corpus storage file for non-junk traits + */ + nsresult getTraitFile(nsIFile ** aFile); + + /** + * read token strings from the data file + * + * @param stream file stream with token data + * @param fileSize file size + * @param aTraitId id for the trait whose counts will be read + * @param aIsAdd true to add the counts, false to remove them + * + * @return true if successful, false if error + */ + bool readTokens(FILE* stream, int64_t fileSize, uint32_t aTraitId, + bool aIsAdd); + + /** + * write token strings to the data file + */ + bool writeTokens(FILE* stream, bool shrink, uint32_t aTraitId); + + /** + * remove counts for a token string + */ + void remove(const char* word, uint32_t aTraitId, uint32_t aCount); + + /** + * add counts for a token string, adding the token string if new + */ + CorpusToken* add(const char* word, uint32_t aTraitId, uint32_t aCount); + + /** + * change counts in a trait in the traits array, adding the trait if needed + */ + nsresult updateTrait(CorpusToken* token, uint32_t aTraitId, + int32_t aCountChange); + nsCOMPtr<nsIFile> mTrainingFile; // file used to store junk training data + nsCOMPtr<nsIFile> mTraitFile; // file used to store non-junk + // training data + nsTArray<TraitPerToken> mTraitStore; // memory for linked-list of counts + uint32_t mNextTraitIndex; // index in mTraitStore to first empty + // TraitPerToken + nsTArray<uint32_t> mMessageCounts; // count of messages per trait + // represented in the store + nsTArray<uint32_t> mMessageCountsId; // Parallel array to mMessageCounts, with + // the corresponding trait ID +}; + +class nsBayesianFilter : public nsIJunkMailPlugin, nsIMsgCorpus, + nsIObserver, nsSupportsWeakReference { +public: + NS_DECL_ISUPPORTS + NS_DECL_NSIMSGFILTERPLUGIN + NS_DECL_NSIJUNKMAILPLUGIN + NS_DECL_NSIMSGCORPUS + NS_DECL_NSIOBSERVER + + nsBayesianFilter(); + + nsresult Init(); + + nsresult tokenizeMessage(const char* messageURI, nsIMsgWindow *aMsgWindow, TokenAnalyzer* analyzer); + void classifyMessage(Tokenizer& tokens, const char* messageURI, + nsIJunkMailClassificationListener* listener); + + void classifyMessage( + Tokenizer& tokenizer, + const char* messageURI, + nsTArray<uint32_t>& aProTraits, + nsTArray<uint32_t>& aAntiTraits, + nsIJunkMailClassificationListener* listener, + nsIMsgTraitClassificationListener* aTraitListener, + nsIMsgTraitDetailListener* aDetailListener); + + void observeMessage(Tokenizer& tokens, const char* messageURI, + nsTArray<uint32_t>& oldClassifications, + nsTArray<uint32_t>& newClassifications, + nsIJunkMailClassificationListener* listener, + nsIMsgTraitClassificationListener* aTraitListener); + + +protected: + virtual ~nsBayesianFilter(); + + static void TimerCallback(nsITimer* aTimer, void* aClosure); + + CorpusStore mCorpus; + double mJunkProbabilityThreshold; + int32_t mMaximumTokenCount; + bool mTrainingDataDirty; + int32_t mMinFlushInterval; // in milliseconds, must be positive + //and not too close to 0 + nsCOMPtr<nsITimer> mTimer; + + // index in mAnalysisStore for first empty AnalysisPerToken + uint32_t mNextAnalysisIndex; + // memory for linked list of AnalysisPerToken objects + nsTArray<AnalysisPerToken> mAnalysisStore; + /** + * Determine the location in mAnalysisStore where the AnalysisPerToken + * object for a particular token and trait is stored + */ + uint32_t getAnalysisIndex(Token& token, uint32_t aTraitIndex); + /** + * Set the value of the AnalysisPerToken object for a particular + * token and trait + */ + nsresult setAnalysis(Token& token, uint32_t aTraitIndex, + double aDistance, double aProbability); +}; + +#endif // _nsBayesianFilter_h__ diff --git a/mailnews/extensions/bayesian-spam-filter/src/nsBayesianFilterCID.h b/mailnews/extensions/bayesian-spam-filter/src/nsBayesianFilterCID.h new file mode 100644 index 000000000..b1a54a1fc --- /dev/null +++ b/mailnews/extensions/bayesian-spam-filter/src/nsBayesianFilterCID.h @@ -0,0 +1,22 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef nsBayesianFilterCID_h__ +#define nsBayesianFilterCID_h__ + +#include "nsISupports.h" +#include "nsIFactory.h" +#include "nsIComponentManager.h" + +#include "nsIMsgMdnGenerator.h" + +#define NS_BAYESIANFILTER_CONTRACTID \ + "@mozilla.org/messenger/filter-plugin;1?name=bayesianfilter" +#define NS_BAYESIANFILTER_CID \ +{ /* F1070BFA-D539-11D6-90CA-00039310A47A */ \ + 0xF1070BFA, 0xD539, 0x11D6, \ + { 0x90, 0xCA, 0x00, 0x03, 0x93, 0x10, 0xA4, 0x7A }} + +#endif /* nsBayesianFilterCID_h__ */ diff --git a/mailnews/extensions/bayesian-spam-filter/src/nsIncompleteGamma.h b/mailnews/extensions/bayesian-spam-filter/src/nsIncompleteGamma.h new file mode 100644 index 000000000..1f7388b16 --- /dev/null +++ b/mailnews/extensions/bayesian-spam-filter/src/nsIncompleteGamma.h @@ -0,0 +1,259 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef nsIncompleteGamma_h__ +#define nsIncompleteGamma_h__ + +/* An implementation of the incomplete gamma functions for real + arguments. P is defined as + + x + / + 1 [ a - 1 - t + P(a, x) = -------- I t e dt + Gamma(a) ] + / + 0 + + and + + infinity + / + 1 [ a - 1 - t + Q(a, x) = -------- I t e dt + Gamma(a) ] + / + x + + so that P(a,x) + Q(a,x) = 1. + + Both a series expansion and a continued fraction exist. This + implementation uses the more efficient method based on the arguments. + + Either case involves calculating a multiplicative term: + e^(-x)*x^a/Gamma(a). + Here we calculate the log of this term. Most math libraries have a + "lgamma" function but it is not re-entrant. Some libraries have a + "lgamma_r" which is re-entrant. Use it if possible. I have included a + simple replacement but it is certainly not as accurate. + + Relative errors are almost always < 1e-10 and usually < 1e-14. Very + small and very large arguments cause trouble. + + The region where a < 0.5 and x < 0.5 has poor error properties and is + not too stable. Get a better routine if you need results in this + region. + + The error argument will be set negative if there is a domain error or + positive for an internal calculation error, currently lack of + convergence. A value is always returned, though. + + */ + +#include <math.h> +#include <float.h> + +// the main routine +static double nsIncompleteGammaP (double a, double x, int *error); + +// nsLnGamma(z): either a wrapper around lgamma_r or the internal function. +// C_m = B[2*m]/(2*m*(2*m-1)) where B is a Bernoulli number +static const double C_1 = 1.0 / 12.0; +static const double C_2 = -1.0 / 360.0; +static const double C_3 = 1.0 / 1260.0; +static const double C_4 = -1.0 / 1680.0; +static const double C_5 = 1.0 / 1188.0; +static const double C_6 = -691.0 / 360360.0; +static const double C_7 = 1.0 / 156.0; +static const double C_8 = -3617.0 / 122400.0; +static const double C_9 = 43867.0 / 244188.0; +static const double C_10 = -174611.0 / 125400.0; +static const double C_11 = 77683.0 / 5796.0; + +// truncated asymptotic series in 1/z +static inline double lngamma_asymp (double z) +{ + double w, w2, sum; + w = 1.0 / z; + w2 = w * w; + sum = w * (w2 * (w2 * (w2 * (w2 * (w2 * (w2 * (w2 * (w2 * (w2 + * (C_11 * w2 + C_10) + C_9) + C_8) + C_7) + C_6) + + C_5) + C_4) + C_3) + C_2) + C_1); + + return sum; +} + +struct fact_table_s +{ + double fact; + double lnfact; +}; + +// for speed and accuracy +static const struct fact_table_s FactTable[] = { + {1.000000000000000, 0.0000000000000000000000e+00}, + {1.000000000000000, 0.0000000000000000000000e+00}, + {2.000000000000000, 6.9314718055994530942869e-01}, + {6.000000000000000, 1.7917594692280550007892e+00}, + {24.00000000000000, 3.1780538303479456197550e+00}, + {120.0000000000000, 4.7874917427820459941458e+00}, + {720.0000000000000, 6.5792512120101009952602e+00}, + {5040.000000000000, 8.5251613610654142999881e+00}, + {40320.00000000000, 1.0604602902745250228925e+01}, + {362880.0000000000, 1.2801827480081469610995e+01}, + {3628800.000000000, 1.5104412573075515295248e+01}, + {39916800.00000000, 1.7502307845873885839769e+01}, + {479001600.0000000, 1.9987214495661886149228e+01}, + {6227020800.000000, 2.2552163853123422886104e+01}, + {87178291200.00000, 2.5191221182738681499610e+01}, + {1307674368000.000, 2.7899271383840891566988e+01}, + {20922789888000.00, 3.0671860106080672803835e+01}, + {355687428096000.0, 3.3505073450136888885825e+01}, + {6402373705728000., 3.6395445208033053576674e+01} +}; +#define FactTableLength (int)(sizeof(FactTable)/sizeof(FactTable[0])) + +// for speed +static const double ln_2pi_2 = 0.918938533204672741803; // log(2*PI)/2 + +/* A simple lgamma function, not very robust. + + Valid for z_in > 0 ONLY. + + For z_in > 8 precision is quite good, relative errors < 1e-14 and + usually better. For z_in < 8 relative errors increase but are usually + < 1e-10. In two small regions, 1 +/- .001 and 2 +/- .001 errors + increase quickly. +*/ +static double nsLnGamma (double z_in, int *gsign) +{ + double scale, z, sum, result; + *gsign = 1; + + int zi = (int) z_in; + if (z_in == (double) zi) + { + if (0 < zi && zi <= FactTableLength) + return FactTable[zi - 1].lnfact; // gamma(z) = (z-1)! + } + + for (scale = 1.0, z = z_in; z < 8.0; ++z) + scale *= z; + + sum = lngamma_asymp (z); + result = (z - 0.5) * log (z) - z + ln_2pi_2 - log (scale); + result += sum; + return result; +} + +// log( e^(-x)*x^a/Gamma(a) ) +static inline double lnPQfactor (double a, double x) +{ + int gsign; // ignored because a > 0 + return a * log (x) - x - nsLnGamma (a, &gsign); +} + +static double Pseries (double a, double x, int *error) +{ + double sum, term; + const double eps = 2.0 * DBL_EPSILON; + const int imax = 5000; + int i; + + sum = term = 1.0 / a; + for (i = 1; i < imax; ++i) + { + term *= x / (a + i); + sum += term; + if (fabs (term) < eps * fabs (sum)) + break; + } + + if (i >= imax) + *error = 1; + + return sum; +} + +static double Qcontfrac (double a, double x, int *error) +{ + double result, D, C, e, f, term; + const double eps = 2.0 * DBL_EPSILON; + const double small = + DBL_EPSILON * DBL_EPSILON * DBL_EPSILON * DBL_EPSILON; + const int imax = 5000; + int i; + + // modified Lentz method + f = x - a + 1.0; + if (fabs (f) < small) + f = small; + C = f + 1.0 / small; + D = 1.0 / f; + result = D; + for (i = 1; i < imax; ++i) + { + e = i * (a - i); + f += 2.0; + D = f + e * D; + if (fabs (D) < small) + D = small; + D = 1.0 / D; + C = f + e / C; + if (fabs (C) < small) + C = small; + term = C * D; + result *= term; + if (fabs (term - 1.0) < eps) + break; + } + + if (i >= imax) + *error = 1; + return result; +} + +static double nsIncompleteGammaP (double a, double x, int *error) +{ + double result, dom, ldom; + // domain errors. the return values are meaningless but have + // to return something. + *error = -1; + if (a <= 0.0) + return 1.0; + if (x < 0.0) + return 0.0; + *error = 0; + if (x == 0.0) + return 0.0; + + ldom = lnPQfactor (a, x); + dom = exp (ldom); + // might need to adjust the crossover point + if (a <= 0.5) + { + if (x < a + 1.0) + result = dom * Pseries (a, x, error); + else + result = 1.0 - dom * Qcontfrac (a, x, error); + } + else + { + if (x < a) + result = dom * Pseries (a, x, error); + else + result = 1.0 - dom * Qcontfrac (a, x, error); + } + + // not clear if this can ever happen + if (result > 1.0) + result = 1.0; + if (result < 0.0) + result = 0.0; + return result; +} + +#endif + diff --git a/mailnews/extensions/dsn/content/am-dsn.js b/mailnews/extensions/dsn/content/am-dsn.js new file mode 100644 index 000000000..2c8a5f923 --- /dev/null +++ b/mailnews/extensions/dsn/content/am-dsn.js @@ -0,0 +1,36 @@ +/* -*- Mode: Java; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +var useCustomPrefs; +var requestAlways; +var gIdentity; + +function onInit() +{ + useCustomPrefs = document.getElementById("identity.dsn_use_custom_prefs"); + requestAlways = document.getElementById("identity.dsn_always_request_on"); + + EnableDisableCustomSettings(); + + return true; +} + +function onSave() +{ +} + +function EnableDisableCustomSettings() { + if (useCustomPrefs && (useCustomPrefs.getAttribute("value") == "false")) + requestAlways.setAttribute("disabled", "true"); + else + requestAlways.removeAttribute("disabled"); + + return true; +} + +function onPreInit(account, accountValues) +{ + gIdentity = account.defaultIdentity; +} diff --git a/mailnews/extensions/dsn/content/am-dsn.xul b/mailnews/extensions/dsn/content/am-dsn.xul new file mode 100644 index 000000000..1327021ca --- /dev/null +++ b/mailnews/extensions/dsn/content/am-dsn.xul @@ -0,0 +1,57 @@ +<?xml version="1.0"?> + +<!-- + + This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> + +<?xml-stylesheet href="chrome://messenger/skin/accountManage.css" type="text/css"?> + +<!DOCTYPE page SYSTEM "chrome://messenger/locale/am-dsn.dtd"> + +<page xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul" + onload="parent.onPanelLoaded('am-dsn.xul');"> + + <stringbundle id="bundle_smime" src="chrome://messenger/locale/am-dsn.properties"/> + <script type="application/javascript" src="chrome://messenger/content/AccountManager.js"/> + <script type="application/javascript" src="chrome://messenger/content/am-dsn.js"/> + + <dialogheader title="&pane.title;"/> + + <groupbox> + + <caption label="&pane.title;"/> + + <hbox id="prefChoices" align="center"> + <radiogroup id="identity.dsn_use_custom_prefs" + wsm_persist="true" + genericattr="true" + preftype="bool" + prefstring="mail.identity.%identitykey%.dsn_use_custom_prefs" + oncommand="EnableDisableCustomSettings();"> + + <radio id="identity.select_global_prefs" + value="false" + label="&useGlobalPrefs.label;" + accesskey="&useGlobalPrefs.accesskey;"/> + + <radio id="identity.select_custom_prefs" + value="true" + label="&useCustomPrefs.label;" + accesskey="&useCustomPrefs.accesskey;"/> + </radiogroup> + </hbox> + + <vbox id="dsnSettings" class="indent" align="start"> + <checkbox id="identity.dsn_always_request_on" + label="&requestAlways.label;" + accesskey="&requestAlways.accesskey;" + wsm_persist="true" + genericattr="true" + iscontrolcontainer="true" + preftype="bool" + prefstring="mail.identity.%identitykey%.dsn_always_request_on"/> + </vbox> + </groupbox> +</page> diff --git a/mailnews/extensions/dsn/content/dsn.js b/mailnews/extensions/dsn/content/dsn.js new file mode 100644 index 000000000..043aa1b94 --- /dev/null +++ b/mailnews/extensions/dsn/content/dsn.js @@ -0,0 +1,9 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * default prefs for dsn + */ +pref("mail.identity.default.dsn_use_custom_prefs", false); // false: Use global true: Use custom +pref("mail.identity.default.dsn_always_request_on", false); diff --git a/mailnews/extensions/dsn/jar.mn b/mailnews/extensions/dsn/jar.mn new file mode 100644 index 000000000..2ecdd7057 --- /dev/null +++ b/mailnews/extensions/dsn/jar.mn @@ -0,0 +1,9 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifdef MOZ_SUITE +messenger.jar: + content/messenger/am-dsn.xul (content/am-dsn.xul) + content/messenger/am-dsn.js (content/am-dsn.js) +#endif diff --git a/mailnews/extensions/dsn/moz.build b/mailnews/extensions/dsn/moz.build new file mode 100644 index 000000000..10cc8cb56 --- /dev/null +++ b/mailnews/extensions/dsn/moz.build @@ -0,0 +1,15 @@ +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +EXTRA_COMPONENTS += [ + 'src/dsn-service.js', + 'src/dsn-service.manifest', +] + +JAR_MANIFESTS += ['jar.mn'] + +JS_PREFERENCE_FILES += [ + 'content/dsn.js', +]
\ No newline at end of file diff --git a/mailnews/extensions/dsn/src/dsn-service.js b/mailnews/extensions/dsn/src/dsn-service.js new file mode 100644 index 000000000..76ceeb04b --- /dev/null +++ b/mailnews/extensions/dsn/src/dsn-service.js @@ -0,0 +1,24 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +Components.utils.import("resource://gre/modules/XPCOMUtils.jsm"); + +function DSNService() {} + +DSNService.prototype = { + name: "dsn", + chromePackageName: "messenger", + showPanel: function(server) { + // don't show the panel for news, rss, or local accounts + return (server.type != "nntp" && server.type != "rss" && + server.type != "none"); + }, + + QueryInterface: XPCOMUtils.generateQI([Components.interfaces.nsIMsgAccountManagerExtension]), + classID: Components.ID("{849dab91-9bc9-4508-a0ee-c2453e7c092d}"), +}; + +var components = [DSNService]; +var NSGetFactory = XPCOMUtils.generateNSGetFactory(components); diff --git a/mailnews/extensions/dsn/src/dsn-service.manifest b/mailnews/extensions/dsn/src/dsn-service.manifest new file mode 100644 index 000000000..2853fb3e8 --- /dev/null +++ b/mailnews/extensions/dsn/src/dsn-service.manifest @@ -0,0 +1,3 @@ +component {849dab91-9bc9-4508-a0ee-c2453e7c092d} dsn-service.js +contract @mozilla.org/accountmanager/extension;1?name=dsn {849dab91-9bc9-4508-a0ee-c2453e7c092d} +category mailnews-accountmanager-extensions dsn-account-manager-extension @mozilla.org/accountmanager/extension;1?name=dsn diff --git a/mailnews/extensions/fts3/data/README b/mailnews/extensions/fts3/data/README new file mode 100644 index 000000000..d7c13abbb --- /dev/null +++ b/mailnews/extensions/fts3/data/README @@ -0,0 +1,5 @@ +The data files in this directory come from the ICU project: +http://bugs.icu-project.org/trac/browser/icu/trunk/source/data/unidata/norm2 + +They are intended to be consumed by the ICU project's gennorm2 script. We have +our own script that processes them. diff --git a/mailnews/extensions/fts3/data/generate_table.py b/mailnews/extensions/fts3/data/generate_table.py new file mode 100644 index 000000000..f6b012685 --- /dev/null +++ b/mailnews/extensions/fts3/data/generate_table.py @@ -0,0 +1,264 @@ +#!/usr/bin/python +# ***** BEGIN LICENSE BLOCK ***** +# Version: MPL 1.1/GPL 2.0/LGPL 2.1 +# +# The contents of this file are subject to the Mozilla Public License Version +# 1.1 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +# for the specific language governing rights and limitations under the +# License. +# +# The Original Code is Mozilla Thunderbird. +# +# The Initial Developer of the Original Code is Mozilla Japan. +# Portions created by the Initial Developer are Copyright (C) 2010 +# the Initial Developer. All Rights Reserved. +# +# Contributor(s): +# Makoto Kato <m_kato@ga2.so-net.ne.jp> +# Andrew Sutherland <asutherland@asutherland.org> +# +# Alternatively, the contents of this file may be used under the terms of +# either the GNU General Public License Version 2 or later (the "GPL"), or +# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), +# in which case the provisions of the GPL or the LGPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of either the GPL or the LGPL, and not to allow others to +# use your version of this file under the terms of the MPL, indicate your +# decision by deleting the provisions above and replace them with the notice +# and other provisions required by the GPL or the LGPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the MPL, the GPL or the LGPL. +# +# ***** END LICENSE BLOCK ***** + +import re + +def printTable(f, t): + i = f + while i <= t: + c = array[i] + print "0x%04x," % c, + i = i + 1 + if not i % 8: + print "\n\t", + +print '''/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* ***** BEGIN LICENSE BLOCK ***** + * Version: MPL 1.1/GPL 2.0/LGPL 2.1 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is mozilla.org code. + * + * The Initial Developer of the Original Code is Mozilla Japan. + * Portions created by the Initial Developer are Copyright (C) 2010 + * the Initial Developer. All Rights Reserved. + * + * Contributor(s): + * Makoto Kato <m_kato@ga2.so-net.ne.jp> + * Andrew Sutherland <asutherland@asutherland.org> + * + * Alternatively, the contents of this file may be used under the terms of + * either of the GNU General Public License Version 2 or later (the "GPL"), + * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), + * in which case the provisions of the GPL or the LGPL are applicable instead + * of those above. If you wish to allow use of your version of this file only + * under the terms of either the GPL or the LGPL, and not to allow others to + * use your version of this file under the terms of the MPL, indicate your + * decision by deleting the provisions above and replace them with the notice + * and other provisions required by the GPL or the LGPL. If you do not delete + * the provisions above, a recipient may use your version of this file under + * the terms of any one of the MPL, the GPL or the LGPL. + * + * ***** END LICENSE BLOCK ***** */ + +/* THIS FILE IS GENERATED BY generate_table.py. DON'T EDIT THIS */ +''' + +p = re.compile('([0-9A-F]{4,5})(?:\.\.([0-9A-F]{4,5}))?[=\>]([0-9A-F]{4,5})?') +G_FROM = 1 +G_TO = 2 +G_FIRSTVAL = 3 + +# Array whose value at index i is the unicode value unicode character i should +# map to. +array = [] +# Contents of gNormalizeTable. We insert zero entries for sub-pages where we +# have no mappings. We insert references to the tables where we do have +# such tables. +globalTable = [] +globalTable.append("0") +# The (exclusive) upper bound of the conversion table, unicode character-wise. +# This is 0x10000 because our generated table is only 16-bit. This also limits +# the values we can map to; we perform an identity mapping for target values +# that >= maxmapping. +maxmapping = 0x10000 +sizePerTable = 64 + +# Map characters that the mapping tells us to obliterate to the NUKE_CHAR +# (such lines look like "FFF0..FFF8>") +# We do this because if we didn't do this, we would emit these characters as +# part of a token, which we definitely don't want. +NUKE_CHAR = 0x20 + +# --- load case folding table +# entries in the file look like: +# 0041>0061 +# 02D8>0020 0306 +# 2000..200A>0020 +# +# The 0041 (uppercase A) tells us it lowercases to 0061 (lowercase a). +# The 02D8 is a "spacing clone[s] of diacritic" breve which gets decomposed into +# a space character and a breve. This entry/type of entry also shows up in +# 'nfkc.txt'. +# The 2000..200A covers a range of space characters and maps them down to the +# 'normal' space character. + +file = open('nfkc_cf.txt') + +m = None +line = "\n" +i = 0x0 +while i < maxmapping and line: + if not m: + line = file.readline() + m = p.match(line) + if not m: + continue + low = int(m.group(G_FROM), 16) + # if G_TO is present, use it, otherwise fallback to low + high = m.group(G_TO) and int(m.group(G_TO), 16) or low + # if G_FIRSTVAL is present use it, otherwise use NUKE_CHAR + val = (m.group(G_FIRSTVAL) and int(m.group(G_FIRSTVAL), 16) + or NUKE_CHAR) + continue + + + if i >= low and i <= high: + if val >= maxmapping: + array.append(i) + else: + array.append(val) + if i == high: + m = None + else: + array.append(i) + i = i + 1 +file.close() + +# --- load normalization / decomposition table +# It is important that this file gets processed second because the other table +# will tell us about mappings from uppercase U with diaeresis to lowercase u +# with diaeresis. We obviously don't want that clobbering our value. (Although +# this would work out if we propagated backwards rather than forwards...) +# +# - entries in this file that we care about look like: +# 00A0>0020 +# 0100=0041 0304 +# +# They are found in the "Canonical and compatibility decomposition mappings" +# section. +# +# The 00A0 is mapping NBSP to the normal space character. +# The 0100 (a capital A with a bar over top of) is equivalent to 0041 (capital +# A) plus a 0304 (combining overline). We do not care about the combining +# marks which is why our regular expression does not capture it. +# +# +# - entries that we do not care about look like: +# 0300..0314:230 +# +# These map marks to their canonical combining class which appears to be a way +# of specifying the precedence / order in which marks should be combined. The +# key thing is we don't care about them. +file = open('nfkc.txt') +line = file.readline() +m = p.match(line) +while line: + if not m: + line = file.readline() + m = p.match(line) + continue + + low = int(m.group(G_FROM), 16) + # if G_TO is present, use it, otherwise fallback to low + high = m.group(G_TO) and int(m.group(G_TO), 16) or low + # if G_FIRSTVAL is present use it, otherwise fall back to NUKE_CHAR + val = m.group(G_FIRSTVAL) and int(m.group(G_FIRSTVAL), 16) or NUKE_CHAR + for i in range(low, high+1): + if i < maxmapping and val < maxmapping: + array[i] = val + m = None +file.close() + +# --- generate a normalized table to support case and accent folding + +i = 0 +needTerm = False; +while i < maxmapping: + if not i % sizePerTable: + # table is empty? + j = i + while j < i + sizePerTable: + if array[j] != j: + break + j += 1 + + if j == i + sizePerTable: + if i: + globalTable.append("0") + i += sizePerTable + continue + + if needTerm: + print "};\n" + globalTable.append("gNormalizeTable%04x" % i) + print "static const unsigned short gNormalizeTable%04x[] = {\n\t" % i, + print "/* U+%04x */\n\t" % i, + needTerm = True + # Decomposition does not case-fold, so we want to compensate by + # performing a lookup here. Because decomposition chains can be + # example: 01d5, a capital U with a diaeresis and a bar. yes, really. + # 01d5 -> 00dc -> 0055 (U) -> 0075 (u) + c = array[i] + while c != array[c]: + c = array[c] + if c >= 0x41 and c <= 0x5a: + raise Exception('got an uppercase character somehow: %x => %x' + % (i, c)) + print "0x%04x," % c, + i = i + 1 + if not i % 8: + print "\n\t", + +print "};\n\nstatic const unsigned short* gNormalizeTable[] = {", +i = 0 +while i < (maxmapping / sizePerTable): + if not i % 4: + print "\n\t", + print globalTable[i] + ",", + i += 1 + +print ''' +}; + +unsigned int normalize_character(const unsigned int c) +{ + if (c >= ''' + ('0x%x' % (maxmapping,)) + ''' || !gNormalizeTable[c >> 6]) + return c; + return gNormalizeTable[c >> 6][c & 0x3f]; +} +''' diff --git a/mailnews/extensions/fts3/data/nfkc.txt b/mailnews/extensions/fts3/data/nfkc.txt new file mode 100644 index 000000000..08aaf353f --- /dev/null +++ b/mailnews/extensions/fts3/data/nfkc.txt @@ -0,0 +1,5786 @@ +# Copyright (C) 1999-2010, International Business Machines +# Corporation and others. All Rights Reserved. +# +# file name: nfkc.txt +# +# machine-generated on: 2009-11-30 +# + +# Canonical_Combining_Class (ccc) values +0300..0314:230 +0315:232 +0316..0319:220 +031A:232 +031B:216 +031C..0320:220 +0321..0322:202 +0323..0326:220 +0327..0328:202 +0329..0333:220 +0334..0338:1 +0339..033C:220 +033D..0344:230 +0345:240 +0346:230 +0347..0349:220 +034A..034C:230 +034D..034E:220 +0350..0352:230 +0353..0356:220 +0357:230 +0358:232 +0359..035A:220 +035B:230 +035C:233 +035D..035E:234 +035F:233 +0360..0361:234 +0362:233 +0363..036F:230 +0483..0487:230 +0591:220 +0592..0595:230 +0596:220 +0597..0599:230 +059A:222 +059B:220 +059C..05A1:230 +05A2..05A7:220 +05A8..05A9:230 +05AA:220 +05AB..05AC:230 +05AD:222 +05AE:228 +05AF:230 +05B0:10 +05B1:11 +05B2:12 +05B3:13 +05B4:14 +05B5:15 +05B6:16 +05B7:17 +05B8:18 +05B9..05BA:19 +05BB:20 +05BC:21 +05BD:22 +05BF:23 +05C1:24 +05C2:25 +05C4:230 +05C5:220 +05C7:18 +0610..0617:230 +0618:30 +0619:31 +061A:32 +064B:27 +064C:28 +064D:29 +064E:30 +064F:31 +0650:32 +0651:33 +0652:34 +0653..0654:230 +0655..0656:220 +0657..065B:230 +065C:220 +065D..065E:230 +0670:35 +06D6..06DC:230 +06DF..06E2:230 +06E3:220 +06E4:230 +06E7..06E8:230 +06EA:220 +06EB..06EC:230 +06ED:220 +0711:36 +0730:230 +0731:220 +0732..0733:230 +0734:220 +0735..0736:230 +0737..0739:220 +073A:230 +073B..073C:220 +073D:230 +073E:220 +073F..0741:230 +0742:220 +0743:230 +0744:220 +0745:230 +0746:220 +0747:230 +0748:220 +0749..074A:230 +07EB..07F1:230 +07F2:220 +07F3:230 +0816..0819:230 +081B..0823:230 +0825..0827:230 +0829..082D:230 +093C:7 +094D:9 +0951:230 +0952:220 +0953..0954:230 +09BC:7 +09CD:9 +0A3C:7 +0A4D:9 +0ABC:7 +0ACD:9 +0B3C:7 +0B4D:9 +0BCD:9 +0C4D:9 +0C55:84 +0C56:91 +0CBC:7 +0CCD:9 +0D4D:9 +0DCA:9 +0E38..0E39:103 +0E3A:9 +0E48..0E4B:107 +0EB8..0EB9:118 +0EC8..0ECB:122 +0F18..0F19:220 +0F35:220 +0F37:220 +0F39:216 +0F71:129 +0F72:130 +0F74:132 +0F7A..0F7D:130 +0F80:130 +0F82..0F83:230 +0F84:9 +0F86..0F87:230 +0FC6:220 +1037:7 +1039..103A:9 +108D:220 +135F:230 +1714:9 +1734:9 +17D2:9 +17DD:230 +18A9:228 +1939:222 +193A:230 +193B:220 +1A17:230 +1A18:220 +1A60:9 +1A75..1A7C:230 +1A7F:220 +1B34:7 +1B44:9 +1B6B:230 +1B6C:220 +1B6D..1B73:230 +1BAA:9 +1C37:7 +1CD0..1CD2:230 +1CD4:1 +1CD5..1CD9:220 +1CDA..1CDB:230 +1CDC..1CDF:220 +1CE0:230 +1CE2..1CE8:1 +1CED:220 +1DC0..1DC1:230 +1DC2:220 +1DC3..1DC9:230 +1DCA:220 +1DCB..1DCC:230 +1DCD:234 +1DCE:214 +1DCF:220 +1DD0:202 +1DD1..1DE6:230 +1DFD:220 +1DFE:230 +1DFF:220 +20D0..20D1:230 +20D2..20D3:1 +20D4..20D7:230 +20D8..20DA:1 +20DB..20DC:230 +20E1:230 +20E5..20E6:1 +20E7:230 +20E8:220 +20E9:230 +20EA..20EB:1 +20EC..20EF:220 +20F0:230 +2CEF..2CF1:230 +2DE0..2DFF:230 +302A:218 +302B:228 +302C:232 +302D:222 +302E..302F:224 +3099..309A:8 +A66F:230 +A67C..A67D:230 +A6F0..A6F1:230 +A806:9 +A8C4:9 +A8E0..A8F1:230 +A92B..A92D:220 +A953:9 +A9B3:7 +A9C0:9 +AAB0:230 +AAB2..AAB3:230 +AAB4:220 +AAB7..AAB8:230 +AABE..AABF:230 +AAC1:230 +ABED:9 +FB1E:26 +FE20..FE26:230 +101FD:220 +10A0D:220 +10A0F:230 +10A38:230 +10A39:1 +10A3A:220 +10A3F:9 +110B9:9 +110BA:7 +1D165..1D166:216 +1D167..1D169:1 +1D16D:226 +1D16E..1D172:216 +1D17B..1D182:220 +1D185..1D189:230 +1D18A..1D18B:220 +1D1AA..1D1AD:230 +1D242..1D244:230 + +# Canonical and compatibility decomposition mappings +00A0>0020 +00A8>0020 0308 +00AA>0061 +00AF>0020 0304 +00B2>0032 +00B3>0033 +00B4>0020 0301 +00B5>03BC +00B8>0020 0327 +00B9>0031 +00BA>006F +00BC>0031 2044 0034 +00BD>0031 2044 0032 +00BE>0033 2044 0034 +00C0=0041 0300 +00C1=0041 0301 +00C2=0041 0302 +00C3=0041 0303 +00C4=0041 0308 +00C5=0041 030A +00C7=0043 0327 +00C8=0045 0300 +00C9=0045 0301 +00CA=0045 0302 +00CB=0045 0308 +00CC=0049 0300 +00CD=0049 0301 +00CE=0049 0302 +00CF=0049 0308 +00D1=004E 0303 +00D2=004F 0300 +00D3=004F 0301 +00D4=004F 0302 +00D5=004F 0303 +00D6=004F 0308 +00D9=0055 0300 +00DA=0055 0301 +00DB=0055 0302 +00DC=0055 0308 +00DD=0059 0301 +00E0=0061 0300 +00E1=0061 0301 +00E2=0061 0302 +00E3=0061 0303 +00E4=0061 0308 +00E5=0061 030A +00E7=0063 0327 +00E8=0065 0300 +00E9=0065 0301 +00EA=0065 0302 +00EB=0065 0308 +00EC=0069 0300 +00ED=0069 0301 +00EE=0069 0302 +00EF=0069 0308 +00F1=006E 0303 +00F2=006F 0300 +00F3=006F 0301 +00F4=006F 0302 +00F5=006F 0303 +00F6=006F 0308 +00F9=0075 0300 +00FA=0075 0301 +00FB=0075 0302 +00FC=0075 0308 +00FD=0079 0301 +00FF=0079 0308 +0100=0041 0304 +0101=0061 0304 +0102=0041 0306 +0103=0061 0306 +0104=0041 0328 +0105=0061 0328 +0106=0043 0301 +0107=0063 0301 +0108=0043 0302 +0109=0063 0302 +010A=0043 0307 +010B=0063 0307 +010C=0043 030C +010D=0063 030C +010E=0044 030C +010F=0064 030C +0112=0045 0304 +0113=0065 0304 +0114=0045 0306 +0115=0065 0306 +0116=0045 0307 +0117=0065 0307 +0118=0045 0328 +0119=0065 0328 +011A=0045 030C +011B=0065 030C +011C=0047 0302 +011D=0067 0302 +011E=0047 0306 +011F=0067 0306 +0120=0047 0307 +0121=0067 0307 +0122=0047 0327 +0123=0067 0327 +0124=0048 0302 +0125=0068 0302 +0128=0049 0303 +0129=0069 0303 +012A=0049 0304 +012B=0069 0304 +012C=0049 0306 +012D=0069 0306 +012E=0049 0328 +012F=0069 0328 +0130=0049 0307 +0132>0049 004A +0133>0069 006A +0134=004A 0302 +0135=006A 0302 +0136=004B 0327 +0137=006B 0327 +0139=004C 0301 +013A=006C 0301 +013B=004C 0327 +013C=006C 0327 +013D=004C 030C +013E=006C 030C +013F>004C 00B7 +0140>006C 00B7 +0143=004E 0301 +0144=006E 0301 +0145=004E 0327 +0146=006E 0327 +0147=004E 030C +0148=006E 030C +0149>02BC 006E +014C=004F 0304 +014D=006F 0304 +014E=004F 0306 +014F=006F 0306 +0150=004F 030B +0151=006F 030B +0154=0052 0301 +0155=0072 0301 +0156=0052 0327 +0157=0072 0327 +0158=0052 030C +0159=0072 030C +015A=0053 0301 +015B=0073 0301 +015C=0053 0302 +015D=0073 0302 +015E=0053 0327 +015F=0073 0327 +0160=0053 030C +0161=0073 030C +0162=0054 0327 +0163=0074 0327 +0164=0054 030C +0165=0074 030C +0168=0055 0303 +0169=0075 0303 +016A=0055 0304 +016B=0075 0304 +016C=0055 0306 +016D=0075 0306 +016E=0055 030A +016F=0075 030A +0170=0055 030B +0171=0075 030B +0172=0055 0328 +0173=0075 0328 +0174=0057 0302 +0175=0077 0302 +0176=0059 0302 +0177=0079 0302 +0178=0059 0308 +0179=005A 0301 +017A=007A 0301 +017B=005A 0307 +017C=007A 0307 +017D=005A 030C +017E=007A 030C +017F>0073 +01A0=004F 031B +01A1=006F 031B +01AF=0055 031B +01B0=0075 031B +01C4>0044 017D +01C5>0044 017E +01C6>0064 017E +01C7>004C 004A +01C8>004C 006A +01C9>006C 006A +01CA>004E 004A +01CB>004E 006A +01CC>006E 006A +01CD=0041 030C +01CE=0061 030C +01CF=0049 030C +01D0=0069 030C +01D1=004F 030C +01D2=006F 030C +01D3=0055 030C +01D4=0075 030C +01D5=00DC 0304 +01D6=00FC 0304 +01D7=00DC 0301 +01D8=00FC 0301 +01D9=00DC 030C +01DA=00FC 030C +01DB=00DC 0300 +01DC=00FC 0300 +01DE=00C4 0304 +01DF=00E4 0304 +01E0=0226 0304 +01E1=0227 0304 +01E2=00C6 0304 +01E3=00E6 0304 +01E6=0047 030C +01E7=0067 030C +01E8=004B 030C +01E9=006B 030C +01EA=004F 0328 +01EB=006F 0328 +01EC=01EA 0304 +01ED=01EB 0304 +01EE=01B7 030C +01EF=0292 030C +01F0=006A 030C +01F1>0044 005A +01F2>0044 007A +01F3>0064 007A +01F4=0047 0301 +01F5=0067 0301 +01F8=004E 0300 +01F9=006E 0300 +01FA=00C5 0301 +01FB=00E5 0301 +01FC=00C6 0301 +01FD=00E6 0301 +01FE=00D8 0301 +01FF=00F8 0301 +0200=0041 030F +0201=0061 030F +0202=0041 0311 +0203=0061 0311 +0204=0045 030F +0205=0065 030F +0206=0045 0311 +0207=0065 0311 +0208=0049 030F +0209=0069 030F +020A=0049 0311 +020B=0069 0311 +020C=004F 030F +020D=006F 030F +020E=004F 0311 +020F=006F 0311 +0210=0052 030F +0211=0072 030F +0212=0052 0311 +0213=0072 0311 +0214=0055 030F +0215=0075 030F +0216=0055 0311 +0217=0075 0311 +0218=0053 0326 +0219=0073 0326 +021A=0054 0326 +021B=0074 0326 +021E=0048 030C +021F=0068 030C +0226=0041 0307 +0227=0061 0307 +0228=0045 0327 +0229=0065 0327 +022A=00D6 0304 +022B=00F6 0304 +022C=00D5 0304 +022D=00F5 0304 +022E=004F 0307 +022F=006F 0307 +0230=022E 0304 +0231=022F 0304 +0232=0059 0304 +0233=0079 0304 +02B0>0068 +02B1>0266 +02B2>006A +02B3>0072 +02B4>0279 +02B5>027B +02B6>0281 +02B7>0077 +02B8>0079 +02D8>0020 0306 +02D9>0020 0307 +02DA>0020 030A +02DB>0020 0328 +02DC>0020 0303 +02DD>0020 030B +02E0>0263 +02E1>006C +02E2>0073 +02E3>0078 +02E4>0295 +0340>0300 +0341>0301 +0343>0313 +0344>0308 0301 +0374>02B9 +037A>0020 0345 +037E>003B +0384>0020 0301 +0385>00A8 0301 +0386=0391 0301 +0387>00B7 +0388=0395 0301 +0389=0397 0301 +038A=0399 0301 +038C=039F 0301 +038E=03A5 0301 +038F=03A9 0301 +0390=03CA 0301 +03AA=0399 0308 +03AB=03A5 0308 +03AC=03B1 0301 +03AD=03B5 0301 +03AE=03B7 0301 +03AF=03B9 0301 +03B0=03CB 0301 +03CA=03B9 0308 +03CB=03C5 0308 +03CC=03BF 0301 +03CD=03C5 0301 +03CE=03C9 0301 +03D0>03B2 +03D1>03B8 +03D2>03A5 +03D3>03D2 0301 +03D4>03D2 0308 +03D5>03C6 +03D6>03C0 +03F0>03BA +03F1>03C1 +03F2>03C2 +03F4>0398 +03F5>03B5 +03F9>03A3 +0400=0415 0300 +0401=0415 0308 +0403=0413 0301 +0407=0406 0308 +040C=041A 0301 +040D=0418 0300 +040E=0423 0306 +0419=0418 0306 +0439=0438 0306 +0450=0435 0300 +0451=0435 0308 +0453=0433 0301 +0457=0456 0308 +045C=043A 0301 +045D=0438 0300 +045E=0443 0306 +0476=0474 030F +0477=0475 030F +04C1=0416 0306 +04C2=0436 0306 +04D0=0410 0306 +04D1=0430 0306 +04D2=0410 0308 +04D3=0430 0308 +04D6=0415 0306 +04D7=0435 0306 +04DA=04D8 0308 +04DB=04D9 0308 +04DC=0416 0308 +04DD=0436 0308 +04DE=0417 0308 +04DF=0437 0308 +04E2=0418 0304 +04E3=0438 0304 +04E4=0418 0308 +04E5=0438 0308 +04E6=041E 0308 +04E7=043E 0308 +04EA=04E8 0308 +04EB=04E9 0308 +04EC=042D 0308 +04ED=044D 0308 +04EE=0423 0304 +04EF=0443 0304 +04F0=0423 0308 +04F1=0443 0308 +04F2=0423 030B +04F3=0443 030B +04F4=0427 0308 +04F5=0447 0308 +04F8=042B 0308 +04F9=044B 0308 +0587>0565 0582 +0622=0627 0653 +0623=0627 0654 +0624=0648 0654 +0625=0627 0655 +0626=064A 0654 +0675>0627 0674 +0676>0648 0674 +0677>06C7 0674 +0678>064A 0674 +06C0=06D5 0654 +06C2=06C1 0654 +06D3=06D2 0654 +0929=0928 093C +0931=0930 093C +0934=0933 093C +0958>0915 093C +0959>0916 093C +095A>0917 093C +095B>091C 093C +095C>0921 093C +095D>0922 093C +095E>092B 093C +095F>092F 093C +09CB=09C7 09BE +09CC=09C7 09D7 +09DC>09A1 09BC +09DD>09A2 09BC +09DF>09AF 09BC +0A33>0A32 0A3C +0A36>0A38 0A3C +0A59>0A16 0A3C +0A5A>0A17 0A3C +0A5B>0A1C 0A3C +0A5E>0A2B 0A3C +0B48=0B47 0B56 +0B4B=0B47 0B3E +0B4C=0B47 0B57 +0B5C>0B21 0B3C +0B5D>0B22 0B3C +0B94=0B92 0BD7 +0BCA=0BC6 0BBE +0BCB=0BC7 0BBE +0BCC=0BC6 0BD7 +0C48=0C46 0C56 +0CC0=0CBF 0CD5 +0CC7=0CC6 0CD5 +0CC8=0CC6 0CD6 +0CCA=0CC6 0CC2 +0CCB=0CCA 0CD5 +0D4A=0D46 0D3E +0D4B=0D47 0D3E +0D4C=0D46 0D57 +0DDA=0DD9 0DCA +0DDC=0DD9 0DCF +0DDD=0DDC 0DCA +0DDE=0DD9 0DDF +0E33>0E4D 0E32 +0EB3>0ECD 0EB2 +0EDC>0EAB 0E99 +0EDD>0EAB 0EA1 +0F0C>0F0B +0F43>0F42 0FB7 +0F4D>0F4C 0FB7 +0F52>0F51 0FB7 +0F57>0F56 0FB7 +0F5C>0F5B 0FB7 +0F69>0F40 0FB5 +0F73>0F71 0F72 +0F75>0F71 0F74 +0F76>0FB2 0F80 +0F77>0FB2 0F81 +0F78>0FB3 0F80 +0F79>0FB3 0F81 +0F81>0F71 0F80 +0F93>0F92 0FB7 +0F9D>0F9C 0FB7 +0FA2>0FA1 0FB7 +0FA7>0FA6 0FB7 +0FAC>0FAB 0FB7 +0FB9>0F90 0FB5 +1026=1025 102E +10FC>10DC +1B06=1B05 1B35 +1B08=1B07 1B35 +1B0A=1B09 1B35 +1B0C=1B0B 1B35 +1B0E=1B0D 1B35 +1B12=1B11 1B35 +1B3B=1B3A 1B35 +1B3D=1B3C 1B35 +1B40=1B3E 1B35 +1B41=1B3F 1B35 +1B43=1B42 1B35 +1D2C>0041 +1D2D>00C6 +1D2E>0042 +1D30>0044 +1D31>0045 +1D32>018E +1D33>0047 +1D34>0048 +1D35>0049 +1D36>004A +1D37>004B +1D38>004C +1D39>004D +1D3A>004E +1D3C>004F +1D3D>0222 +1D3E>0050 +1D3F>0052 +1D40>0054 +1D41>0055 +1D42>0057 +1D43>0061 +1D44>0250 +1D45>0251 +1D46>1D02 +1D47>0062 +1D48>0064 +1D49>0065 +1D4A>0259 +1D4B>025B +1D4C>025C +1D4D>0067 +1D4F>006B +1D50>006D +1D51>014B +1D52>006F +1D53>0254 +1D54>1D16 +1D55>1D17 +1D56>0070 +1D57>0074 +1D58>0075 +1D59>1D1D +1D5A>026F +1D5B>0076 +1D5C>1D25 +1D5D>03B2 +1D5E>03B3 +1D5F>03B4 +1D60>03C6 +1D61>03C7 +1D62>0069 +1D63>0072 +1D64>0075 +1D65>0076 +1D66>03B2 +1D67>03B3 +1D68>03C1 +1D69>03C6 +1D6A>03C7 +1D78>043D +1D9B>0252 +1D9C>0063 +1D9D>0255 +1D9E>00F0 +1D9F>025C +1DA0>0066 +1DA1>025F +1DA2>0261 +1DA3>0265 +1DA4>0268 +1DA5>0269 +1DA6>026A +1DA7>1D7B +1DA8>029D +1DA9>026D +1DAA>1D85 +1DAB>029F +1DAC>0271 +1DAD>0270 +1DAE>0272 +1DAF>0273 +1DB0>0274 +1DB1>0275 +1DB2>0278 +1DB3>0282 +1DB4>0283 +1DB5>01AB +1DB6>0289 +1DB7>028A +1DB8>1D1C +1DB9>028B +1DBA>028C +1DBB>007A +1DBC>0290 +1DBD>0291 +1DBE>0292 +1DBF>03B8 +1E00=0041 0325 +1E01=0061 0325 +1E02=0042 0307 +1E03=0062 0307 +1E04=0042 0323 +1E05=0062 0323 +1E06=0042 0331 +1E07=0062 0331 +1E08=00C7 0301 +1E09=00E7 0301 +1E0A=0044 0307 +1E0B=0064 0307 +1E0C=0044 0323 +1E0D=0064 0323 +1E0E=0044 0331 +1E0F=0064 0331 +1E10=0044 0327 +1E11=0064 0327 +1E12=0044 032D +1E13=0064 032D +1E14=0112 0300 +1E15=0113 0300 +1E16=0112 0301 +1E17=0113 0301 +1E18=0045 032D +1E19=0065 032D +1E1A=0045 0330 +1E1B=0065 0330 +1E1C=0228 0306 +1E1D=0229 0306 +1E1E=0046 0307 +1E1F=0066 0307 +1E20=0047 0304 +1E21=0067 0304 +1E22=0048 0307 +1E23=0068 0307 +1E24=0048 0323 +1E25=0068 0323 +1E26=0048 0308 +1E27=0068 0308 +1E28=0048 0327 +1E29=0068 0327 +1E2A=0048 032E +1E2B=0068 032E +1E2C=0049 0330 +1E2D=0069 0330 +1E2E=00CF 0301 +1E2F=00EF 0301 +1E30=004B 0301 +1E31=006B 0301 +1E32=004B 0323 +1E33=006B 0323 +1E34=004B 0331 +1E35=006B 0331 +1E36=004C 0323 +1E37=006C 0323 +1E38=1E36 0304 +1E39=1E37 0304 +1E3A=004C 0331 +1E3B=006C 0331 +1E3C=004C 032D +1E3D=006C 032D +1E3E=004D 0301 +1E3F=006D 0301 +1E40=004D 0307 +1E41=006D 0307 +1E42=004D 0323 +1E43=006D 0323 +1E44=004E 0307 +1E45=006E 0307 +1E46=004E 0323 +1E47=006E 0323 +1E48=004E 0331 +1E49=006E 0331 +1E4A=004E 032D +1E4B=006E 032D +1E4C=00D5 0301 +1E4D=00F5 0301 +1E4E=00D5 0308 +1E4F=00F5 0308 +1E50=014C 0300 +1E51=014D 0300 +1E52=014C 0301 +1E53=014D 0301 +1E54=0050 0301 +1E55=0070 0301 +1E56=0050 0307 +1E57=0070 0307 +1E58=0052 0307 +1E59=0072 0307 +1E5A=0052 0323 +1E5B=0072 0323 +1E5C=1E5A 0304 +1E5D=1E5B 0304 +1E5E=0052 0331 +1E5F=0072 0331 +1E60=0053 0307 +1E61=0073 0307 +1E62=0053 0323 +1E63=0073 0323 +1E64=015A 0307 +1E65=015B 0307 +1E66=0160 0307 +1E67=0161 0307 +1E68=1E62 0307 +1E69=1E63 0307 +1E6A=0054 0307 +1E6B=0074 0307 +1E6C=0054 0323 +1E6D=0074 0323 +1E6E=0054 0331 +1E6F=0074 0331 +1E70=0054 032D +1E71=0074 032D +1E72=0055 0324 +1E73=0075 0324 +1E74=0055 0330 +1E75=0075 0330 +1E76=0055 032D +1E77=0075 032D +1E78=0168 0301 +1E79=0169 0301 +1E7A=016A 0308 +1E7B=016B 0308 +1E7C=0056 0303 +1E7D=0076 0303 +1E7E=0056 0323 +1E7F=0076 0323 +1E80=0057 0300 +1E81=0077 0300 +1E82=0057 0301 +1E83=0077 0301 +1E84=0057 0308 +1E85=0077 0308 +1E86=0057 0307 +1E87=0077 0307 +1E88=0057 0323 +1E89=0077 0323 +1E8A=0058 0307 +1E8B=0078 0307 +1E8C=0058 0308 +1E8D=0078 0308 +1E8E=0059 0307 +1E8F=0079 0307 +1E90=005A 0302 +1E91=007A 0302 +1E92=005A 0323 +1E93=007A 0323 +1E94=005A 0331 +1E95=007A 0331 +1E96=0068 0331 +1E97=0074 0308 +1E98=0077 030A +1E99=0079 030A +1E9A>0061 02BE +1E9B>017F 0307 +1EA0=0041 0323 +1EA1=0061 0323 +1EA2=0041 0309 +1EA3=0061 0309 +1EA4=00C2 0301 +1EA5=00E2 0301 +1EA6=00C2 0300 +1EA7=00E2 0300 +1EA8=00C2 0309 +1EA9=00E2 0309 +1EAA=00C2 0303 +1EAB=00E2 0303 +1EAC=1EA0 0302 +1EAD=1EA1 0302 +1EAE=0102 0301 +1EAF=0103 0301 +1EB0=0102 0300 +1EB1=0103 0300 +1EB2=0102 0309 +1EB3=0103 0309 +1EB4=0102 0303 +1EB5=0103 0303 +1EB6=1EA0 0306 +1EB7=1EA1 0306 +1EB8=0045 0323 +1EB9=0065 0323 +1EBA=0045 0309 +1EBB=0065 0309 +1EBC=0045 0303 +1EBD=0065 0303 +1EBE=00CA 0301 +1EBF=00EA 0301 +1EC0=00CA 0300 +1EC1=00EA 0300 +1EC2=00CA 0309 +1EC3=00EA 0309 +1EC4=00CA 0303 +1EC5=00EA 0303 +1EC6=1EB8 0302 +1EC7=1EB9 0302 +1EC8=0049 0309 +1EC9=0069 0309 +1ECA=0049 0323 +1ECB=0069 0323 +1ECC=004F 0323 +1ECD=006F 0323 +1ECE=004F 0309 +1ECF=006F 0309 +1ED0=00D4 0301 +1ED1=00F4 0301 +1ED2=00D4 0300 +1ED3=00F4 0300 +1ED4=00D4 0309 +1ED5=00F4 0309 +1ED6=00D4 0303 +1ED7=00F4 0303 +1ED8=1ECC 0302 +1ED9=1ECD 0302 +1EDA=01A0 0301 +1EDB=01A1 0301 +1EDC=01A0 0300 +1EDD=01A1 0300 +1EDE=01A0 0309 +1EDF=01A1 0309 +1EE0=01A0 0303 +1EE1=01A1 0303 +1EE2=01A0 0323 +1EE3=01A1 0323 +1EE4=0055 0323 +1EE5=0075 0323 +1EE6=0055 0309 +1EE7=0075 0309 +1EE8=01AF 0301 +1EE9=01B0 0301 +1EEA=01AF 0300 +1EEB=01B0 0300 +1EEC=01AF 0309 +1EED=01B0 0309 +1EEE=01AF 0303 +1EEF=01B0 0303 +1EF0=01AF 0323 +1EF1=01B0 0323 +1EF2=0059 0300 +1EF3=0079 0300 +1EF4=0059 0323 +1EF5=0079 0323 +1EF6=0059 0309 +1EF7=0079 0309 +1EF8=0059 0303 +1EF9=0079 0303 +1F00=03B1 0313 +1F01=03B1 0314 +1F02=1F00 0300 +1F03=1F01 0300 +1F04=1F00 0301 +1F05=1F01 0301 +1F06=1F00 0342 +1F07=1F01 0342 +1F08=0391 0313 +1F09=0391 0314 +1F0A=1F08 0300 +1F0B=1F09 0300 +1F0C=1F08 0301 +1F0D=1F09 0301 +1F0E=1F08 0342 +1F0F=1F09 0342 +1F10=03B5 0313 +1F11=03B5 0314 +1F12=1F10 0300 +1F13=1F11 0300 +1F14=1F10 0301 +1F15=1F11 0301 +1F18=0395 0313 +1F19=0395 0314 +1F1A=1F18 0300 +1F1B=1F19 0300 +1F1C=1F18 0301 +1F1D=1F19 0301 +1F20=03B7 0313 +1F21=03B7 0314 +1F22=1F20 0300 +1F23=1F21 0300 +1F24=1F20 0301 +1F25=1F21 0301 +1F26=1F20 0342 +1F27=1F21 0342 +1F28=0397 0313 +1F29=0397 0314 +1F2A=1F28 0300 +1F2B=1F29 0300 +1F2C=1F28 0301 +1F2D=1F29 0301 +1F2E=1F28 0342 +1F2F=1F29 0342 +1F30=03B9 0313 +1F31=03B9 0314 +1F32=1F30 0300 +1F33=1F31 0300 +1F34=1F30 0301 +1F35=1F31 0301 +1F36=1F30 0342 +1F37=1F31 0342 +1F38=0399 0313 +1F39=0399 0314 +1F3A=1F38 0300 +1F3B=1F39 0300 +1F3C=1F38 0301 +1F3D=1F39 0301 +1F3E=1F38 0342 +1F3F=1F39 0342 +1F40=03BF 0313 +1F41=03BF 0314 +1F42=1F40 0300 +1F43=1F41 0300 +1F44=1F40 0301 +1F45=1F41 0301 +1F48=039F 0313 +1F49=039F 0314 +1F4A=1F48 0300 +1F4B=1F49 0300 +1F4C=1F48 0301 +1F4D=1F49 0301 +1F50=03C5 0313 +1F51=03C5 0314 +1F52=1F50 0300 +1F53=1F51 0300 +1F54=1F50 0301 +1F55=1F51 0301 +1F56=1F50 0342 +1F57=1F51 0342 +1F59=03A5 0314 +1F5B=1F59 0300 +1F5D=1F59 0301 +1F5F=1F59 0342 +1F60=03C9 0313 +1F61=03C9 0314 +1F62=1F60 0300 +1F63=1F61 0300 +1F64=1F60 0301 +1F65=1F61 0301 +1F66=1F60 0342 +1F67=1F61 0342 +1F68=03A9 0313 +1F69=03A9 0314 +1F6A=1F68 0300 +1F6B=1F69 0300 +1F6C=1F68 0301 +1F6D=1F69 0301 +1F6E=1F68 0342 +1F6F=1F69 0342 +1F70=03B1 0300 +1F71>03AC +1F72=03B5 0300 +1F73>03AD +1F74=03B7 0300 +1F75>03AE +1F76=03B9 0300 +1F77>03AF +1F78=03BF 0300 +1F79>03CC +1F7A=03C5 0300 +1F7B>03CD +1F7C=03C9 0300 +1F7D>03CE +1F80=1F00 0345 +1F81=1F01 0345 +1F82=1F02 0345 +1F83=1F03 0345 +1F84=1F04 0345 +1F85=1F05 0345 +1F86=1F06 0345 +1F87=1F07 0345 +1F88=1F08 0345 +1F89=1F09 0345 +1F8A=1F0A 0345 +1F8B=1F0B 0345 +1F8C=1F0C 0345 +1F8D=1F0D 0345 +1F8E=1F0E 0345 +1F8F=1F0F 0345 +1F90=1F20 0345 +1F91=1F21 0345 +1F92=1F22 0345 +1F93=1F23 0345 +1F94=1F24 0345 +1F95=1F25 0345 +1F96=1F26 0345 +1F97=1F27 0345 +1F98=1F28 0345 +1F99=1F29 0345 +1F9A=1F2A 0345 +1F9B=1F2B 0345 +1F9C=1F2C 0345 +1F9D=1F2D 0345 +1F9E=1F2E 0345 +1F9F=1F2F 0345 +1FA0=1F60 0345 +1FA1=1F61 0345 +1FA2=1F62 0345 +1FA3=1F63 0345 +1FA4=1F64 0345 +1FA5=1F65 0345 +1FA6=1F66 0345 +1FA7=1F67 0345 +1FA8=1F68 0345 +1FA9=1F69 0345 +1FAA=1F6A 0345 +1FAB=1F6B 0345 +1FAC=1F6C 0345 +1FAD=1F6D 0345 +1FAE=1F6E 0345 +1FAF=1F6F 0345 +1FB0=03B1 0306 +1FB1=03B1 0304 +1FB2=1F70 0345 +1FB3=03B1 0345 +1FB4=03AC 0345 +1FB6=03B1 0342 +1FB7=1FB6 0345 +1FB8=0391 0306 +1FB9=0391 0304 +1FBA=0391 0300 +1FBB>0386 +1FBC=0391 0345 +1FBD>0020 0313 +1FBE>03B9 +1FBF>0020 0313 +1FC0>0020 0342 +1FC1>00A8 0342 +1FC2=1F74 0345 +1FC3=03B7 0345 +1FC4=03AE 0345 +1FC6=03B7 0342 +1FC7=1FC6 0345 +1FC8=0395 0300 +1FC9>0388 +1FCA=0397 0300 +1FCB>0389 +1FCC=0397 0345 +1FCD>1FBF 0300 +1FCE>1FBF 0301 +1FCF>1FBF 0342 +1FD0=03B9 0306 +1FD1=03B9 0304 +1FD2=03CA 0300 +1FD3>0390 +1FD6=03B9 0342 +1FD7=03CA 0342 +1FD8=0399 0306 +1FD9=0399 0304 +1FDA=0399 0300 +1FDB>038A +1FDD>1FFE 0300 +1FDE>1FFE 0301 +1FDF>1FFE 0342 +1FE0=03C5 0306 +1FE1=03C5 0304 +1FE2=03CB 0300 +1FE3>03B0 +1FE4=03C1 0313 +1FE5=03C1 0314 +1FE6=03C5 0342 +1FE7=03CB 0342 +1FE8=03A5 0306 +1FE9=03A5 0304 +1FEA=03A5 0300 +1FEB>038E +1FEC=03A1 0314 +1FED>00A8 0300 +1FEE>0385 +1FEF>0060 +1FF2=1F7C 0345 +1FF3=03C9 0345 +1FF4=03CE 0345 +1FF6=03C9 0342 +1FF7=1FF6 0345 +1FF8=039F 0300 +1FF9>038C +1FFA=03A9 0300 +1FFB>038F +1FFC=03A9 0345 +1FFD>00B4 +1FFE>0020 0314 +2000>2002 +2001>2003 +2002>0020 +2003>0020 +2004>0020 +2005>0020 +2006>0020 +2007>0020 +2008>0020 +2009>0020 +200A>0020 +2011>2010 +2017>0020 0333 +2024>002E +2025>002E 002E +2026>002E 002E 002E +202F>0020 +2033>2032 2032 +2034>2032 2032 2032 +2036>2035 2035 +2037>2035 2035 2035 +203C>0021 0021 +203E>0020 0305 +2047>003F 003F +2048>003F 0021 +2049>0021 003F +2057>2032 2032 2032 2032 +205F>0020 +2070>0030 +2071>0069 +2074>0034 +2075>0035 +2076>0036 +2077>0037 +2078>0038 +2079>0039 +207A>002B +207B>2212 +207C>003D +207D>0028 +207E>0029 +207F>006E +2080>0030 +2081>0031 +2082>0032 +2083>0033 +2084>0034 +2085>0035 +2086>0036 +2087>0037 +2088>0038 +2089>0039 +208A>002B +208B>2212 +208C>003D +208D>0028 +208E>0029 +2090>0061 +2091>0065 +2092>006F +2093>0078 +2094>0259 +20A8>0052 0073 +2100>0061 002F 0063 +2101>0061 002F 0073 +2102>0043 +2103>00B0 0043 +2105>0063 002F 006F +2106>0063 002F 0075 +2107>0190 +2109>00B0 0046 +210A>0067 +210B>0048 +210C>0048 +210D>0048 +210E>0068 +210F>0127 +2110>0049 +2111>0049 +2112>004C +2113>006C +2115>004E +2116>004E 006F +2119>0050 +211A>0051 +211B>0052 +211C>0052 +211D>0052 +2120>0053 004D +2121>0054 0045 004C +2122>0054 004D +2124>005A +2126>03A9 +2128>005A +212A>004B +212B>00C5 +212C>0042 +212D>0043 +212F>0065 +2130>0045 +2131>0046 +2133>004D +2134>006F +2135>05D0 +2136>05D1 +2137>05D2 +2138>05D3 +2139>0069 +213B>0046 0041 0058 +213C>03C0 +213D>03B3 +213E>0393 +213F>03A0 +2140>2211 +2145>0044 +2146>0064 +2147>0065 +2148>0069 +2149>006A +2150>0031 2044 0037 +2151>0031 2044 0039 +2152>0031 2044 0031 0030 +2153>0031 2044 0033 +2154>0032 2044 0033 +2155>0031 2044 0035 +2156>0032 2044 0035 +2157>0033 2044 0035 +2158>0034 2044 0035 +2159>0031 2044 0036 +215A>0035 2044 0036 +215B>0031 2044 0038 +215C>0033 2044 0038 +215D>0035 2044 0038 +215E>0037 2044 0038 +215F>0031 2044 +2160>0049 +2161>0049 0049 +2162>0049 0049 0049 +2163>0049 0056 +2164>0056 +2165>0056 0049 +2166>0056 0049 0049 +2167>0056 0049 0049 0049 +2168>0049 0058 +2169>0058 +216A>0058 0049 +216B>0058 0049 0049 +216C>004C +216D>0043 +216E>0044 +216F>004D +2170>0069 +2171>0069 0069 +2172>0069 0069 0069 +2173>0069 0076 +2174>0076 +2175>0076 0069 +2176>0076 0069 0069 +2177>0076 0069 0069 0069 +2178>0069 0078 +2179>0078 +217A>0078 0069 +217B>0078 0069 0069 +217C>006C +217D>0063 +217E>0064 +217F>006D +2189>0030 2044 0033 +219A=2190 0338 +219B=2192 0338 +21AE=2194 0338 +21CD=21D0 0338 +21CE=21D4 0338 +21CF=21D2 0338 +2204=2203 0338 +2209=2208 0338 +220C=220B 0338 +2224=2223 0338 +2226=2225 0338 +222C>222B 222B +222D>222B 222B 222B +222F>222E 222E +2230>222E 222E 222E +2241=223C 0338 +2244=2243 0338 +2247=2245 0338 +2249=2248 0338 +2260=003D 0338 +2262=2261 0338 +226D=224D 0338 +226E=003C 0338 +226F=003E 0338 +2270=2264 0338 +2271=2265 0338 +2274=2272 0338 +2275=2273 0338 +2278=2276 0338 +2279=2277 0338 +2280=227A 0338 +2281=227B 0338 +2284=2282 0338 +2285=2283 0338 +2288=2286 0338 +2289=2287 0338 +22AC=22A2 0338 +22AD=22A8 0338 +22AE=22A9 0338 +22AF=22AB 0338 +22E0=227C 0338 +22E1=227D 0338 +22E2=2291 0338 +22E3=2292 0338 +22EA=22B2 0338 +22EB=22B3 0338 +22EC=22B4 0338 +22ED=22B5 0338 +2329>3008 +232A>3009 +2460>0031 +2461>0032 +2462>0033 +2463>0034 +2464>0035 +2465>0036 +2466>0037 +2467>0038 +2468>0039 +2469>0031 0030 +246A>0031 0031 +246B>0031 0032 +246C>0031 0033 +246D>0031 0034 +246E>0031 0035 +246F>0031 0036 +2470>0031 0037 +2471>0031 0038 +2472>0031 0039 +2473>0032 0030 +2474>0028 0031 0029 +2475>0028 0032 0029 +2476>0028 0033 0029 +2477>0028 0034 0029 +2478>0028 0035 0029 +2479>0028 0036 0029 +247A>0028 0037 0029 +247B>0028 0038 0029 +247C>0028 0039 0029 +247D>0028 0031 0030 0029 +247E>0028 0031 0031 0029 +247F>0028 0031 0032 0029 +2480>0028 0031 0033 0029 +2481>0028 0031 0034 0029 +2482>0028 0031 0035 0029 +2483>0028 0031 0036 0029 +2484>0028 0031 0037 0029 +2485>0028 0031 0038 0029 +2486>0028 0031 0039 0029 +2487>0028 0032 0030 0029 +2488>0031 002E +2489>0032 002E +248A>0033 002E +248B>0034 002E +248C>0035 002E +248D>0036 002E +248E>0037 002E +248F>0038 002E +2490>0039 002E +2491>0031 0030 002E +2492>0031 0031 002E +2493>0031 0032 002E +2494>0031 0033 002E +2495>0031 0034 002E +2496>0031 0035 002E +2497>0031 0036 002E +2498>0031 0037 002E +2499>0031 0038 002E +249A>0031 0039 002E +249B>0032 0030 002E +249C>0028 0061 0029 +249D>0028 0062 0029 +249E>0028 0063 0029 +249F>0028 0064 0029 +24A0>0028 0065 0029 +24A1>0028 0066 0029 +24A2>0028 0067 0029 +24A3>0028 0068 0029 +24A4>0028 0069 0029 +24A5>0028 006A 0029 +24A6>0028 006B 0029 +24A7>0028 006C 0029 +24A8>0028 006D 0029 +24A9>0028 006E 0029 +24AA>0028 006F 0029 +24AB>0028 0070 0029 +24AC>0028 0071 0029 +24AD>0028 0072 0029 +24AE>0028 0073 0029 +24AF>0028 0074 0029 +24B0>0028 0075 0029 +24B1>0028 0076 0029 +24B2>0028 0077 0029 +24B3>0028 0078 0029 +24B4>0028 0079 0029 +24B5>0028 007A 0029 +24B6>0041 +24B7>0042 +24B8>0043 +24B9>0044 +24BA>0045 +24BB>0046 +24BC>0047 +24BD>0048 +24BE>0049 +24BF>004A +24C0>004B +24C1>004C +24C2>004D +24C3>004E +24C4>004F +24C5>0050 +24C6>0051 +24C7>0052 +24C8>0053 +24C9>0054 +24CA>0055 +24CB>0056 +24CC>0057 +24CD>0058 +24CE>0059 +24CF>005A +24D0>0061 +24D1>0062 +24D2>0063 +24D3>0064 +24D4>0065 +24D5>0066 +24D6>0067 +24D7>0068 +24D8>0069 +24D9>006A +24DA>006B +24DB>006C +24DC>006D +24DD>006E +24DE>006F +24DF>0070 +24E0>0071 +24E1>0072 +24E2>0073 +24E3>0074 +24E4>0075 +24E5>0076 +24E6>0077 +24E7>0078 +24E8>0079 +24E9>007A +24EA>0030 +2A0C>222B 222B 222B 222B +2A74>003A 003A 003D +2A75>003D 003D +2A76>003D 003D 003D +2ADC>2ADD 0338 +2C7C>006A +2C7D>0056 +2D6F>2D61 +2E9F>6BCD +2EF3>9F9F +2F00>4E00 +2F01>4E28 +2F02>4E36 +2F03>4E3F +2F04>4E59 +2F05>4E85 +2F06>4E8C +2F07>4EA0 +2F08>4EBA +2F09>513F +2F0A>5165 +2F0B>516B +2F0C>5182 +2F0D>5196 +2F0E>51AB +2F0F>51E0 +2F10>51F5 +2F11>5200 +2F12>529B +2F13>52F9 +2F14>5315 +2F15>531A +2F16>5338 +2F17>5341 +2F18>535C +2F19>5369 +2F1A>5382 +2F1B>53B6 +2F1C>53C8 +2F1D>53E3 +2F1E>56D7 +2F1F>571F +2F20>58EB +2F21>5902 +2F22>590A +2F23>5915 +2F24>5927 +2F25>5973 +2F26>5B50 +2F27>5B80 +2F28>5BF8 +2F29>5C0F +2F2A>5C22 +2F2B>5C38 +2F2C>5C6E +2F2D>5C71 +2F2E>5DDB +2F2F>5DE5 +2F30>5DF1 +2F31>5DFE +2F32>5E72 +2F33>5E7A +2F34>5E7F +2F35>5EF4 +2F36>5EFE +2F37>5F0B +2F38>5F13 +2F39>5F50 +2F3A>5F61 +2F3B>5F73 +2F3C>5FC3 +2F3D>6208 +2F3E>6236 +2F3F>624B +2F40>652F +2F41>6534 +2F42>6587 +2F43>6597 +2F44>65A4 +2F45>65B9 +2F46>65E0 +2F47>65E5 +2F48>66F0 +2F49>6708 +2F4A>6728 +2F4B>6B20 +2F4C>6B62 +2F4D>6B79 +2F4E>6BB3 +2F4F>6BCB +2F50>6BD4 +2F51>6BDB +2F52>6C0F +2F53>6C14 +2F54>6C34 +2F55>706B +2F56>722A +2F57>7236 +2F58>723B +2F59>723F +2F5A>7247 +2F5B>7259 +2F5C>725B +2F5D>72AC +2F5E>7384 +2F5F>7389 +2F60>74DC +2F61>74E6 +2F62>7518 +2F63>751F +2F64>7528 +2F65>7530 +2F66>758B +2F67>7592 +2F68>7676 +2F69>767D +2F6A>76AE +2F6B>76BF +2F6C>76EE +2F6D>77DB +2F6E>77E2 +2F6F>77F3 +2F70>793A +2F71>79B8 +2F72>79BE +2F73>7A74 +2F74>7ACB +2F75>7AF9 +2F76>7C73 +2F77>7CF8 +2F78>7F36 +2F79>7F51 +2F7A>7F8A +2F7B>7FBD +2F7C>8001 +2F7D>800C +2F7E>8012 +2F7F>8033 +2F80>807F +2F81>8089 +2F82>81E3 +2F83>81EA +2F84>81F3 +2F85>81FC +2F86>820C +2F87>821B +2F88>821F +2F89>826E +2F8A>8272 +2F8B>8278 +2F8C>864D +2F8D>866B +2F8E>8840 +2F8F>884C +2F90>8863 +2F91>897E +2F92>898B +2F93>89D2 +2F94>8A00 +2F95>8C37 +2F96>8C46 +2F97>8C55 +2F98>8C78 +2F99>8C9D +2F9A>8D64 +2F9B>8D70 +2F9C>8DB3 +2F9D>8EAB +2F9E>8ECA +2F9F>8F9B +2FA0>8FB0 +2FA1>8FB5 +2FA2>9091 +2FA3>9149 +2FA4>91C6 +2FA5>91CC +2FA6>91D1 +2FA7>9577 +2FA8>9580 +2FA9>961C +2FAA>96B6 +2FAB>96B9 +2FAC>96E8 +2FAD>9751 +2FAE>975E +2FAF>9762 +2FB0>9769 +2FB1>97CB +2FB2>97ED +2FB3>97F3 +2FB4>9801 +2FB5>98A8 +2FB6>98DB +2FB7>98DF +2FB8>9996 +2FB9>9999 +2FBA>99AC +2FBB>9AA8 +2FBC>9AD8 +2FBD>9ADF +2FBE>9B25 +2FBF>9B2F +2FC0>9B32 +2FC1>9B3C +2FC2>9B5A +2FC3>9CE5 +2FC4>9E75 +2FC5>9E7F +2FC6>9EA5 +2FC7>9EBB +2FC8>9EC3 +2FC9>9ECD +2FCA>9ED1 +2FCB>9EF9 +2FCC>9EFD +2FCD>9F0E +2FCE>9F13 +2FCF>9F20 +2FD0>9F3B +2FD1>9F4A +2FD2>9F52 +2FD3>9F8D +2FD4>9F9C +2FD5>9FA0 +3000>0020 +3036>3012 +3038>5341 +3039>5344 +303A>5345 +304C=304B 3099 +304E=304D 3099 +3050=304F 3099 +3052=3051 3099 +3054=3053 3099 +3056=3055 3099 +3058=3057 3099 +305A=3059 3099 +305C=305B 3099 +305E=305D 3099 +3060=305F 3099 +3062=3061 3099 +3065=3064 3099 +3067=3066 3099 +3069=3068 3099 +3070=306F 3099 +3071=306F 309A +3073=3072 3099 +3074=3072 309A +3076=3075 3099 +3077=3075 309A +3079=3078 3099 +307A=3078 309A +307C=307B 3099 +307D=307B 309A +3094=3046 3099 +309B>0020 3099 +309C>0020 309A +309E=309D 3099 +309F>3088 308A +30AC=30AB 3099 +30AE=30AD 3099 +30B0=30AF 3099 +30B2=30B1 3099 +30B4=30B3 3099 +30B6=30B5 3099 +30B8=30B7 3099 +30BA=30B9 3099 +30BC=30BB 3099 +30BE=30BD 3099 +30C0=30BF 3099 +30C2=30C1 3099 +30C5=30C4 3099 +30C7=30C6 3099 +30C9=30C8 3099 +30D0=30CF 3099 +30D1=30CF 309A +30D3=30D2 3099 +30D4=30D2 309A +30D6=30D5 3099 +30D7=30D5 309A +30D9=30D8 3099 +30DA=30D8 309A +30DC=30DB 3099 +30DD=30DB 309A +30F4=30A6 3099 +30F7=30EF 3099 +30F8=30F0 3099 +30F9=30F1 3099 +30FA=30F2 3099 +30FE=30FD 3099 +30FF>30B3 30C8 +3131>1100 +3132>1101 +3133>11AA +3134>1102 +3135>11AC +3136>11AD +3137>1103 +3138>1104 +3139>1105 +313A>11B0 +313B>11B1 +313C>11B2 +313D>11B3 +313E>11B4 +313F>11B5 +3140>111A +3141>1106 +3142>1107 +3143>1108 +3144>1121 +3145>1109 +3146>110A +3147>110B +3148>110C +3149>110D +314A>110E +314B>110F +314C>1110 +314D>1111 +314E>1112 +314F>1161 +3150>1162 +3151>1163 +3152>1164 +3153>1165 +3154>1166 +3155>1167 +3156>1168 +3157>1169 +3158>116A +3159>116B +315A>116C +315B>116D +315C>116E +315D>116F +315E>1170 +315F>1171 +3160>1172 +3161>1173 +3162>1174 +3163>1175 +3164>1160 +3165>1114 +3166>1115 +3167>11C7 +3168>11C8 +3169>11CC +316A>11CE +316B>11D3 +316C>11D7 +316D>11D9 +316E>111C +316F>11DD +3170>11DF +3171>111D +3172>111E +3173>1120 +3174>1122 +3175>1123 +3176>1127 +3177>1129 +3178>112B +3179>112C +317A>112D +317B>112E +317C>112F +317D>1132 +317E>1136 +317F>1140 +3180>1147 +3181>114C +3182>11F1 +3183>11F2 +3184>1157 +3185>1158 +3186>1159 +3187>1184 +3188>1185 +3189>1188 +318A>1191 +318B>1192 +318C>1194 +318D>119E +318E>11A1 +3192>4E00 +3193>4E8C +3194>4E09 +3195>56DB +3196>4E0A +3197>4E2D +3198>4E0B +3199>7532 +319A>4E59 +319B>4E19 +319C>4E01 +319D>5929 +319E>5730 +319F>4EBA +3200>0028 1100 0029 +3201>0028 1102 0029 +3202>0028 1103 0029 +3203>0028 1105 0029 +3204>0028 1106 0029 +3205>0028 1107 0029 +3206>0028 1109 0029 +3207>0028 110B 0029 +3208>0028 110C 0029 +3209>0028 110E 0029 +320A>0028 110F 0029 +320B>0028 1110 0029 +320C>0028 1111 0029 +320D>0028 1112 0029 +320E>0028 1100 1161 0029 +320F>0028 1102 1161 0029 +3210>0028 1103 1161 0029 +3211>0028 1105 1161 0029 +3212>0028 1106 1161 0029 +3213>0028 1107 1161 0029 +3214>0028 1109 1161 0029 +3215>0028 110B 1161 0029 +3216>0028 110C 1161 0029 +3217>0028 110E 1161 0029 +3218>0028 110F 1161 0029 +3219>0028 1110 1161 0029 +321A>0028 1111 1161 0029 +321B>0028 1112 1161 0029 +321C>0028 110C 116E 0029 +321D>0028 110B 1169 110C 1165 11AB 0029 +321E>0028 110B 1169 1112 116E 0029 +3220>0028 4E00 0029 +3221>0028 4E8C 0029 +3222>0028 4E09 0029 +3223>0028 56DB 0029 +3224>0028 4E94 0029 +3225>0028 516D 0029 +3226>0028 4E03 0029 +3227>0028 516B 0029 +3228>0028 4E5D 0029 +3229>0028 5341 0029 +322A>0028 6708 0029 +322B>0028 706B 0029 +322C>0028 6C34 0029 +322D>0028 6728 0029 +322E>0028 91D1 0029 +322F>0028 571F 0029 +3230>0028 65E5 0029 +3231>0028 682A 0029 +3232>0028 6709 0029 +3233>0028 793E 0029 +3234>0028 540D 0029 +3235>0028 7279 0029 +3236>0028 8CA1 0029 +3237>0028 795D 0029 +3238>0028 52B4 0029 +3239>0028 4EE3 0029 +323A>0028 547C 0029 +323B>0028 5B66 0029 +323C>0028 76E3 0029 +323D>0028 4F01 0029 +323E>0028 8CC7 0029 +323F>0028 5354 0029 +3240>0028 796D 0029 +3241>0028 4F11 0029 +3242>0028 81EA 0029 +3243>0028 81F3 0029 +3244>554F +3245>5E7C +3246>6587 +3247>7B8F +3250>0050 0054 0045 +3251>0032 0031 +3252>0032 0032 +3253>0032 0033 +3254>0032 0034 +3255>0032 0035 +3256>0032 0036 +3257>0032 0037 +3258>0032 0038 +3259>0032 0039 +325A>0033 0030 +325B>0033 0031 +325C>0033 0032 +325D>0033 0033 +325E>0033 0034 +325F>0033 0035 +3260>1100 +3261>1102 +3262>1103 +3263>1105 +3264>1106 +3265>1107 +3266>1109 +3267>110B +3268>110C +3269>110E +326A>110F +326B>1110 +326C>1111 +326D>1112 +326E>1100 1161 +326F>1102 1161 +3270>1103 1161 +3271>1105 1161 +3272>1106 1161 +3273>1107 1161 +3274>1109 1161 +3275>110B 1161 +3276>110C 1161 +3277>110E 1161 +3278>110F 1161 +3279>1110 1161 +327A>1111 1161 +327B>1112 1161 +327C>110E 1161 11B7 1100 1169 +327D>110C 116E 110B 1174 +327E>110B 116E +3280>4E00 +3281>4E8C +3282>4E09 +3283>56DB +3284>4E94 +3285>516D +3286>4E03 +3287>516B +3288>4E5D +3289>5341 +328A>6708 +328B>706B +328C>6C34 +328D>6728 +328E>91D1 +328F>571F +3290>65E5 +3291>682A +3292>6709 +3293>793E +3294>540D +3295>7279 +3296>8CA1 +3297>795D +3298>52B4 +3299>79D8 +329A>7537 +329B>5973 +329C>9069 +329D>512A +329E>5370 +329F>6CE8 +32A0>9805 +32A1>4F11 +32A2>5199 +32A3>6B63 +32A4>4E0A +32A5>4E2D +32A6>4E0B +32A7>5DE6 +32A8>53F3 +32A9>533B +32AA>5B97 +32AB>5B66 +32AC>76E3 +32AD>4F01 +32AE>8CC7 +32AF>5354 +32B0>591C +32B1>0033 0036 +32B2>0033 0037 +32B3>0033 0038 +32B4>0033 0039 +32B5>0034 0030 +32B6>0034 0031 +32B7>0034 0032 +32B8>0034 0033 +32B9>0034 0034 +32BA>0034 0035 +32BB>0034 0036 +32BC>0034 0037 +32BD>0034 0038 +32BE>0034 0039 +32BF>0035 0030 +32C0>0031 6708 +32C1>0032 6708 +32C2>0033 6708 +32C3>0034 6708 +32C4>0035 6708 +32C5>0036 6708 +32C6>0037 6708 +32C7>0038 6708 +32C8>0039 6708 +32C9>0031 0030 6708 +32CA>0031 0031 6708 +32CB>0031 0032 6708 +32CC>0048 0067 +32CD>0065 0072 0067 +32CE>0065 0056 +32CF>004C 0054 0044 +32D0>30A2 +32D1>30A4 +32D2>30A6 +32D3>30A8 +32D4>30AA +32D5>30AB +32D6>30AD +32D7>30AF +32D8>30B1 +32D9>30B3 +32DA>30B5 +32DB>30B7 +32DC>30B9 +32DD>30BB +32DE>30BD +32DF>30BF +32E0>30C1 +32E1>30C4 +32E2>30C6 +32E3>30C8 +32E4>30CA +32E5>30CB +32E6>30CC +32E7>30CD +32E8>30CE +32E9>30CF +32EA>30D2 +32EB>30D5 +32EC>30D8 +32ED>30DB +32EE>30DE +32EF>30DF +32F0>30E0 +32F1>30E1 +32F2>30E2 +32F3>30E4 +32F4>30E6 +32F5>30E8 +32F6>30E9 +32F7>30EA +32F8>30EB +32F9>30EC +32FA>30ED +32FB>30EF +32FC>30F0 +32FD>30F1 +32FE>30F2 +3300>30A2 30D1 30FC 30C8 +3301>30A2 30EB 30D5 30A1 +3302>30A2 30F3 30DA 30A2 +3303>30A2 30FC 30EB +3304>30A4 30CB 30F3 30B0 +3305>30A4 30F3 30C1 +3306>30A6 30A9 30F3 +3307>30A8 30B9 30AF 30FC 30C9 +3308>30A8 30FC 30AB 30FC +3309>30AA 30F3 30B9 +330A>30AA 30FC 30E0 +330B>30AB 30A4 30EA +330C>30AB 30E9 30C3 30C8 +330D>30AB 30ED 30EA 30FC +330E>30AC 30ED 30F3 +330F>30AC 30F3 30DE +3310>30AE 30AC +3311>30AE 30CB 30FC +3312>30AD 30E5 30EA 30FC +3313>30AE 30EB 30C0 30FC +3314>30AD 30ED +3315>30AD 30ED 30B0 30E9 30E0 +3316>30AD 30ED 30E1 30FC 30C8 30EB +3317>30AD 30ED 30EF 30C3 30C8 +3318>30B0 30E9 30E0 +3319>30B0 30E9 30E0 30C8 30F3 +331A>30AF 30EB 30BC 30A4 30ED +331B>30AF 30ED 30FC 30CD +331C>30B1 30FC 30B9 +331D>30B3 30EB 30CA +331E>30B3 30FC 30DD +331F>30B5 30A4 30AF 30EB +3320>30B5 30F3 30C1 30FC 30E0 +3321>30B7 30EA 30F3 30B0 +3322>30BB 30F3 30C1 +3323>30BB 30F3 30C8 +3324>30C0 30FC 30B9 +3325>30C7 30B7 +3326>30C9 30EB +3327>30C8 30F3 +3328>30CA 30CE +3329>30CE 30C3 30C8 +332A>30CF 30A4 30C4 +332B>30D1 30FC 30BB 30F3 30C8 +332C>30D1 30FC 30C4 +332D>30D0 30FC 30EC 30EB +332E>30D4 30A2 30B9 30C8 30EB +332F>30D4 30AF 30EB +3330>30D4 30B3 +3331>30D3 30EB +3332>30D5 30A1 30E9 30C3 30C9 +3333>30D5 30A3 30FC 30C8 +3334>30D6 30C3 30B7 30A7 30EB +3335>30D5 30E9 30F3 +3336>30D8 30AF 30BF 30FC 30EB +3337>30DA 30BD +3338>30DA 30CB 30D2 +3339>30D8 30EB 30C4 +333A>30DA 30F3 30B9 +333B>30DA 30FC 30B8 +333C>30D9 30FC 30BF +333D>30DD 30A4 30F3 30C8 +333E>30DC 30EB 30C8 +333F>30DB 30F3 +3340>30DD 30F3 30C9 +3341>30DB 30FC 30EB +3342>30DB 30FC 30F3 +3343>30DE 30A4 30AF 30ED +3344>30DE 30A4 30EB +3345>30DE 30C3 30CF +3346>30DE 30EB 30AF +3347>30DE 30F3 30B7 30E7 30F3 +3348>30DF 30AF 30ED 30F3 +3349>30DF 30EA +334A>30DF 30EA 30D0 30FC 30EB +334B>30E1 30AC +334C>30E1 30AC 30C8 30F3 +334D>30E1 30FC 30C8 30EB +334E>30E4 30FC 30C9 +334F>30E4 30FC 30EB +3350>30E6 30A2 30F3 +3351>30EA 30C3 30C8 30EB +3352>30EA 30E9 +3353>30EB 30D4 30FC +3354>30EB 30FC 30D6 30EB +3355>30EC 30E0 +3356>30EC 30F3 30C8 30B2 30F3 +3357>30EF 30C3 30C8 +3358>0030 70B9 +3359>0031 70B9 +335A>0032 70B9 +335B>0033 70B9 +335C>0034 70B9 +335D>0035 70B9 +335E>0036 70B9 +335F>0037 70B9 +3360>0038 70B9 +3361>0039 70B9 +3362>0031 0030 70B9 +3363>0031 0031 70B9 +3364>0031 0032 70B9 +3365>0031 0033 70B9 +3366>0031 0034 70B9 +3367>0031 0035 70B9 +3368>0031 0036 70B9 +3369>0031 0037 70B9 +336A>0031 0038 70B9 +336B>0031 0039 70B9 +336C>0032 0030 70B9 +336D>0032 0031 70B9 +336E>0032 0032 70B9 +336F>0032 0033 70B9 +3370>0032 0034 70B9 +3371>0068 0050 0061 +3372>0064 0061 +3373>0041 0055 +3374>0062 0061 0072 +3375>006F 0056 +3376>0070 0063 +3377>0064 006D +3378>0064 006D 00B2 +3379>0064 006D 00B3 +337A>0049 0055 +337B>5E73 6210 +337C>662D 548C +337D>5927 6B63 +337E>660E 6CBB +337F>682A 5F0F 4F1A 793E +3380>0070 0041 +3381>006E 0041 +3382>03BC 0041 +3383>006D 0041 +3384>006B 0041 +3385>004B 0042 +3386>004D 0042 +3387>0047 0042 +3388>0063 0061 006C +3389>006B 0063 0061 006C +338A>0070 0046 +338B>006E 0046 +338C>03BC 0046 +338D>03BC 0067 +338E>006D 0067 +338F>006B 0067 +3390>0048 007A +3391>006B 0048 007A +3392>004D 0048 007A +3393>0047 0048 007A +3394>0054 0048 007A +3395>03BC 2113 +3396>006D 2113 +3397>0064 2113 +3398>006B 2113 +3399>0066 006D +339A>006E 006D +339B>03BC 006D +339C>006D 006D +339D>0063 006D +339E>006B 006D +339F>006D 006D 00B2 +33A0>0063 006D 00B2 +33A1>006D 00B2 +33A2>006B 006D 00B2 +33A3>006D 006D 00B3 +33A4>0063 006D 00B3 +33A5>006D 00B3 +33A6>006B 006D 00B3 +33A7>006D 2215 0073 +33A8>006D 2215 0073 00B2 +33A9>0050 0061 +33AA>006B 0050 0061 +33AB>004D 0050 0061 +33AC>0047 0050 0061 +33AD>0072 0061 0064 +33AE>0072 0061 0064 2215 0073 +33AF>0072 0061 0064 2215 0073 00B2 +33B0>0070 0073 +33B1>006E 0073 +33B2>03BC 0073 +33B3>006D 0073 +33B4>0070 0056 +33B5>006E 0056 +33B6>03BC 0056 +33B7>006D 0056 +33B8>006B 0056 +33B9>004D 0056 +33BA>0070 0057 +33BB>006E 0057 +33BC>03BC 0057 +33BD>006D 0057 +33BE>006B 0057 +33BF>004D 0057 +33C0>006B 03A9 +33C1>004D 03A9 +33C2>0061 002E 006D 002E +33C3>0042 0071 +33C4>0063 0063 +33C5>0063 0064 +33C6>0043 2215 006B 0067 +33C7>0043 006F 002E +33C8>0064 0042 +33C9>0047 0079 +33CA>0068 0061 +33CB>0048 0050 +33CC>0069 006E +33CD>004B 004B +33CE>004B 004D +33CF>006B 0074 +33D0>006C 006D +33D1>006C 006E +33D2>006C 006F 0067 +33D3>006C 0078 +33D4>006D 0062 +33D5>006D 0069 006C +33D6>006D 006F 006C +33D7>0050 0048 +33D8>0070 002E 006D 002E +33D9>0050 0050 004D +33DA>0050 0052 +33DB>0073 0072 +33DC>0053 0076 +33DD>0057 0062 +33DE>0056 2215 006D +33DF>0041 2215 006D +33E0>0031 65E5 +33E1>0032 65E5 +33E2>0033 65E5 +33E3>0034 65E5 +33E4>0035 65E5 +33E5>0036 65E5 +33E6>0037 65E5 +33E7>0038 65E5 +33E8>0039 65E5 +33E9>0031 0030 65E5 +33EA>0031 0031 65E5 +33EB>0031 0032 65E5 +33EC>0031 0033 65E5 +33ED>0031 0034 65E5 +33EE>0031 0035 65E5 +33EF>0031 0036 65E5 +33F0>0031 0037 65E5 +33F1>0031 0038 65E5 +33F2>0031 0039 65E5 +33F3>0032 0030 65E5 +33F4>0032 0031 65E5 +33F5>0032 0032 65E5 +33F6>0032 0033 65E5 +33F7>0032 0034 65E5 +33F8>0032 0035 65E5 +33F9>0032 0036 65E5 +33FA>0032 0037 65E5 +33FB>0032 0038 65E5 +33FC>0032 0039 65E5 +33FD>0033 0030 65E5 +33FE>0033 0031 65E5 +33FF>0067 0061 006C +A770>A76F +F900>8C48 +F901>66F4 +F902>8ECA +F903>8CC8 +F904>6ED1 +F905>4E32 +F906>53E5 +F907>9F9C +F908>9F9C +F909>5951 +F90A>91D1 +F90B>5587 +F90C>5948 +F90D>61F6 +F90E>7669 +F90F>7F85 +F910>863F +F911>87BA +F912>88F8 +F913>908F +F914>6A02 +F915>6D1B +F916>70D9 +F917>73DE +F918>843D +F919>916A +F91A>99F1 +F91B>4E82 +F91C>5375 +F91D>6B04 +F91E>721B +F91F>862D +F920>9E1E +F921>5D50 +F922>6FEB +F923>85CD +F924>8964 +F925>62C9 +F926>81D8 +F927>881F +F928>5ECA +F929>6717 +F92A>6D6A +F92B>72FC +F92C>90CE +F92D>4F86 +F92E>51B7 +F92F>52DE +F930>64C4 +F931>6AD3 +F932>7210 +F933>76E7 +F934>8001 +F935>8606 +F936>865C +F937>8DEF +F938>9732 +F939>9B6F +F93A>9DFA +F93B>788C +F93C>797F +F93D>7DA0 +F93E>83C9 +F93F>9304 +F940>9E7F +F941>8AD6 +F942>58DF +F943>5F04 +F944>7C60 +F945>807E +F946>7262 +F947>78CA +F948>8CC2 +F949>96F7 +F94A>58D8 +F94B>5C62 +F94C>6A13 +F94D>6DDA +F94E>6F0F +F94F>7D2F +F950>7E37 +F951>964B +F952>52D2 +F953>808B +F954>51DC +F955>51CC +F956>7A1C +F957>7DBE +F958>83F1 +F959>9675 +F95A>8B80 +F95B>62CF +F95C>6A02 +F95D>8AFE +F95E>4E39 +F95F>5BE7 +F960>6012 +F961>7387 +F962>7570 +F963>5317 +F964>78FB +F965>4FBF +F966>5FA9 +F967>4E0D +F968>6CCC +F969>6578 +F96A>7D22 +F96B>53C3 +F96C>585E +F96D>7701 +F96E>8449 +F96F>8AAA +F970>6BBA +F971>8FB0 +F972>6C88 +F973>62FE +F974>82E5 +F975>63A0 +F976>7565 +F977>4EAE +F978>5169 +F979>51C9 +F97A>6881 +F97B>7CE7 +F97C>826F +F97D>8AD2 +F97E>91CF +F97F>52F5 +F980>5442 +F981>5973 +F982>5EEC +F983>65C5 +F984>6FFE +F985>792A +F986>95AD +F987>9A6A +F988>9E97 +F989>9ECE +F98A>529B +F98B>66C6 +F98C>6B77 +F98D>8F62 +F98E>5E74 +F98F>6190 +F990>6200 +F991>649A +F992>6F23 +F993>7149 +F994>7489 +F995>79CA +F996>7DF4 +F997>806F +F998>8F26 +F999>84EE +F99A>9023 +F99B>934A +F99C>5217 +F99D>52A3 +F99E>54BD +F99F>70C8 +F9A0>88C2 +F9A1>8AAA +F9A2>5EC9 +F9A3>5FF5 +F9A4>637B +F9A5>6BAE +F9A6>7C3E +F9A7>7375 +F9A8>4EE4 +F9A9>56F9 +F9AA>5BE7 +F9AB>5DBA +F9AC>601C +F9AD>73B2 +F9AE>7469 +F9AF>7F9A +F9B0>8046 +F9B1>9234 +F9B2>96F6 +F9B3>9748 +F9B4>9818 +F9B5>4F8B +F9B6>79AE +F9B7>91B4 +F9B8>96B8 +F9B9>60E1 +F9BA>4E86 +F9BB>50DA +F9BC>5BEE +F9BD>5C3F +F9BE>6599 +F9BF>6A02 +F9C0>71CE +F9C1>7642 +F9C2>84FC +F9C3>907C +F9C4>9F8D +F9C5>6688 +F9C6>962E +F9C7>5289 +F9C8>677B +F9C9>67F3 +F9CA>6D41 +F9CB>6E9C +F9CC>7409 +F9CD>7559 +F9CE>786B +F9CF>7D10 +F9D0>985E +F9D1>516D +F9D2>622E +F9D3>9678 +F9D4>502B +F9D5>5D19 +F9D6>6DEA +F9D7>8F2A +F9D8>5F8B +F9D9>6144 +F9DA>6817 +F9DB>7387 +F9DC>9686 +F9DD>5229 +F9DE>540F +F9DF>5C65 +F9E0>6613 +F9E1>674E +F9E2>68A8 +F9E3>6CE5 +F9E4>7406 +F9E5>75E2 +F9E6>7F79 +F9E7>88CF +F9E8>88E1 +F9E9>91CC +F9EA>96E2 +F9EB>533F +F9EC>6EBA +F9ED>541D +F9EE>71D0 +F9EF>7498 +F9F0>85FA +F9F1>96A3 +F9F2>9C57 +F9F3>9E9F +F9F4>6797 +F9F5>6DCB +F9F6>81E8 +F9F7>7ACB +F9F8>7B20 +F9F9>7C92 +F9FA>72C0 +F9FB>7099 +F9FC>8B58 +F9FD>4EC0 +F9FE>8336 +F9FF>523A +FA00>5207 +FA01>5EA6 +FA02>62D3 +FA03>7CD6 +FA04>5B85 +FA05>6D1E +FA06>66B4 +FA07>8F3B +FA08>884C +FA09>964D +FA0A>898B +FA0B>5ED3 +FA0C>5140 +FA0D>55C0 +FA10>585A +FA12>6674 +FA15>51DE +FA16>732A +FA17>76CA +FA18>793C +FA19>795E +FA1A>7965 +FA1B>798F +FA1C>9756 +FA1D>7CBE +FA1E>7FBD +FA20>8612 +FA22>8AF8 +FA25>9038 +FA26>90FD +FA2A>98EF +FA2B>98FC +FA2C>9928 +FA2D>9DB4 +FA30>4FAE +FA31>50E7 +FA32>514D +FA33>52C9 +FA34>52E4 +FA35>5351 +FA36>559D +FA37>5606 +FA38>5668 +FA39>5840 +FA3A>58A8 +FA3B>5C64 +FA3C>5C6E +FA3D>6094 +FA3E>6168 +FA3F>618E +FA40>61F2 +FA41>654F +FA42>65E2 +FA43>6691 +FA44>6885 +FA45>6D77 +FA46>6E1A +FA47>6F22 +FA48>716E +FA49>722B +FA4A>7422 +FA4B>7891 +FA4C>793E +FA4D>7949 +FA4E>7948 +FA4F>7950 +FA50>7956 +FA51>795D +FA52>798D +FA53>798E +FA54>7A40 +FA55>7A81 +FA56>7BC0 +FA57>7DF4 +FA58>7E09 +FA59>7E41 +FA5A>7F72 +FA5B>8005 +FA5C>81ED +FA5D>8279 +FA5E>8279 +FA5F>8457 +FA60>8910 +FA61>8996 +FA62>8B01 +FA63>8B39 +FA64>8CD3 +FA65>8D08 +FA66>8FB6 +FA67>9038 +FA68>96E3 +FA69>97FF +FA6A>983B +FA6B>6075 +FA6C>242EE +FA6D>8218 +FA70>4E26 +FA71>51B5 +FA72>5168 +FA73>4F80 +FA74>5145 +FA75>5180 +FA76>52C7 +FA77>52FA +FA78>559D +FA79>5555 +FA7A>5599 +FA7B>55E2 +FA7C>585A +FA7D>58B3 +FA7E>5944 +FA7F>5954 +FA80>5A62 +FA81>5B28 +FA82>5ED2 +FA83>5ED9 +FA84>5F69 +FA85>5FAD +FA86>60D8 +FA87>614E +FA88>6108 +FA89>618E +FA8A>6160 +FA8B>61F2 +FA8C>6234 +FA8D>63C4 +FA8E>641C +FA8F>6452 +FA90>6556 +FA91>6674 +FA92>6717 +FA93>671B +FA94>6756 +FA95>6B79 +FA96>6BBA +FA97>6D41 +FA98>6EDB +FA99>6ECB +FA9A>6F22 +FA9B>701E +FA9C>716E +FA9D>77A7 +FA9E>7235 +FA9F>72AF +FAA0>732A +FAA1>7471 +FAA2>7506 +FAA3>753B +FAA4>761D +FAA5>761F +FAA6>76CA +FAA7>76DB +FAA8>76F4 +FAA9>774A +FAAA>7740 +FAAB>78CC +FAAC>7AB1 +FAAD>7BC0 +FAAE>7C7B +FAAF>7D5B +FAB0>7DF4 +FAB1>7F3E +FAB2>8005 +FAB3>8352 +FAB4>83EF +FAB5>8779 +FAB6>8941 +FAB7>8986 +FAB8>8996 +FAB9>8ABF +FABA>8AF8 +FABB>8ACB +FABC>8B01 +FABD>8AFE +FABE>8AED +FABF>8B39 +FAC0>8B8A +FAC1>8D08 +FAC2>8F38 +FAC3>9072 +FAC4>9199 +FAC5>9276 +FAC6>967C +FAC7>96E3 +FAC8>9756 +FAC9>97DB +FACA>97FF +FACB>980B +FACC>983B +FACD>9B12 +FACE>9F9C +FACF>2284A +FAD0>22844 +FAD1>233D5 +FAD2>3B9D +FAD3>4018 +FAD4>4039 +FAD5>25249 +FAD6>25CD0 +FAD7>27ED3 +FAD8>9F43 +FAD9>9F8E +FB00>0066 0066 +FB01>0066 0069 +FB02>0066 006C +FB03>0066 0066 0069 +FB04>0066 0066 006C +FB05>017F 0074 +FB06>0073 0074 +FB13>0574 0576 +FB14>0574 0565 +FB15>0574 056B +FB16>057E 0576 +FB17>0574 056D +FB1D>05D9 05B4 +FB1F>05F2 05B7 +FB20>05E2 +FB21>05D0 +FB22>05D3 +FB23>05D4 +FB24>05DB +FB25>05DC +FB26>05DD +FB27>05E8 +FB28>05EA +FB29>002B +FB2A>05E9 05C1 +FB2B>05E9 05C2 +FB2C>FB49 05C1 +FB2D>FB49 05C2 +FB2E>05D0 05B7 +FB2F>05D0 05B8 +FB30>05D0 05BC +FB31>05D1 05BC +FB32>05D2 05BC +FB33>05D3 05BC +FB34>05D4 05BC +FB35>05D5 05BC +FB36>05D6 05BC +FB38>05D8 05BC +FB39>05D9 05BC +FB3A>05DA 05BC +FB3B>05DB 05BC +FB3C>05DC 05BC +FB3E>05DE 05BC +FB40>05E0 05BC +FB41>05E1 05BC +FB43>05E3 05BC +FB44>05E4 05BC +FB46>05E6 05BC +FB47>05E7 05BC +FB48>05E8 05BC +FB49>05E9 05BC +FB4A>05EA 05BC +FB4B>05D5 05B9 +FB4C>05D1 05BF +FB4D>05DB 05BF +FB4E>05E4 05BF +FB4F>05D0 05DC +FB50>0671 +FB51>0671 +FB52>067B +FB53>067B +FB54>067B +FB55>067B +FB56>067E +FB57>067E +FB58>067E +FB59>067E +FB5A>0680 +FB5B>0680 +FB5C>0680 +FB5D>0680 +FB5E>067A +FB5F>067A +FB60>067A +FB61>067A +FB62>067F +FB63>067F +FB64>067F +FB65>067F +FB66>0679 +FB67>0679 +FB68>0679 +FB69>0679 +FB6A>06A4 +FB6B>06A4 +FB6C>06A4 +FB6D>06A4 +FB6E>06A6 +FB6F>06A6 +FB70>06A6 +FB71>06A6 +FB72>0684 +FB73>0684 +FB74>0684 +FB75>0684 +FB76>0683 +FB77>0683 +FB78>0683 +FB79>0683 +FB7A>0686 +FB7B>0686 +FB7C>0686 +FB7D>0686 +FB7E>0687 +FB7F>0687 +FB80>0687 +FB81>0687 +FB82>068D +FB83>068D +FB84>068C +FB85>068C +FB86>068E +FB87>068E +FB88>0688 +FB89>0688 +FB8A>0698 +FB8B>0698 +FB8C>0691 +FB8D>0691 +FB8E>06A9 +FB8F>06A9 +FB90>06A9 +FB91>06A9 +FB92>06AF +FB93>06AF +FB94>06AF +FB95>06AF +FB96>06B3 +FB97>06B3 +FB98>06B3 +FB99>06B3 +FB9A>06B1 +FB9B>06B1 +FB9C>06B1 +FB9D>06B1 +FB9E>06BA +FB9F>06BA +FBA0>06BB +FBA1>06BB +FBA2>06BB +FBA3>06BB +FBA4>06C0 +FBA5>06C0 +FBA6>06C1 +FBA7>06C1 +FBA8>06C1 +FBA9>06C1 +FBAA>06BE +FBAB>06BE +FBAC>06BE +FBAD>06BE +FBAE>06D2 +FBAF>06D2 +FBB0>06D3 +FBB1>06D3 +FBD3>06AD +FBD4>06AD +FBD5>06AD +FBD6>06AD +FBD7>06C7 +FBD8>06C7 +FBD9>06C6 +FBDA>06C6 +FBDB>06C8 +FBDC>06C8 +FBDD>0677 +FBDE>06CB +FBDF>06CB +FBE0>06C5 +FBE1>06C5 +FBE2>06C9 +FBE3>06C9 +FBE4>06D0 +FBE5>06D0 +FBE6>06D0 +FBE7>06D0 +FBE8>0649 +FBE9>0649 +FBEA>0626 0627 +FBEB>0626 0627 +FBEC>0626 06D5 +FBED>0626 06D5 +FBEE>0626 0648 +FBEF>0626 0648 +FBF0>0626 06C7 +FBF1>0626 06C7 +FBF2>0626 06C6 +FBF3>0626 06C6 +FBF4>0626 06C8 +FBF5>0626 06C8 +FBF6>0626 06D0 +FBF7>0626 06D0 +FBF8>0626 06D0 +FBF9>0626 0649 +FBFA>0626 0649 +FBFB>0626 0649 +FBFC>06CC +FBFD>06CC +FBFE>06CC +FBFF>06CC +FC00>0626 062C +FC01>0626 062D +FC02>0626 0645 +FC03>0626 0649 +FC04>0626 064A +FC05>0628 062C +FC06>0628 062D +FC07>0628 062E +FC08>0628 0645 +FC09>0628 0649 +FC0A>0628 064A +FC0B>062A 062C +FC0C>062A 062D +FC0D>062A 062E +FC0E>062A 0645 +FC0F>062A 0649 +FC10>062A 064A +FC11>062B 062C +FC12>062B 0645 +FC13>062B 0649 +FC14>062B 064A +FC15>062C 062D +FC16>062C 0645 +FC17>062D 062C +FC18>062D 0645 +FC19>062E 062C +FC1A>062E 062D +FC1B>062E 0645 +FC1C>0633 062C +FC1D>0633 062D +FC1E>0633 062E +FC1F>0633 0645 +FC20>0635 062D +FC21>0635 0645 +FC22>0636 062C +FC23>0636 062D +FC24>0636 062E +FC25>0636 0645 +FC26>0637 062D +FC27>0637 0645 +FC28>0638 0645 +FC29>0639 062C +FC2A>0639 0645 +FC2B>063A 062C +FC2C>063A 0645 +FC2D>0641 062C +FC2E>0641 062D +FC2F>0641 062E +FC30>0641 0645 +FC31>0641 0649 +FC32>0641 064A +FC33>0642 062D +FC34>0642 0645 +FC35>0642 0649 +FC36>0642 064A +FC37>0643 0627 +FC38>0643 062C +FC39>0643 062D +FC3A>0643 062E +FC3B>0643 0644 +FC3C>0643 0645 +FC3D>0643 0649 +FC3E>0643 064A +FC3F>0644 062C +FC40>0644 062D +FC41>0644 062E +FC42>0644 0645 +FC43>0644 0649 +FC44>0644 064A +FC45>0645 062C +FC46>0645 062D +FC47>0645 062E +FC48>0645 0645 +FC49>0645 0649 +FC4A>0645 064A +FC4B>0646 062C +FC4C>0646 062D +FC4D>0646 062E +FC4E>0646 0645 +FC4F>0646 0649 +FC50>0646 064A +FC51>0647 062C +FC52>0647 0645 +FC53>0647 0649 +FC54>0647 064A +FC55>064A 062C +FC56>064A 062D +FC57>064A 062E +FC58>064A 0645 +FC59>064A 0649 +FC5A>064A 064A +FC5B>0630 0670 +FC5C>0631 0670 +FC5D>0649 0670 +FC5E>0020 064C 0651 +FC5F>0020 064D 0651 +FC60>0020 064E 0651 +FC61>0020 064F 0651 +FC62>0020 0650 0651 +FC63>0020 0651 0670 +FC64>0626 0631 +FC65>0626 0632 +FC66>0626 0645 +FC67>0626 0646 +FC68>0626 0649 +FC69>0626 064A +FC6A>0628 0631 +FC6B>0628 0632 +FC6C>0628 0645 +FC6D>0628 0646 +FC6E>0628 0649 +FC6F>0628 064A +FC70>062A 0631 +FC71>062A 0632 +FC72>062A 0645 +FC73>062A 0646 +FC74>062A 0649 +FC75>062A 064A +FC76>062B 0631 +FC77>062B 0632 +FC78>062B 0645 +FC79>062B 0646 +FC7A>062B 0649 +FC7B>062B 064A +FC7C>0641 0649 +FC7D>0641 064A +FC7E>0642 0649 +FC7F>0642 064A +FC80>0643 0627 +FC81>0643 0644 +FC82>0643 0645 +FC83>0643 0649 +FC84>0643 064A +FC85>0644 0645 +FC86>0644 0649 +FC87>0644 064A +FC88>0645 0627 +FC89>0645 0645 +FC8A>0646 0631 +FC8B>0646 0632 +FC8C>0646 0645 +FC8D>0646 0646 +FC8E>0646 0649 +FC8F>0646 064A +FC90>0649 0670 +FC91>064A 0631 +FC92>064A 0632 +FC93>064A 0645 +FC94>064A 0646 +FC95>064A 0649 +FC96>064A 064A +FC97>0626 062C +FC98>0626 062D +FC99>0626 062E +FC9A>0626 0645 +FC9B>0626 0647 +FC9C>0628 062C +FC9D>0628 062D +FC9E>0628 062E +FC9F>0628 0645 +FCA0>0628 0647 +FCA1>062A 062C +FCA2>062A 062D +FCA3>062A 062E +FCA4>062A 0645 +FCA5>062A 0647 +FCA6>062B 0645 +FCA7>062C 062D +FCA8>062C 0645 +FCA9>062D 062C +FCAA>062D 0645 +FCAB>062E 062C +FCAC>062E 0645 +FCAD>0633 062C +FCAE>0633 062D +FCAF>0633 062E +FCB0>0633 0645 +FCB1>0635 062D +FCB2>0635 062E +FCB3>0635 0645 +FCB4>0636 062C +FCB5>0636 062D +FCB6>0636 062E +FCB7>0636 0645 +FCB8>0637 062D +FCB9>0638 0645 +FCBA>0639 062C +FCBB>0639 0645 +FCBC>063A 062C +FCBD>063A 0645 +FCBE>0641 062C +FCBF>0641 062D +FCC0>0641 062E +FCC1>0641 0645 +FCC2>0642 062D +FCC3>0642 0645 +FCC4>0643 062C +FCC5>0643 062D +FCC6>0643 062E +FCC7>0643 0644 +FCC8>0643 0645 +FCC9>0644 062C +FCCA>0644 062D +FCCB>0644 062E +FCCC>0644 0645 +FCCD>0644 0647 +FCCE>0645 062C +FCCF>0645 062D +FCD0>0645 062E +FCD1>0645 0645 +FCD2>0646 062C +FCD3>0646 062D +FCD4>0646 062E +FCD5>0646 0645 +FCD6>0646 0647 +FCD7>0647 062C +FCD8>0647 0645 +FCD9>0647 0670 +FCDA>064A 062C +FCDB>064A 062D +FCDC>064A 062E +FCDD>064A 0645 +FCDE>064A 0647 +FCDF>0626 0645 +FCE0>0626 0647 +FCE1>0628 0645 +FCE2>0628 0647 +FCE3>062A 0645 +FCE4>062A 0647 +FCE5>062B 0645 +FCE6>062B 0647 +FCE7>0633 0645 +FCE8>0633 0647 +FCE9>0634 0645 +FCEA>0634 0647 +FCEB>0643 0644 +FCEC>0643 0645 +FCED>0644 0645 +FCEE>0646 0645 +FCEF>0646 0647 +FCF0>064A 0645 +FCF1>064A 0647 +FCF2>0640 064E 0651 +FCF3>0640 064F 0651 +FCF4>0640 0650 0651 +FCF5>0637 0649 +FCF6>0637 064A +FCF7>0639 0649 +FCF8>0639 064A +FCF9>063A 0649 +FCFA>063A 064A +FCFB>0633 0649 +FCFC>0633 064A +FCFD>0634 0649 +FCFE>0634 064A +FCFF>062D 0649 +FD00>062D 064A +FD01>062C 0649 +FD02>062C 064A +FD03>062E 0649 +FD04>062E 064A +FD05>0635 0649 +FD06>0635 064A +FD07>0636 0649 +FD08>0636 064A +FD09>0634 062C +FD0A>0634 062D +FD0B>0634 062E +FD0C>0634 0645 +FD0D>0634 0631 +FD0E>0633 0631 +FD0F>0635 0631 +FD10>0636 0631 +FD11>0637 0649 +FD12>0637 064A +FD13>0639 0649 +FD14>0639 064A +FD15>063A 0649 +FD16>063A 064A +FD17>0633 0649 +FD18>0633 064A +FD19>0634 0649 +FD1A>0634 064A +FD1B>062D 0649 +FD1C>062D 064A +FD1D>062C 0649 +FD1E>062C 064A +FD1F>062E 0649 +FD20>062E 064A +FD21>0635 0649 +FD22>0635 064A +FD23>0636 0649 +FD24>0636 064A +FD25>0634 062C +FD26>0634 062D +FD27>0634 062E +FD28>0634 0645 +FD29>0634 0631 +FD2A>0633 0631 +FD2B>0635 0631 +FD2C>0636 0631 +FD2D>0634 062C +FD2E>0634 062D +FD2F>0634 062E +FD30>0634 0645 +FD31>0633 0647 +FD32>0634 0647 +FD33>0637 0645 +FD34>0633 062C +FD35>0633 062D +FD36>0633 062E +FD37>0634 062C +FD38>0634 062D +FD39>0634 062E +FD3A>0637 0645 +FD3B>0638 0645 +FD3C>0627 064B +FD3D>0627 064B +FD50>062A 062C 0645 +FD51>062A 062D 062C +FD52>062A 062D 062C +FD53>062A 062D 0645 +FD54>062A 062E 0645 +FD55>062A 0645 062C +FD56>062A 0645 062D +FD57>062A 0645 062E +FD58>062C 0645 062D +FD59>062C 0645 062D +FD5A>062D 0645 064A +FD5B>062D 0645 0649 +FD5C>0633 062D 062C +FD5D>0633 062C 062D +FD5E>0633 062C 0649 +FD5F>0633 0645 062D +FD60>0633 0645 062D +FD61>0633 0645 062C +FD62>0633 0645 0645 +FD63>0633 0645 0645 +FD64>0635 062D 062D +FD65>0635 062D 062D +FD66>0635 0645 0645 +FD67>0634 062D 0645 +FD68>0634 062D 0645 +FD69>0634 062C 064A +FD6A>0634 0645 062E +FD6B>0634 0645 062E +FD6C>0634 0645 0645 +FD6D>0634 0645 0645 +FD6E>0636 062D 0649 +FD6F>0636 062E 0645 +FD70>0636 062E 0645 +FD71>0637 0645 062D +FD72>0637 0645 062D +FD73>0637 0645 0645 +FD74>0637 0645 064A +FD75>0639 062C 0645 +FD76>0639 0645 0645 +FD77>0639 0645 0645 +FD78>0639 0645 0649 +FD79>063A 0645 0645 +FD7A>063A 0645 064A +FD7B>063A 0645 0649 +FD7C>0641 062E 0645 +FD7D>0641 062E 0645 +FD7E>0642 0645 062D +FD7F>0642 0645 0645 +FD80>0644 062D 0645 +FD81>0644 062D 064A +FD82>0644 062D 0649 +FD83>0644 062C 062C +FD84>0644 062C 062C +FD85>0644 062E 0645 +FD86>0644 062E 0645 +FD87>0644 0645 062D +FD88>0644 0645 062D +FD89>0645 062D 062C +FD8A>0645 062D 0645 +FD8B>0645 062D 064A +FD8C>0645 062C 062D +FD8D>0645 062C 0645 +FD8E>0645 062E 062C +FD8F>0645 062E 0645 +FD92>0645 062C 062E +FD93>0647 0645 062C +FD94>0647 0645 0645 +FD95>0646 062D 0645 +FD96>0646 062D 0649 +FD97>0646 062C 0645 +FD98>0646 062C 0645 +FD99>0646 062C 0649 +FD9A>0646 0645 064A +FD9B>0646 0645 0649 +FD9C>064A 0645 0645 +FD9D>064A 0645 0645 +FD9E>0628 062E 064A +FD9F>062A 062C 064A +FDA0>062A 062C 0649 +FDA1>062A 062E 064A +FDA2>062A 062E 0649 +FDA3>062A 0645 064A +FDA4>062A 0645 0649 +FDA5>062C 0645 064A +FDA6>062C 062D 0649 +FDA7>062C 0645 0649 +FDA8>0633 062E 0649 +FDA9>0635 062D 064A +FDAA>0634 062D 064A +FDAB>0636 062D 064A +FDAC>0644 062C 064A +FDAD>0644 0645 064A +FDAE>064A 062D 064A +FDAF>064A 062C 064A +FDB0>064A 0645 064A +FDB1>0645 0645 064A +FDB2>0642 0645 064A +FDB3>0646 062D 064A +FDB4>0642 0645 062D +FDB5>0644 062D 0645 +FDB6>0639 0645 064A +FDB7>0643 0645 064A +FDB8>0646 062C 062D +FDB9>0645 062E 064A +FDBA>0644 062C 0645 +FDBB>0643 0645 0645 +FDBC>0644 062C 0645 +FDBD>0646 062C 062D +FDBE>062C 062D 064A +FDBF>062D 062C 064A +FDC0>0645 062C 064A +FDC1>0641 0645 064A +FDC2>0628 062D 064A +FDC3>0643 0645 0645 +FDC4>0639 062C 0645 +FDC5>0635 0645 0645 +FDC6>0633 062E 064A +FDC7>0646 062C 064A +FDF0>0635 0644 06D2 +FDF1>0642 0644 06D2 +FDF2>0627 0644 0644 0647 +FDF3>0627 0643 0628 0631 +FDF4>0645 062D 0645 062F +FDF5>0635 0644 0639 0645 +FDF6>0631 0633 0648 0644 +FDF7>0639 0644 064A 0647 +FDF8>0648 0633 0644 0645 +FDF9>0635 0644 0649 +FDFA>0635 0644 0649 0020 0627 0644 0644 0647 0020 0639 0644 064A 0647 0020 0648 0633 0644 0645 +FDFB>062C 0644 0020 062C 0644 0627 0644 0647 +FDFC>0631 06CC 0627 0644 +FE10>002C +FE11>3001 +FE12>3002 +FE13>003A +FE14>003B +FE15>0021 +FE16>003F +FE17>3016 +FE18>3017 +FE19>2026 +FE30>2025 +FE31>2014 +FE32>2013 +FE33>005F +FE34>005F +FE35>0028 +FE36>0029 +FE37>007B +FE38>007D +FE39>3014 +FE3A>3015 +FE3B>3010 +FE3C>3011 +FE3D>300A +FE3E>300B +FE3F>3008 +FE40>3009 +FE41>300C +FE42>300D +FE43>300E +FE44>300F +FE47>005B +FE48>005D +FE49>203E +FE4A>203E +FE4B>203E +FE4C>203E +FE4D>005F +FE4E>005F +FE4F>005F +FE50>002C +FE51>3001 +FE52>002E +FE54>003B +FE55>003A +FE56>003F +FE57>0021 +FE58>2014 +FE59>0028 +FE5A>0029 +FE5B>007B +FE5C>007D +FE5D>3014 +FE5E>3015 +FE5F>0023 +FE60>0026 +FE61>002A +FE62>002B +FE63>002D +FE64>003C +FE65>003E +FE66>003D +FE68>005C +FE69>0024 +FE6A>0025 +FE6B>0040 +FE70>0020 064B +FE71>0640 064B +FE72>0020 064C +FE74>0020 064D +FE76>0020 064E +FE77>0640 064E +FE78>0020 064F +FE79>0640 064F +FE7A>0020 0650 +FE7B>0640 0650 +FE7C>0020 0651 +FE7D>0640 0651 +FE7E>0020 0652 +FE7F>0640 0652 +FE80>0621 +FE81>0622 +FE82>0622 +FE83>0623 +FE84>0623 +FE85>0624 +FE86>0624 +FE87>0625 +FE88>0625 +FE89>0626 +FE8A>0626 +FE8B>0626 +FE8C>0626 +FE8D>0627 +FE8E>0627 +FE8F>0628 +FE90>0628 +FE91>0628 +FE92>0628 +FE93>0629 +FE94>0629 +FE95>062A +FE96>062A +FE97>062A +FE98>062A +FE99>062B +FE9A>062B +FE9B>062B +FE9C>062B +FE9D>062C +FE9E>062C +FE9F>062C +FEA0>062C +FEA1>062D +FEA2>062D +FEA3>062D +FEA4>062D +FEA5>062E +FEA6>062E +FEA7>062E +FEA8>062E +FEA9>062F +FEAA>062F +FEAB>0630 +FEAC>0630 +FEAD>0631 +FEAE>0631 +FEAF>0632 +FEB0>0632 +FEB1>0633 +FEB2>0633 +FEB3>0633 +FEB4>0633 +FEB5>0634 +FEB6>0634 +FEB7>0634 +FEB8>0634 +FEB9>0635 +FEBA>0635 +FEBB>0635 +FEBC>0635 +FEBD>0636 +FEBE>0636 +FEBF>0636 +FEC0>0636 +FEC1>0637 +FEC2>0637 +FEC3>0637 +FEC4>0637 +FEC5>0638 +FEC6>0638 +FEC7>0638 +FEC8>0638 +FEC9>0639 +FECA>0639 +FECB>0639 +FECC>0639 +FECD>063A +FECE>063A +FECF>063A +FED0>063A +FED1>0641 +FED2>0641 +FED3>0641 +FED4>0641 +FED5>0642 +FED6>0642 +FED7>0642 +FED8>0642 +FED9>0643 +FEDA>0643 +FEDB>0643 +FEDC>0643 +FEDD>0644 +FEDE>0644 +FEDF>0644 +FEE0>0644 +FEE1>0645 +FEE2>0645 +FEE3>0645 +FEE4>0645 +FEE5>0646 +FEE6>0646 +FEE7>0646 +FEE8>0646 +FEE9>0647 +FEEA>0647 +FEEB>0647 +FEEC>0647 +FEED>0648 +FEEE>0648 +FEEF>0649 +FEF0>0649 +FEF1>064A +FEF2>064A +FEF3>064A +FEF4>064A +FEF5>0644 0622 +FEF6>0644 0622 +FEF7>0644 0623 +FEF8>0644 0623 +FEF9>0644 0625 +FEFA>0644 0625 +FEFB>0644 0627 +FEFC>0644 0627 +FF01>0021 +FF02>0022 +FF03>0023 +FF04>0024 +FF05>0025 +FF06>0026 +FF07>0027 +FF08>0028 +FF09>0029 +FF0A>002A +FF0B>002B +FF0C>002C +FF0D>002D +FF0E>002E +FF0F>002F +FF10>0030 +FF11>0031 +FF12>0032 +FF13>0033 +FF14>0034 +FF15>0035 +FF16>0036 +FF17>0037 +FF18>0038 +FF19>0039 +FF1A>003A +FF1B>003B +FF1C>003C +FF1D>003D +FF1E>003E +FF1F>003F +FF20>0040 +FF21>0041 +FF22>0042 +FF23>0043 +FF24>0044 +FF25>0045 +FF26>0046 +FF27>0047 +FF28>0048 +FF29>0049 +FF2A>004A +FF2B>004B +FF2C>004C +FF2D>004D +FF2E>004E +FF2F>004F +FF30>0050 +FF31>0051 +FF32>0052 +FF33>0053 +FF34>0054 +FF35>0055 +FF36>0056 +FF37>0057 +FF38>0058 +FF39>0059 +FF3A>005A +FF3B>005B +FF3C>005C +FF3D>005D +FF3E>005E +FF3F>005F +FF40>0060 +FF41>0061 +FF42>0062 +FF43>0063 +FF44>0064 +FF45>0065 +FF46>0066 +FF47>0067 +FF48>0068 +FF49>0069 +FF4A>006A +FF4B>006B +FF4C>006C +FF4D>006D +FF4E>006E +FF4F>006F +FF50>0070 +FF51>0071 +FF52>0072 +FF53>0073 +FF54>0074 +FF55>0075 +FF56>0076 +FF57>0077 +FF58>0078 +FF59>0079 +FF5A>007A +FF5B>007B +FF5C>007C +FF5D>007D +FF5E>007E +FF5F>2985 +FF60>2986 +FF61>3002 +FF62>300C +FF63>300D +FF64>3001 +FF65>30FB +FF66>30F2 +FF67>30A1 +FF68>30A3 +FF69>30A5 +FF6A>30A7 +FF6B>30A9 +FF6C>30E3 +FF6D>30E5 +FF6E>30E7 +FF6F>30C3 +FF70>30FC +FF71>30A2 +FF72>30A4 +FF73>30A6 +FF74>30A8 +FF75>30AA +FF76>30AB +FF77>30AD +FF78>30AF +FF79>30B1 +FF7A>30B3 +FF7B>30B5 +FF7C>30B7 +FF7D>30B9 +FF7E>30BB +FF7F>30BD +FF80>30BF +FF81>30C1 +FF82>30C4 +FF83>30C6 +FF84>30C8 +FF85>30CA +FF86>30CB +FF87>30CC +FF88>30CD +FF89>30CE +FF8A>30CF +FF8B>30D2 +FF8C>30D5 +FF8D>30D8 +FF8E>30DB +FF8F>30DE +FF90>30DF +FF91>30E0 +FF92>30E1 +FF93>30E2 +FF94>30E4 +FF95>30E6 +FF96>30E8 +FF97>30E9 +FF98>30EA +FF99>30EB +FF9A>30EC +FF9B>30ED +FF9C>30EF +FF9D>30F3 +FF9E>3099 +FF9F>309A +FFA0>3164 +FFA1>3131 +FFA2>3132 +FFA3>3133 +FFA4>3134 +FFA5>3135 +FFA6>3136 +FFA7>3137 +FFA8>3138 +FFA9>3139 +FFAA>313A +FFAB>313B +FFAC>313C +FFAD>313D +FFAE>313E +FFAF>313F +FFB0>3140 +FFB1>3141 +FFB2>3142 +FFB3>3143 +FFB4>3144 +FFB5>3145 +FFB6>3146 +FFB7>3147 +FFB8>3148 +FFB9>3149 +FFBA>314A +FFBB>314B +FFBC>314C +FFBD>314D +FFBE>314E +FFC2>314F +FFC3>3150 +FFC4>3151 +FFC5>3152 +FFC6>3153 +FFC7>3154 +FFCA>3155 +FFCB>3156 +FFCC>3157 +FFCD>3158 +FFCE>3159 +FFCF>315A +FFD2>315B +FFD3>315C +FFD4>315D +FFD5>315E +FFD6>315F +FFD7>3160 +FFDA>3161 +FFDB>3162 +FFDC>3163 +FFE0>00A2 +FFE1>00A3 +FFE2>00AC +FFE3>00AF +FFE4>00A6 +FFE5>00A5 +FFE6>20A9 +FFE8>2502 +FFE9>2190 +FFEA>2191 +FFEB>2192 +FFEC>2193 +FFED>25A0 +FFEE>25CB +1109A=11099 110BA +1109C=1109B 110BA +110AB=110A5 110BA +1D15E>1D157 1D165 +1D15F>1D158 1D165 +1D160>1D15F 1D16E +1D161>1D15F 1D16F +1D162>1D15F 1D170 +1D163>1D15F 1D171 +1D164>1D15F 1D172 +1D1BB>1D1B9 1D165 +1D1BC>1D1BA 1D165 +1D1BD>1D1BB 1D16E +1D1BE>1D1BC 1D16E +1D1BF>1D1BB 1D16F +1D1C0>1D1BC 1D16F +1D400>0041 +1D401>0042 +1D402>0043 +1D403>0044 +1D404>0045 +1D405>0046 +1D406>0047 +1D407>0048 +1D408>0049 +1D409>004A +1D40A>004B +1D40B>004C +1D40C>004D +1D40D>004E +1D40E>004F +1D40F>0050 +1D410>0051 +1D411>0052 +1D412>0053 +1D413>0054 +1D414>0055 +1D415>0056 +1D416>0057 +1D417>0058 +1D418>0059 +1D419>005A +1D41A>0061 +1D41B>0062 +1D41C>0063 +1D41D>0064 +1D41E>0065 +1D41F>0066 +1D420>0067 +1D421>0068 +1D422>0069 +1D423>006A +1D424>006B +1D425>006C +1D426>006D +1D427>006E +1D428>006F +1D429>0070 +1D42A>0071 +1D42B>0072 +1D42C>0073 +1D42D>0074 +1D42E>0075 +1D42F>0076 +1D430>0077 +1D431>0078 +1D432>0079 +1D433>007A +1D434>0041 +1D435>0042 +1D436>0043 +1D437>0044 +1D438>0045 +1D439>0046 +1D43A>0047 +1D43B>0048 +1D43C>0049 +1D43D>004A +1D43E>004B +1D43F>004C +1D440>004D +1D441>004E +1D442>004F +1D443>0050 +1D444>0051 +1D445>0052 +1D446>0053 +1D447>0054 +1D448>0055 +1D449>0056 +1D44A>0057 +1D44B>0058 +1D44C>0059 +1D44D>005A +1D44E>0061 +1D44F>0062 +1D450>0063 +1D451>0064 +1D452>0065 +1D453>0066 +1D454>0067 +1D456>0069 +1D457>006A +1D458>006B +1D459>006C +1D45A>006D +1D45B>006E +1D45C>006F +1D45D>0070 +1D45E>0071 +1D45F>0072 +1D460>0073 +1D461>0074 +1D462>0075 +1D463>0076 +1D464>0077 +1D465>0078 +1D466>0079 +1D467>007A +1D468>0041 +1D469>0042 +1D46A>0043 +1D46B>0044 +1D46C>0045 +1D46D>0046 +1D46E>0047 +1D46F>0048 +1D470>0049 +1D471>004A +1D472>004B +1D473>004C +1D474>004D +1D475>004E +1D476>004F +1D477>0050 +1D478>0051 +1D479>0052 +1D47A>0053 +1D47B>0054 +1D47C>0055 +1D47D>0056 +1D47E>0057 +1D47F>0058 +1D480>0059 +1D481>005A +1D482>0061 +1D483>0062 +1D484>0063 +1D485>0064 +1D486>0065 +1D487>0066 +1D488>0067 +1D489>0068 +1D48A>0069 +1D48B>006A +1D48C>006B +1D48D>006C +1D48E>006D +1D48F>006E +1D490>006F +1D491>0070 +1D492>0071 +1D493>0072 +1D494>0073 +1D495>0074 +1D496>0075 +1D497>0076 +1D498>0077 +1D499>0078 +1D49A>0079 +1D49B>007A +1D49C>0041 +1D49E>0043 +1D49F>0044 +1D4A2>0047 +1D4A5>004A +1D4A6>004B +1D4A9>004E +1D4AA>004F +1D4AB>0050 +1D4AC>0051 +1D4AE>0053 +1D4AF>0054 +1D4B0>0055 +1D4B1>0056 +1D4B2>0057 +1D4B3>0058 +1D4B4>0059 +1D4B5>005A +1D4B6>0061 +1D4B7>0062 +1D4B8>0063 +1D4B9>0064 +1D4BB>0066 +1D4BD>0068 +1D4BE>0069 +1D4BF>006A +1D4C0>006B +1D4C1>006C +1D4C2>006D +1D4C3>006E +1D4C5>0070 +1D4C6>0071 +1D4C7>0072 +1D4C8>0073 +1D4C9>0074 +1D4CA>0075 +1D4CB>0076 +1D4CC>0077 +1D4CD>0078 +1D4CE>0079 +1D4CF>007A +1D4D0>0041 +1D4D1>0042 +1D4D2>0043 +1D4D3>0044 +1D4D4>0045 +1D4D5>0046 +1D4D6>0047 +1D4D7>0048 +1D4D8>0049 +1D4D9>004A +1D4DA>004B +1D4DB>004C +1D4DC>004D +1D4DD>004E +1D4DE>004F +1D4DF>0050 +1D4E0>0051 +1D4E1>0052 +1D4E2>0053 +1D4E3>0054 +1D4E4>0055 +1D4E5>0056 +1D4E6>0057 +1D4E7>0058 +1D4E8>0059 +1D4E9>005A +1D4EA>0061 +1D4EB>0062 +1D4EC>0063 +1D4ED>0064 +1D4EE>0065 +1D4EF>0066 +1D4F0>0067 +1D4F1>0068 +1D4F2>0069 +1D4F3>006A +1D4F4>006B +1D4F5>006C +1D4F6>006D +1D4F7>006E +1D4F8>006F +1D4F9>0070 +1D4FA>0071 +1D4FB>0072 +1D4FC>0073 +1D4FD>0074 +1D4FE>0075 +1D4FF>0076 +1D500>0077 +1D501>0078 +1D502>0079 +1D503>007A +1D504>0041 +1D505>0042 +1D507>0044 +1D508>0045 +1D509>0046 +1D50A>0047 +1D50D>004A +1D50E>004B +1D50F>004C +1D510>004D +1D511>004E +1D512>004F +1D513>0050 +1D514>0051 +1D516>0053 +1D517>0054 +1D518>0055 +1D519>0056 +1D51A>0057 +1D51B>0058 +1D51C>0059 +1D51E>0061 +1D51F>0062 +1D520>0063 +1D521>0064 +1D522>0065 +1D523>0066 +1D524>0067 +1D525>0068 +1D526>0069 +1D527>006A +1D528>006B +1D529>006C +1D52A>006D +1D52B>006E +1D52C>006F +1D52D>0070 +1D52E>0071 +1D52F>0072 +1D530>0073 +1D531>0074 +1D532>0075 +1D533>0076 +1D534>0077 +1D535>0078 +1D536>0079 +1D537>007A +1D538>0041 +1D539>0042 +1D53B>0044 +1D53C>0045 +1D53D>0046 +1D53E>0047 +1D540>0049 +1D541>004A +1D542>004B +1D543>004C +1D544>004D +1D546>004F +1D54A>0053 +1D54B>0054 +1D54C>0055 +1D54D>0056 +1D54E>0057 +1D54F>0058 +1D550>0059 +1D552>0061 +1D553>0062 +1D554>0063 +1D555>0064 +1D556>0065 +1D557>0066 +1D558>0067 +1D559>0068 +1D55A>0069 +1D55B>006A +1D55C>006B +1D55D>006C +1D55E>006D +1D55F>006E +1D560>006F +1D561>0070 +1D562>0071 +1D563>0072 +1D564>0073 +1D565>0074 +1D566>0075 +1D567>0076 +1D568>0077 +1D569>0078 +1D56A>0079 +1D56B>007A +1D56C>0041 +1D56D>0042 +1D56E>0043 +1D56F>0044 +1D570>0045 +1D571>0046 +1D572>0047 +1D573>0048 +1D574>0049 +1D575>004A +1D576>004B +1D577>004C +1D578>004D +1D579>004E +1D57A>004F +1D57B>0050 +1D57C>0051 +1D57D>0052 +1D57E>0053 +1D57F>0054 +1D580>0055 +1D581>0056 +1D582>0057 +1D583>0058 +1D584>0059 +1D585>005A +1D586>0061 +1D587>0062 +1D588>0063 +1D589>0064 +1D58A>0065 +1D58B>0066 +1D58C>0067 +1D58D>0068 +1D58E>0069 +1D58F>006A +1D590>006B +1D591>006C +1D592>006D +1D593>006E +1D594>006F +1D595>0070 +1D596>0071 +1D597>0072 +1D598>0073 +1D599>0074 +1D59A>0075 +1D59B>0076 +1D59C>0077 +1D59D>0078 +1D59E>0079 +1D59F>007A +1D5A0>0041 +1D5A1>0042 +1D5A2>0043 +1D5A3>0044 +1D5A4>0045 +1D5A5>0046 +1D5A6>0047 +1D5A7>0048 +1D5A8>0049 +1D5A9>004A +1D5AA>004B +1D5AB>004C +1D5AC>004D +1D5AD>004E +1D5AE>004F +1D5AF>0050 +1D5B0>0051 +1D5B1>0052 +1D5B2>0053 +1D5B3>0054 +1D5B4>0055 +1D5B5>0056 +1D5B6>0057 +1D5B7>0058 +1D5B8>0059 +1D5B9>005A +1D5BA>0061 +1D5BB>0062 +1D5BC>0063 +1D5BD>0064 +1D5BE>0065 +1D5BF>0066 +1D5C0>0067 +1D5C1>0068 +1D5C2>0069 +1D5C3>006A +1D5C4>006B +1D5C5>006C +1D5C6>006D +1D5C7>006E +1D5C8>006F +1D5C9>0070 +1D5CA>0071 +1D5CB>0072 +1D5CC>0073 +1D5CD>0074 +1D5CE>0075 +1D5CF>0076 +1D5D0>0077 +1D5D1>0078 +1D5D2>0079 +1D5D3>007A +1D5D4>0041 +1D5D5>0042 +1D5D6>0043 +1D5D7>0044 +1D5D8>0045 +1D5D9>0046 +1D5DA>0047 +1D5DB>0048 +1D5DC>0049 +1D5DD>004A +1D5DE>004B +1D5DF>004C +1D5E0>004D +1D5E1>004E +1D5E2>004F +1D5E3>0050 +1D5E4>0051 +1D5E5>0052 +1D5E6>0053 +1D5E7>0054 +1D5E8>0055 +1D5E9>0056 +1D5EA>0057 +1D5EB>0058 +1D5EC>0059 +1D5ED>005A +1D5EE>0061 +1D5EF>0062 +1D5F0>0063 +1D5F1>0064 +1D5F2>0065 +1D5F3>0066 +1D5F4>0067 +1D5F5>0068 +1D5F6>0069 +1D5F7>006A +1D5F8>006B +1D5F9>006C +1D5FA>006D +1D5FB>006E +1D5FC>006F +1D5FD>0070 +1D5FE>0071 +1D5FF>0072 +1D600>0073 +1D601>0074 +1D602>0075 +1D603>0076 +1D604>0077 +1D605>0078 +1D606>0079 +1D607>007A +1D608>0041 +1D609>0042 +1D60A>0043 +1D60B>0044 +1D60C>0045 +1D60D>0046 +1D60E>0047 +1D60F>0048 +1D610>0049 +1D611>004A +1D612>004B +1D613>004C +1D614>004D +1D615>004E +1D616>004F +1D617>0050 +1D618>0051 +1D619>0052 +1D61A>0053 +1D61B>0054 +1D61C>0055 +1D61D>0056 +1D61E>0057 +1D61F>0058 +1D620>0059 +1D621>005A +1D622>0061 +1D623>0062 +1D624>0063 +1D625>0064 +1D626>0065 +1D627>0066 +1D628>0067 +1D629>0068 +1D62A>0069 +1D62B>006A +1D62C>006B +1D62D>006C +1D62E>006D +1D62F>006E +1D630>006F +1D631>0070 +1D632>0071 +1D633>0072 +1D634>0073 +1D635>0074 +1D636>0075 +1D637>0076 +1D638>0077 +1D639>0078 +1D63A>0079 +1D63B>007A +1D63C>0041 +1D63D>0042 +1D63E>0043 +1D63F>0044 +1D640>0045 +1D641>0046 +1D642>0047 +1D643>0048 +1D644>0049 +1D645>004A +1D646>004B +1D647>004C +1D648>004D +1D649>004E +1D64A>004F +1D64B>0050 +1D64C>0051 +1D64D>0052 +1D64E>0053 +1D64F>0054 +1D650>0055 +1D651>0056 +1D652>0057 +1D653>0058 +1D654>0059 +1D655>005A +1D656>0061 +1D657>0062 +1D658>0063 +1D659>0064 +1D65A>0065 +1D65B>0066 +1D65C>0067 +1D65D>0068 +1D65E>0069 +1D65F>006A +1D660>006B +1D661>006C +1D662>006D +1D663>006E +1D664>006F +1D665>0070 +1D666>0071 +1D667>0072 +1D668>0073 +1D669>0074 +1D66A>0075 +1D66B>0076 +1D66C>0077 +1D66D>0078 +1D66E>0079 +1D66F>007A +1D670>0041 +1D671>0042 +1D672>0043 +1D673>0044 +1D674>0045 +1D675>0046 +1D676>0047 +1D677>0048 +1D678>0049 +1D679>004A +1D67A>004B +1D67B>004C +1D67C>004D +1D67D>004E +1D67E>004F +1D67F>0050 +1D680>0051 +1D681>0052 +1D682>0053 +1D683>0054 +1D684>0055 +1D685>0056 +1D686>0057 +1D687>0058 +1D688>0059 +1D689>005A +1D68A>0061 +1D68B>0062 +1D68C>0063 +1D68D>0064 +1D68E>0065 +1D68F>0066 +1D690>0067 +1D691>0068 +1D692>0069 +1D693>006A +1D694>006B +1D695>006C +1D696>006D +1D697>006E +1D698>006F +1D699>0070 +1D69A>0071 +1D69B>0072 +1D69C>0073 +1D69D>0074 +1D69E>0075 +1D69F>0076 +1D6A0>0077 +1D6A1>0078 +1D6A2>0079 +1D6A3>007A +1D6A4>0131 +1D6A5>0237 +1D6A8>0391 +1D6A9>0392 +1D6AA>0393 +1D6AB>0394 +1D6AC>0395 +1D6AD>0396 +1D6AE>0397 +1D6AF>0398 +1D6B0>0399 +1D6B1>039A +1D6B2>039B +1D6B3>039C +1D6B4>039D +1D6B5>039E +1D6B6>039F +1D6B7>03A0 +1D6B8>03A1 +1D6B9>03F4 +1D6BA>03A3 +1D6BB>03A4 +1D6BC>03A5 +1D6BD>03A6 +1D6BE>03A7 +1D6BF>03A8 +1D6C0>03A9 +1D6C1>2207 +1D6C2>03B1 +1D6C3>03B2 +1D6C4>03B3 +1D6C5>03B4 +1D6C6>03B5 +1D6C7>03B6 +1D6C8>03B7 +1D6C9>03B8 +1D6CA>03B9 +1D6CB>03BA +1D6CC>03BB +1D6CD>03BC +1D6CE>03BD +1D6CF>03BE +1D6D0>03BF +1D6D1>03C0 +1D6D2>03C1 +1D6D3>03C2 +1D6D4>03C3 +1D6D5>03C4 +1D6D6>03C5 +1D6D7>03C6 +1D6D8>03C7 +1D6D9>03C8 +1D6DA>03C9 +1D6DB>2202 +1D6DC>03F5 +1D6DD>03D1 +1D6DE>03F0 +1D6DF>03D5 +1D6E0>03F1 +1D6E1>03D6 +1D6E2>0391 +1D6E3>0392 +1D6E4>0393 +1D6E5>0394 +1D6E6>0395 +1D6E7>0396 +1D6E8>0397 +1D6E9>0398 +1D6EA>0399 +1D6EB>039A +1D6EC>039B +1D6ED>039C +1D6EE>039D +1D6EF>039E +1D6F0>039F +1D6F1>03A0 +1D6F2>03A1 +1D6F3>03F4 +1D6F4>03A3 +1D6F5>03A4 +1D6F6>03A5 +1D6F7>03A6 +1D6F8>03A7 +1D6F9>03A8 +1D6FA>03A9 +1D6FB>2207 +1D6FC>03B1 +1D6FD>03B2 +1D6FE>03B3 +1D6FF>03B4 +1D700>03B5 +1D701>03B6 +1D702>03B7 +1D703>03B8 +1D704>03B9 +1D705>03BA +1D706>03BB +1D707>03BC +1D708>03BD +1D709>03BE +1D70A>03BF +1D70B>03C0 +1D70C>03C1 +1D70D>03C2 +1D70E>03C3 +1D70F>03C4 +1D710>03C5 +1D711>03C6 +1D712>03C7 +1D713>03C8 +1D714>03C9 +1D715>2202 +1D716>03F5 +1D717>03D1 +1D718>03F0 +1D719>03D5 +1D71A>03F1 +1D71B>03D6 +1D71C>0391 +1D71D>0392 +1D71E>0393 +1D71F>0394 +1D720>0395 +1D721>0396 +1D722>0397 +1D723>0398 +1D724>0399 +1D725>039A +1D726>039B +1D727>039C +1D728>039D +1D729>039E +1D72A>039F +1D72B>03A0 +1D72C>03A1 +1D72D>03F4 +1D72E>03A3 +1D72F>03A4 +1D730>03A5 +1D731>03A6 +1D732>03A7 +1D733>03A8 +1D734>03A9 +1D735>2207 +1D736>03B1 +1D737>03B2 +1D738>03B3 +1D739>03B4 +1D73A>03B5 +1D73B>03B6 +1D73C>03B7 +1D73D>03B8 +1D73E>03B9 +1D73F>03BA +1D740>03BB +1D741>03BC +1D742>03BD +1D743>03BE +1D744>03BF +1D745>03C0 +1D746>03C1 +1D747>03C2 +1D748>03C3 +1D749>03C4 +1D74A>03C5 +1D74B>03C6 +1D74C>03C7 +1D74D>03C8 +1D74E>03C9 +1D74F>2202 +1D750>03F5 +1D751>03D1 +1D752>03F0 +1D753>03D5 +1D754>03F1 +1D755>03D6 +1D756>0391 +1D757>0392 +1D758>0393 +1D759>0394 +1D75A>0395 +1D75B>0396 +1D75C>0397 +1D75D>0398 +1D75E>0399 +1D75F>039A +1D760>039B +1D761>039C +1D762>039D +1D763>039E +1D764>039F +1D765>03A0 +1D766>03A1 +1D767>03F4 +1D768>03A3 +1D769>03A4 +1D76A>03A5 +1D76B>03A6 +1D76C>03A7 +1D76D>03A8 +1D76E>03A9 +1D76F>2207 +1D770>03B1 +1D771>03B2 +1D772>03B3 +1D773>03B4 +1D774>03B5 +1D775>03B6 +1D776>03B7 +1D777>03B8 +1D778>03B9 +1D779>03BA +1D77A>03BB +1D77B>03BC +1D77C>03BD +1D77D>03BE +1D77E>03BF +1D77F>03C0 +1D780>03C1 +1D781>03C2 +1D782>03C3 +1D783>03C4 +1D784>03C5 +1D785>03C6 +1D786>03C7 +1D787>03C8 +1D788>03C9 +1D789>2202 +1D78A>03F5 +1D78B>03D1 +1D78C>03F0 +1D78D>03D5 +1D78E>03F1 +1D78F>03D6 +1D790>0391 +1D791>0392 +1D792>0393 +1D793>0394 +1D794>0395 +1D795>0396 +1D796>0397 +1D797>0398 +1D798>0399 +1D799>039A +1D79A>039B +1D79B>039C +1D79C>039D +1D79D>039E +1D79E>039F +1D79F>03A0 +1D7A0>03A1 +1D7A1>03F4 +1D7A2>03A3 +1D7A3>03A4 +1D7A4>03A5 +1D7A5>03A6 +1D7A6>03A7 +1D7A7>03A8 +1D7A8>03A9 +1D7A9>2207 +1D7AA>03B1 +1D7AB>03B2 +1D7AC>03B3 +1D7AD>03B4 +1D7AE>03B5 +1D7AF>03B6 +1D7B0>03B7 +1D7B1>03B8 +1D7B2>03B9 +1D7B3>03BA +1D7B4>03BB +1D7B5>03BC +1D7B6>03BD +1D7B7>03BE +1D7B8>03BF +1D7B9>03C0 +1D7BA>03C1 +1D7BB>03C2 +1D7BC>03C3 +1D7BD>03C4 +1D7BE>03C5 +1D7BF>03C6 +1D7C0>03C7 +1D7C1>03C8 +1D7C2>03C9 +1D7C3>2202 +1D7C4>03F5 +1D7C5>03D1 +1D7C6>03F0 +1D7C7>03D5 +1D7C8>03F1 +1D7C9>03D6 +1D7CA>03DC +1D7CB>03DD +1D7CE>0030 +1D7CF>0031 +1D7D0>0032 +1D7D1>0033 +1D7D2>0034 +1D7D3>0035 +1D7D4>0036 +1D7D5>0037 +1D7D6>0038 +1D7D7>0039 +1D7D8>0030 +1D7D9>0031 +1D7DA>0032 +1D7DB>0033 +1D7DC>0034 +1D7DD>0035 +1D7DE>0036 +1D7DF>0037 +1D7E0>0038 +1D7E1>0039 +1D7E2>0030 +1D7E3>0031 +1D7E4>0032 +1D7E5>0033 +1D7E6>0034 +1D7E7>0035 +1D7E8>0036 +1D7E9>0037 +1D7EA>0038 +1D7EB>0039 +1D7EC>0030 +1D7ED>0031 +1D7EE>0032 +1D7EF>0033 +1D7F0>0034 +1D7F1>0035 +1D7F2>0036 +1D7F3>0037 +1D7F4>0038 +1D7F5>0039 +1D7F6>0030 +1D7F7>0031 +1D7F8>0032 +1D7F9>0033 +1D7FA>0034 +1D7FB>0035 +1D7FC>0036 +1D7FD>0037 +1D7FE>0038 +1D7FF>0039 +1F100>0030 002E +1F101>0030 002C +1F102>0031 002C +1F103>0032 002C +1F104>0033 002C +1F105>0034 002C +1F106>0035 002C +1F107>0036 002C +1F108>0037 002C +1F109>0038 002C +1F10A>0039 002C +1F110>0028 0041 0029 +1F111>0028 0042 0029 +1F112>0028 0043 0029 +1F113>0028 0044 0029 +1F114>0028 0045 0029 +1F115>0028 0046 0029 +1F116>0028 0047 0029 +1F117>0028 0048 0029 +1F118>0028 0049 0029 +1F119>0028 004A 0029 +1F11A>0028 004B 0029 +1F11B>0028 004C 0029 +1F11C>0028 004D 0029 +1F11D>0028 004E 0029 +1F11E>0028 004F 0029 +1F11F>0028 0050 0029 +1F120>0028 0051 0029 +1F121>0028 0052 0029 +1F122>0028 0053 0029 +1F123>0028 0054 0029 +1F124>0028 0055 0029 +1F125>0028 0056 0029 +1F126>0028 0057 0029 +1F127>0028 0058 0029 +1F128>0028 0059 0029 +1F129>0028 005A 0029 +1F12A>3014 0053 3015 +1F12B>0043 +1F12C>0052 +1F12D>0043 0044 +1F12E>0057 005A +1F131>0042 +1F13D>004E +1F13F>0050 +1F142>0053 +1F146>0057 +1F14A>0048 0056 +1F14B>004D 0056 +1F14C>0053 0044 +1F14D>0053 0053 +1F14E>0050 0050 0056 +1F190>0044 004A +1F200>307B 304B +1F210>624B +1F211>5B57 +1F212>53CC +1F213>30C7 +1F214>4E8C +1F215>591A +1F216>89E3 +1F217>5929 +1F218>4EA4 +1F219>6620 +1F21A>7121 +1F21B>6599 +1F21C>524D +1F21D>5F8C +1F21E>518D +1F21F>65B0 +1F220>521D +1F221>7D42 +1F222>751F +1F223>8CA9 +1F224>58F0 +1F225>5439 +1F226>6F14 +1F227>6295 +1F228>6355 +1F229>4E00 +1F22A>4E09 +1F22B>904A +1F22C>5DE6 +1F22D>4E2D +1F22E>53F3 +1F22F>6307 +1F230>8D70 +1F231>6253 +1F240>3014 672C 3015 +1F241>3014 4E09 3015 +1F242>3014 4E8C 3015 +1F243>3014 5B89 3015 +1F244>3014 70B9 3015 +1F245>3014 6253 3015 +1F246>3014 76D7 3015 +1F247>3014 52DD 3015 +1F248>3014 6557 3015 +2F800>4E3D +2F801>4E38 +2F802>4E41 +2F803>20122 +2F804>4F60 +2F805>4FAE +2F806>4FBB +2F807>5002 +2F808>507A +2F809>5099 +2F80A>50E7 +2F80B>50CF +2F80C>349E +2F80D>2063A +2F80E>514D +2F80F>5154 +2F810>5164 +2F811>5177 +2F812>2051C +2F813>34B9 +2F814>5167 +2F815>518D +2F816>2054B +2F817>5197 +2F818>51A4 +2F819>4ECC +2F81A>51AC +2F81B>51B5 +2F81C>291DF +2F81D>51F5 +2F81E>5203 +2F81F>34DF +2F820>523B +2F821>5246 +2F822>5272 +2F823>5277 +2F824>3515 +2F825>52C7 +2F826>52C9 +2F827>52E4 +2F828>52FA +2F829>5305 +2F82A>5306 +2F82B>5317 +2F82C>5349 +2F82D>5351 +2F82E>535A +2F82F>5373 +2F830>537D +2F831>537F +2F832>537F +2F833>537F +2F834>20A2C +2F835>7070 +2F836>53CA +2F837>53DF +2F838>20B63 +2F839>53EB +2F83A>53F1 +2F83B>5406 +2F83C>549E +2F83D>5438 +2F83E>5448 +2F83F>5468 +2F840>54A2 +2F841>54F6 +2F842>5510 +2F843>5553 +2F844>5563 +2F845>5584 +2F846>5584 +2F847>5599 +2F848>55AB +2F849>55B3 +2F84A>55C2 +2F84B>5716 +2F84C>5606 +2F84D>5717 +2F84E>5651 +2F84F>5674 +2F850>5207 +2F851>58EE +2F852>57CE +2F853>57F4 +2F854>580D +2F855>578B +2F856>5832 +2F857>5831 +2F858>58AC +2F859>214E4 +2F85A>58F2 +2F85B>58F7 +2F85C>5906 +2F85D>591A +2F85E>5922 +2F85F>5962 +2F860>216A8 +2F861>216EA +2F862>59EC +2F863>5A1B +2F864>5A27 +2F865>59D8 +2F866>5A66 +2F867>36EE +2F868>36FC +2F869>5B08 +2F86A>5B3E +2F86B>5B3E +2F86C>219C8 +2F86D>5BC3 +2F86E>5BD8 +2F86F>5BE7 +2F870>5BF3 +2F871>21B18 +2F872>5BFF +2F873>5C06 +2F874>5F53 +2F875>5C22 +2F876>3781 +2F877>5C60 +2F878>5C6E +2F879>5CC0 +2F87A>5C8D +2F87B>21DE4 +2F87C>5D43 +2F87D>21DE6 +2F87E>5D6E +2F87F>5D6B +2F880>5D7C +2F881>5DE1 +2F882>5DE2 +2F883>382F +2F884>5DFD +2F885>5E28 +2F886>5E3D +2F887>5E69 +2F888>3862 +2F889>22183 +2F88A>387C +2F88B>5EB0 +2F88C>5EB3 +2F88D>5EB6 +2F88E>5ECA +2F88F>2A392 +2F890>5EFE +2F891>22331 +2F892>22331 +2F893>8201 +2F894>5F22 +2F895>5F22 +2F896>38C7 +2F897>232B8 +2F898>261DA +2F899>5F62 +2F89A>5F6B +2F89B>38E3 +2F89C>5F9A +2F89D>5FCD +2F89E>5FD7 +2F89F>5FF9 +2F8A0>6081 +2F8A1>393A +2F8A2>391C +2F8A3>6094 +2F8A4>226D4 +2F8A5>60C7 +2F8A6>6148 +2F8A7>614C +2F8A8>614E +2F8A9>614C +2F8AA>617A +2F8AB>618E +2F8AC>61B2 +2F8AD>61A4 +2F8AE>61AF +2F8AF>61DE +2F8B0>61F2 +2F8B1>61F6 +2F8B2>6210 +2F8B3>621B +2F8B4>625D +2F8B5>62B1 +2F8B6>62D4 +2F8B7>6350 +2F8B8>22B0C +2F8B9>633D +2F8BA>62FC +2F8BB>6368 +2F8BC>6383 +2F8BD>63E4 +2F8BE>22BF1 +2F8BF>6422 +2F8C0>63C5 +2F8C1>63A9 +2F8C2>3A2E +2F8C3>6469 +2F8C4>647E +2F8C5>649D +2F8C6>6477 +2F8C7>3A6C +2F8C8>654F +2F8C9>656C +2F8CA>2300A +2F8CB>65E3 +2F8CC>66F8 +2F8CD>6649 +2F8CE>3B19 +2F8CF>6691 +2F8D0>3B08 +2F8D1>3AE4 +2F8D2>5192 +2F8D3>5195 +2F8D4>6700 +2F8D5>669C +2F8D6>80AD +2F8D7>43D9 +2F8D8>6717 +2F8D9>671B +2F8DA>6721 +2F8DB>675E +2F8DC>6753 +2F8DD>233C3 +2F8DE>3B49 +2F8DF>67FA +2F8E0>6785 +2F8E1>6852 +2F8E2>6885 +2F8E3>2346D +2F8E4>688E +2F8E5>681F +2F8E6>6914 +2F8E7>3B9D +2F8E8>6942 +2F8E9>69A3 +2F8EA>69EA +2F8EB>6AA8 +2F8EC>236A3 +2F8ED>6ADB +2F8EE>3C18 +2F8EF>6B21 +2F8F0>238A7 +2F8F1>6B54 +2F8F2>3C4E +2F8F3>6B72 +2F8F4>6B9F +2F8F5>6BBA +2F8F6>6BBB +2F8F7>23A8D +2F8F8>21D0B +2F8F9>23AFA +2F8FA>6C4E +2F8FB>23CBC +2F8FC>6CBF +2F8FD>6CCD +2F8FE>6C67 +2F8FF>6D16 +2F900>6D3E +2F901>6D77 +2F902>6D41 +2F903>6D69 +2F904>6D78 +2F905>6D85 +2F906>23D1E +2F907>6D34 +2F908>6E2F +2F909>6E6E +2F90A>3D33 +2F90B>6ECB +2F90C>6EC7 +2F90D>23ED1 +2F90E>6DF9 +2F90F>6F6E +2F910>23F5E +2F911>23F8E +2F912>6FC6 +2F913>7039 +2F914>701E +2F915>701B +2F916>3D96 +2F917>704A +2F918>707D +2F919>7077 +2F91A>70AD +2F91B>20525 +2F91C>7145 +2F91D>24263 +2F91E>719C +2F91F>243AB +2F920>7228 +2F921>7235 +2F922>7250 +2F923>24608 +2F924>7280 +2F925>7295 +2F926>24735 +2F927>24814 +2F928>737A +2F929>738B +2F92A>3EAC +2F92B>73A5 +2F92C>3EB8 +2F92D>3EB8 +2F92E>7447 +2F92F>745C +2F930>7471 +2F931>7485 +2F932>74CA +2F933>3F1B +2F934>7524 +2F935>24C36 +2F936>753E +2F937>24C92 +2F938>7570 +2F939>2219F +2F93A>7610 +2F93B>24FA1 +2F93C>24FB8 +2F93D>25044 +2F93E>3FFC +2F93F>4008 +2F940>76F4 +2F941>250F3 +2F942>250F2 +2F943>25119 +2F944>25133 +2F945>771E +2F946>771F +2F947>771F +2F948>774A +2F949>4039 +2F94A>778B +2F94B>4046 +2F94C>4096 +2F94D>2541D +2F94E>784E +2F94F>788C +2F950>78CC +2F951>40E3 +2F952>25626 +2F953>7956 +2F954>2569A +2F955>256C5 +2F956>798F +2F957>79EB +2F958>412F +2F959>7A40 +2F95A>7A4A +2F95B>7A4F +2F95C>2597C +2F95D>25AA7 +2F95E>25AA7 +2F95F>7AEE +2F960>4202 +2F961>25BAB +2F962>7BC6 +2F963>7BC9 +2F964>4227 +2F965>25C80 +2F966>7CD2 +2F967>42A0 +2F968>7CE8 +2F969>7CE3 +2F96A>7D00 +2F96B>25F86 +2F96C>7D63 +2F96D>4301 +2F96E>7DC7 +2F96F>7E02 +2F970>7E45 +2F971>4334 +2F972>26228 +2F973>26247 +2F974>4359 +2F975>262D9 +2F976>7F7A +2F977>2633E +2F978>7F95 +2F979>7FFA +2F97A>8005 +2F97B>264DA +2F97C>26523 +2F97D>8060 +2F97E>265A8 +2F97F>8070 +2F980>2335F +2F981>43D5 +2F982>80B2 +2F983>8103 +2F984>440B +2F985>813E +2F986>5AB5 +2F987>267A7 +2F988>267B5 +2F989>23393 +2F98A>2339C +2F98B>8201 +2F98C>8204 +2F98D>8F9E +2F98E>446B +2F98F>8291 +2F990>828B +2F991>829D +2F992>52B3 +2F993>82B1 +2F994>82B3 +2F995>82BD +2F996>82E6 +2F997>26B3C +2F998>82E5 +2F999>831D +2F99A>8363 +2F99B>83AD +2F99C>8323 +2F99D>83BD +2F99E>83E7 +2F99F>8457 +2F9A0>8353 +2F9A1>83CA +2F9A2>83CC +2F9A3>83DC +2F9A4>26C36 +2F9A5>26D6B +2F9A6>26CD5 +2F9A7>452B +2F9A8>84F1 +2F9A9>84F3 +2F9AA>8516 +2F9AB>273CA +2F9AC>8564 +2F9AD>26F2C +2F9AE>455D +2F9AF>4561 +2F9B0>26FB1 +2F9B1>270D2 +2F9B2>456B +2F9B3>8650 +2F9B4>865C +2F9B5>8667 +2F9B6>8669 +2F9B7>86A9 +2F9B8>8688 +2F9B9>870E +2F9BA>86E2 +2F9BB>8779 +2F9BC>8728 +2F9BD>876B +2F9BE>8786 +2F9BF>45D7 +2F9C0>87E1 +2F9C1>8801 +2F9C2>45F9 +2F9C3>8860 +2F9C4>8863 +2F9C5>27667 +2F9C6>88D7 +2F9C7>88DE +2F9C8>4635 +2F9C9>88FA +2F9CA>34BB +2F9CB>278AE +2F9CC>27966 +2F9CD>46BE +2F9CE>46C7 +2F9CF>8AA0 +2F9D0>8AED +2F9D1>8B8A +2F9D2>8C55 +2F9D3>27CA8 +2F9D4>8CAB +2F9D5>8CC1 +2F9D6>8D1B +2F9D7>8D77 +2F9D8>27F2F +2F9D9>20804 +2F9DA>8DCB +2F9DB>8DBC +2F9DC>8DF0 +2F9DD>208DE +2F9DE>8ED4 +2F9DF>8F38 +2F9E0>285D2 +2F9E1>285ED +2F9E2>9094 +2F9E3>90F1 +2F9E4>9111 +2F9E5>2872E +2F9E6>911B +2F9E7>9238 +2F9E8>92D7 +2F9E9>92D8 +2F9EA>927C +2F9EB>93F9 +2F9EC>9415 +2F9ED>28BFA +2F9EE>958B +2F9EF>4995 +2F9F0>95B7 +2F9F1>28D77 +2F9F2>49E6 +2F9F3>96C3 +2F9F4>5DB2 +2F9F5>9723 +2F9F6>29145 +2F9F7>2921A +2F9F8>4A6E +2F9F9>4A76 +2F9FA>97E0 +2F9FB>2940A +2F9FC>4AB2 +2F9FD>29496 +2F9FE>980B +2F9FF>980B +2FA00>9829 +2FA01>295B6 +2FA02>98E2 +2FA03>4B33 +2FA04>9929 +2FA05>99A7 +2FA06>99C2 +2FA07>99FE +2FA08>4BCE +2FA09>29B30 +2FA0A>9B12 +2FA0B>9C40 +2FA0C>9CFD +2FA0D>4CCE +2FA0E>4CED +2FA0F>9D67 +2FA10>2A0CE +2FA11>4CF8 +2FA12>2A105 +2FA13>2A20E +2FA14>2A291 +2FA15>9EBB +2FA16>4D56 +2FA17>9EF9 +2FA18>9EFE +2FA19>9F05 +2FA1A>9F0F +2FA1B>9F16 +2FA1C>9F3B +2FA1D>2A600 diff --git a/mailnews/extensions/fts3/data/nfkc_cf.txt b/mailnews/extensions/fts3/data/nfkc_cf.txt new file mode 100644 index 000000000..becabbbf3 --- /dev/null +++ b/mailnews/extensions/fts3/data/nfkc_cf.txt @@ -0,0 +1,5376 @@ +# Extracted from: +# DerivedNormalizationProps-5.2.0.txt +# Date: 2009-08-26, 18:18:50 GMT [MD] +# +# Unicode Character Database +# Copyright (c) 1991-2009 Unicode, Inc. +# For terms of use, see http://www.unicode.org/terms_of_use.html +# For documentation, see http://www.unicode.org/reports/tr44/ + +# ================================================ +# This file has been reformatted into syntax for the +# gennorm2 Normalizer2 data generator tool. +# Only the NFKC_CF mappings are retained and reformatted. +# Reformatting via regular expression: s/ *; NFKC_CF; */>/ +# Use this file as the second gennorm2 input file after nfkc.txt. +# ================================================ + +# Derived Property: NFKC_Casefold (NFKC_CF) +# This property removes certain variations from characters: case, compatibility, and default-ignorables. +# It is used for loose matching and certain types of identifiers. +# It is constructed by applying NFKC, CaseFolding, and removal of Default_Ignorable_Code_Points. +# The process of applying these transformations is repeated until a stable result is produced. +# WARNING: Application to STRINGS must apply NFC after mapping each character, because characters may interact. +# For more information, see [http://www.unicode.org/reports/tr44/] +# Omitted code points are unchanged by this mapping. +# @missing: 0000..10FFFF><code point> + +# All code points not explicitly listed for NFKC_Casefold +# have the value <codepoint>. + +0041>0061 +0042>0062 +0043>0063 +0044>0064 +0045>0065 +0046>0066 +0047>0067 +0048>0068 +0049>0069 +004A>006A +004B>006B +004C>006C +004D>006D +004E>006E +004F>006F +0050>0070 +0051>0071 +0052>0072 +0053>0073 +0054>0074 +0055>0075 +0056>0076 +0057>0077 +0058>0078 +0059>0079 +005A>007A +00A0>0020 +00A8>0020 0308 +00AA>0061 +00AD> +00AF>0020 0304 +00B2>0032 +00B3>0033 +00B4>0020 0301 +00B5>03BC +00B8>0020 0327 +00B9>0031 +00BA>006F +00BC>0031 2044 0034 +00BD>0031 2044 0032 +00BE>0033 2044 0034 +00C0>00E0 +00C1>00E1 +00C2>00E2 +00C3>00E3 +00C4>00E4 +00C5>00E5 +00C6>00E6 +00C7>00E7 +00C8>00E8 +00C9>00E9 +00CA>00EA +00CB>00EB +00CC>00EC +00CD>00ED +00CE>00EE +00CF>00EF +00D0>00F0 +00D1>00F1 +00D2>00F2 +00D3>00F3 +00D4>00F4 +00D5>00F5 +00D6>00F6 +00D8>00F8 +00D9>00F9 +00DA>00FA +00DB>00FB +00DC>00FC +00DD>00FD +00DE>00FE +00DF>0073 0073 +0100>0101 +0102>0103 +0104>0105 +0106>0107 +0108>0109 +010A>010B +010C>010D +010E>010F +0110>0111 +0112>0113 +0114>0115 +0116>0117 +0118>0119 +011A>011B +011C>011D +011E>011F +0120>0121 +0122>0123 +0124>0125 +0126>0127 +0128>0129 +012A>012B +012C>012D +012E>012F +0130>0069 0307 +0132..0133>0069 006A +0134>0135 +0136>0137 +0139>013A +013B>013C +013D>013E +013F..0140>006C 00B7 +0141>0142 +0143>0144 +0145>0146 +0147>0148 +0149>02BC 006E +014A>014B +014C>014D +014E>014F +0150>0151 +0152>0153 +0154>0155 +0156>0157 +0158>0159 +015A>015B +015C>015D +015E>015F +0160>0161 +0162>0163 +0164>0165 +0166>0167 +0168>0169 +016A>016B +016C>016D +016E>016F +0170>0171 +0172>0173 +0174>0175 +0176>0177 +0178>00FF +0179>017A +017B>017C +017D>017E +017F>0073 +0181>0253 +0182>0183 +0184>0185 +0186>0254 +0187>0188 +0189>0256 +018A>0257 +018B>018C +018E>01DD +018F>0259 +0190>025B +0191>0192 +0193>0260 +0194>0263 +0196>0269 +0197>0268 +0198>0199 +019C>026F +019D>0272 +019F>0275 +01A0>01A1 +01A2>01A3 +01A4>01A5 +01A6>0280 +01A7>01A8 +01A9>0283 +01AC>01AD +01AE>0288 +01AF>01B0 +01B1>028A +01B2>028B +01B3>01B4 +01B5>01B6 +01B7>0292 +01B8>01B9 +01BC>01BD +01C4..01C6>0064 017E +01C7..01C9>006C 006A +01CA..01CC>006E 006A +01CD>01CE +01CF>01D0 +01D1>01D2 +01D3>01D4 +01D5>01D6 +01D7>01D8 +01D9>01DA +01DB>01DC +01DE>01DF +01E0>01E1 +01E2>01E3 +01E4>01E5 +01E6>01E7 +01E8>01E9 +01EA>01EB +01EC>01ED +01EE>01EF +01F1..01F3>0064 007A +01F4>01F5 +01F6>0195 +01F7>01BF +01F8>01F9 +01FA>01FB +01FC>01FD +01FE>01FF +0200>0201 +0202>0203 +0204>0205 +0206>0207 +0208>0209 +020A>020B +020C>020D +020E>020F +0210>0211 +0212>0213 +0214>0215 +0216>0217 +0218>0219 +021A>021B +021C>021D +021E>021F +0220>019E +0222>0223 +0224>0225 +0226>0227 +0228>0229 +022A>022B +022C>022D +022E>022F +0230>0231 +0232>0233 +023A>2C65 +023B>023C +023D>019A +023E>2C66 +0241>0242 +0243>0180 +0244>0289 +0245>028C +0246>0247 +0248>0249 +024A>024B +024C>024D +024E>024F +02B0>0068 +02B1>0266 +02B2>006A +02B3>0072 +02B4>0279 +02B5>027B +02B6>0281 +02B7>0077 +02B8>0079 +02D8>0020 0306 +02D9>0020 0307 +02DA>0020 030A +02DB>0020 0328 +02DC>0020 0303 +02DD>0020 030B +02E0>0263 +02E1>006C +02E2>0073 +02E3>0078 +02E4>0295 +0340>0300 +0341>0301 +0343>0313 +0344>0308 0301 +0345>03B9 +034F> +0370>0371 +0372>0373 +0374>02B9 +0376>0377 +037A>0020 03B9 +037E>003B +0384>0020 0301 +0385>0020 0308 0301 +0386>03AC +0387>00B7 +0388>03AD +0389>03AE +038A>03AF +038C>03CC +038E>03CD +038F>03CE +0391>03B1 +0392>03B2 +0393>03B3 +0394>03B4 +0395>03B5 +0396>03B6 +0397>03B7 +0398>03B8 +0399>03B9 +039A>03BA +039B>03BB +039C>03BC +039D>03BD +039E>03BE +039F>03BF +03A0>03C0 +03A1>03C1 +03A3>03C3 +03A4>03C4 +03A5>03C5 +03A6>03C6 +03A7>03C7 +03A8>03C8 +03A9>03C9 +03AA>03CA +03AB>03CB +03C2>03C3 +03CF>03D7 +03D0>03B2 +03D1>03B8 +03D2>03C5 +03D3>03CD +03D4>03CB +03D5>03C6 +03D6>03C0 +03D8>03D9 +03DA>03DB +03DC>03DD +03DE>03DF +03E0>03E1 +03E2>03E3 +03E4>03E5 +03E6>03E7 +03E8>03E9 +03EA>03EB +03EC>03ED +03EE>03EF +03F0>03BA +03F1>03C1 +03F2>03C3 +03F4>03B8 +03F5>03B5 +03F7>03F8 +03F9>03C3 +03FA>03FB +03FD>037B +03FE>037C +03FF>037D +0400>0450 +0401>0451 +0402>0452 +0403>0453 +0404>0454 +0405>0455 +0406>0456 +0407>0457 +0408>0458 +0409>0459 +040A>045A +040B>045B +040C>045C +040D>045D +040E>045E +040F>045F +0410>0430 +0411>0431 +0412>0432 +0413>0433 +0414>0434 +0415>0435 +0416>0436 +0417>0437 +0418>0438 +0419>0439 +041A>043A +041B>043B +041C>043C +041D>043D +041E>043E +041F>043F +0420>0440 +0421>0441 +0422>0442 +0423>0443 +0424>0444 +0425>0445 +0426>0446 +0427>0447 +0428>0448 +0429>0449 +042A>044A +042B>044B +042C>044C +042D>044D +042E>044E +042F>044F +0460>0461 +0462>0463 +0464>0465 +0466>0467 +0468>0469 +046A>046B +046C>046D +046E>046F +0470>0471 +0472>0473 +0474>0475 +0476>0477 +0478>0479 +047A>047B +047C>047D +047E>047F +0480>0481 +048A>048B +048C>048D +048E>048F +0490>0491 +0492>0493 +0494>0495 +0496>0497 +0498>0499 +049A>049B +049C>049D +049E>049F +04A0>04A1 +04A2>04A3 +04A4>04A5 +04A6>04A7 +04A8>04A9 +04AA>04AB +04AC>04AD +04AE>04AF +04B0>04B1 +04B2>04B3 +04B4>04B5 +04B6>04B7 +04B8>04B9 +04BA>04BB +04BC>04BD +04BE>04BF +04C0>04CF +04C1>04C2 +04C3>04C4 +04C5>04C6 +04C7>04C8 +04C9>04CA +04CB>04CC +04CD>04CE +04D0>04D1 +04D2>04D3 +04D4>04D5 +04D6>04D7 +04D8>04D9 +04DA>04DB +04DC>04DD +04DE>04DF +04E0>04E1 +04E2>04E3 +04E4>04E5 +04E6>04E7 +04E8>04E9 +04EA>04EB +04EC>04ED +04EE>04EF +04F0>04F1 +04F2>04F3 +04F4>04F5 +04F6>04F7 +04F8>04F9 +04FA>04FB +04FC>04FD +04FE>04FF +0500>0501 +0502>0503 +0504>0505 +0506>0507 +0508>0509 +050A>050B +050C>050D +050E>050F +0510>0511 +0512>0513 +0514>0515 +0516>0517 +0518>0519 +051A>051B +051C>051D +051E>051F +0520>0521 +0522>0523 +0524>0525 +0531>0561 +0532>0562 +0533>0563 +0534>0564 +0535>0565 +0536>0566 +0537>0567 +0538>0568 +0539>0569 +053A>056A +053B>056B +053C>056C +053D>056D +053E>056E +053F>056F +0540>0570 +0541>0571 +0542>0572 +0543>0573 +0544>0574 +0545>0575 +0546>0576 +0547>0577 +0548>0578 +0549>0579 +054A>057A +054B>057B +054C>057C +054D>057D +054E>057E +054F>057F +0550>0580 +0551>0581 +0552>0582 +0553>0583 +0554>0584 +0555>0585 +0556>0586 +0587>0565 0582 +0675>0627 0674 +0676>0648 0674 +0677>06C7 0674 +0678>064A 0674 +0958>0915 093C +0959>0916 093C +095A>0917 093C +095B>091C 093C +095C>0921 093C +095D>0922 093C +095E>092B 093C +095F>092F 093C +09DC>09A1 09BC +09DD>09A2 09BC +09DF>09AF 09BC +0A33>0A32 0A3C +0A36>0A38 0A3C +0A59>0A16 0A3C +0A5A>0A17 0A3C +0A5B>0A1C 0A3C +0A5E>0A2B 0A3C +0B5C>0B21 0B3C +0B5D>0B22 0B3C +0E33>0E4D 0E32 +0EB3>0ECD 0EB2 +0EDC>0EAB 0E99 +0EDD>0EAB 0EA1 +0F0C>0F0B +0F43>0F42 0FB7 +0F4D>0F4C 0FB7 +0F52>0F51 0FB7 +0F57>0F56 0FB7 +0F5C>0F5B 0FB7 +0F69>0F40 0FB5 +0F73>0F71 0F72 +0F75>0F71 0F74 +0F76>0FB2 0F80 +0F77>0FB2 0F71 0F80 +0F78>0FB3 0F80 +0F79>0FB3 0F71 0F80 +0F81>0F71 0F80 +0F93>0F92 0FB7 +0F9D>0F9C 0FB7 +0FA2>0FA1 0FB7 +0FA7>0FA6 0FB7 +0FAC>0FAB 0FB7 +0FB9>0F90 0FB5 +10A0>2D00 +10A1>2D01 +10A2>2D02 +10A3>2D03 +10A4>2D04 +10A5>2D05 +10A6>2D06 +10A7>2D07 +10A8>2D08 +10A9>2D09 +10AA>2D0A +10AB>2D0B +10AC>2D0C +10AD>2D0D +10AE>2D0E +10AF>2D0F +10B0>2D10 +10B1>2D11 +10B2>2D12 +10B3>2D13 +10B4>2D14 +10B5>2D15 +10B6>2D16 +10B7>2D17 +10B8>2D18 +10B9>2D19 +10BA>2D1A +10BB>2D1B +10BC>2D1C +10BD>2D1D +10BE>2D1E +10BF>2D1F +10C0>2D20 +10C1>2D21 +10C2>2D22 +10C3>2D23 +10C4>2D24 +10C5>2D25 +10FC>10DC +115F..1160> +17B4..17B5> +180B..180D> +1D2C>0061 +1D2D>00E6 +1D2E>0062 +1D30>0064 +1D31>0065 +1D32>01DD +1D33>0067 +1D34>0068 +1D35>0069 +1D36>006A +1D37>006B +1D38>006C +1D39>006D +1D3A>006E +1D3C>006F +1D3D>0223 +1D3E>0070 +1D3F>0072 +1D40>0074 +1D41>0075 +1D42>0077 +1D43>0061 +1D44>0250 +1D45>0251 +1D46>1D02 +1D47>0062 +1D48>0064 +1D49>0065 +1D4A>0259 +1D4B>025B +1D4C>025C +1D4D>0067 +1D4F>006B +1D50>006D +1D51>014B +1D52>006F +1D53>0254 +1D54>1D16 +1D55>1D17 +1D56>0070 +1D57>0074 +1D58>0075 +1D59>1D1D +1D5A>026F +1D5B>0076 +1D5C>1D25 +1D5D>03B2 +1D5E>03B3 +1D5F>03B4 +1D60>03C6 +1D61>03C7 +1D62>0069 +1D63>0072 +1D64>0075 +1D65>0076 +1D66>03B2 +1D67>03B3 +1D68>03C1 +1D69>03C6 +1D6A>03C7 +1D78>043D +1D9B>0252 +1D9C>0063 +1D9D>0255 +1D9E>00F0 +1D9F>025C +1DA0>0066 +1DA1>025F +1DA2>0261 +1DA3>0265 +1DA4>0268 +1DA5>0269 +1DA6>026A +1DA7>1D7B +1DA8>029D +1DA9>026D +1DAA>1D85 +1DAB>029F +1DAC>0271 +1DAD>0270 +1DAE>0272 +1DAF>0273 +1DB0>0274 +1DB1>0275 +1DB2>0278 +1DB3>0282 +1DB4>0283 +1DB5>01AB +1DB6>0289 +1DB7>028A +1DB8>1D1C +1DB9>028B +1DBA>028C +1DBB>007A +1DBC>0290 +1DBD>0291 +1DBE>0292 +1DBF>03B8 +1E00>1E01 +1E02>1E03 +1E04>1E05 +1E06>1E07 +1E08>1E09 +1E0A>1E0B +1E0C>1E0D +1E0E>1E0F +1E10>1E11 +1E12>1E13 +1E14>1E15 +1E16>1E17 +1E18>1E19 +1E1A>1E1B +1E1C>1E1D +1E1E>1E1F +1E20>1E21 +1E22>1E23 +1E24>1E25 +1E26>1E27 +1E28>1E29 +1E2A>1E2B +1E2C>1E2D +1E2E>1E2F +1E30>1E31 +1E32>1E33 +1E34>1E35 +1E36>1E37 +1E38>1E39 +1E3A>1E3B +1E3C>1E3D +1E3E>1E3F +1E40>1E41 +1E42>1E43 +1E44>1E45 +1E46>1E47 +1E48>1E49 +1E4A>1E4B +1E4C>1E4D +1E4E>1E4F +1E50>1E51 +1E52>1E53 +1E54>1E55 +1E56>1E57 +1E58>1E59 +1E5A>1E5B +1E5C>1E5D +1E5E>1E5F +1E60>1E61 +1E62>1E63 +1E64>1E65 +1E66>1E67 +1E68>1E69 +1E6A>1E6B +1E6C>1E6D +1E6E>1E6F +1E70>1E71 +1E72>1E73 +1E74>1E75 +1E76>1E77 +1E78>1E79 +1E7A>1E7B +1E7C>1E7D +1E7E>1E7F +1E80>1E81 +1E82>1E83 +1E84>1E85 +1E86>1E87 +1E88>1E89 +1E8A>1E8B +1E8C>1E8D +1E8E>1E8F +1E90>1E91 +1E92>1E93 +1E94>1E95 +1E9A>0061 02BE +1E9B>1E61 +1E9E>0073 0073 +1EA0>1EA1 +1EA2>1EA3 +1EA4>1EA5 +1EA6>1EA7 +1EA8>1EA9 +1EAA>1EAB +1EAC>1EAD +1EAE>1EAF +1EB0>1EB1 +1EB2>1EB3 +1EB4>1EB5 +1EB6>1EB7 +1EB8>1EB9 +1EBA>1EBB +1EBC>1EBD +1EBE>1EBF +1EC0>1EC1 +1EC2>1EC3 +1EC4>1EC5 +1EC6>1EC7 +1EC8>1EC9 +1ECA>1ECB +1ECC>1ECD +1ECE>1ECF +1ED0>1ED1 +1ED2>1ED3 +1ED4>1ED5 +1ED6>1ED7 +1ED8>1ED9 +1EDA>1EDB +1EDC>1EDD +1EDE>1EDF +1EE0>1EE1 +1EE2>1EE3 +1EE4>1EE5 +1EE6>1EE7 +1EE8>1EE9 +1EEA>1EEB +1EEC>1EED +1EEE>1EEF +1EF0>1EF1 +1EF2>1EF3 +1EF4>1EF5 +1EF6>1EF7 +1EF8>1EF9 +1EFA>1EFB +1EFC>1EFD +1EFE>1EFF +1F08>1F00 +1F09>1F01 +1F0A>1F02 +1F0B>1F03 +1F0C>1F04 +1F0D>1F05 +1F0E>1F06 +1F0F>1F07 +1F18>1F10 +1F19>1F11 +1F1A>1F12 +1F1B>1F13 +1F1C>1F14 +1F1D>1F15 +1F28>1F20 +1F29>1F21 +1F2A>1F22 +1F2B>1F23 +1F2C>1F24 +1F2D>1F25 +1F2E>1F26 +1F2F>1F27 +1F38>1F30 +1F39>1F31 +1F3A>1F32 +1F3B>1F33 +1F3C>1F34 +1F3D>1F35 +1F3E>1F36 +1F3F>1F37 +1F48>1F40 +1F49>1F41 +1F4A>1F42 +1F4B>1F43 +1F4C>1F44 +1F4D>1F45 +1F59>1F51 +1F5B>1F53 +1F5D>1F55 +1F5F>1F57 +1F68>1F60 +1F69>1F61 +1F6A>1F62 +1F6B>1F63 +1F6C>1F64 +1F6D>1F65 +1F6E>1F66 +1F6F>1F67 +1F71>03AC +1F73>03AD +1F75>03AE +1F77>03AF +1F79>03CC +1F7B>03CD +1F7D>03CE +1F80>1F00 03B9 +1F81>1F01 03B9 +1F82>1F02 03B9 +1F83>1F03 03B9 +1F84>1F04 03B9 +1F85>1F05 03B9 +1F86>1F06 03B9 +1F87>1F07 03B9 +1F88>1F00 03B9 +1F89>1F01 03B9 +1F8A>1F02 03B9 +1F8B>1F03 03B9 +1F8C>1F04 03B9 +1F8D>1F05 03B9 +1F8E>1F06 03B9 +1F8F>1F07 03B9 +1F90>1F20 03B9 +1F91>1F21 03B9 +1F92>1F22 03B9 +1F93>1F23 03B9 +1F94>1F24 03B9 +1F95>1F25 03B9 +1F96>1F26 03B9 +1F97>1F27 03B9 +1F98>1F20 03B9 +1F99>1F21 03B9 +1F9A>1F22 03B9 +1F9B>1F23 03B9 +1F9C>1F24 03B9 +1F9D>1F25 03B9 +1F9E>1F26 03B9 +1F9F>1F27 03B9 +1FA0>1F60 03B9 +1FA1>1F61 03B9 +1FA2>1F62 03B9 +1FA3>1F63 03B9 +1FA4>1F64 03B9 +1FA5>1F65 03B9 +1FA6>1F66 03B9 +1FA7>1F67 03B9 +1FA8>1F60 03B9 +1FA9>1F61 03B9 +1FAA>1F62 03B9 +1FAB>1F63 03B9 +1FAC>1F64 03B9 +1FAD>1F65 03B9 +1FAE>1F66 03B9 +1FAF>1F67 03B9 +1FB2>1F70 03B9 +1FB3>03B1 03B9 +1FB4>03AC 03B9 +1FB7>1FB6 03B9 +1FB8>1FB0 +1FB9>1FB1 +1FBA>1F70 +1FBB>03AC +1FBC>03B1 03B9 +1FBD>0020 0313 +1FBE>03B9 +1FBF>0020 0313 +1FC0>0020 0342 +1FC1>0020 0308 0342 +1FC2>1F74 03B9 +1FC3>03B7 03B9 +1FC4>03AE 03B9 +1FC7>1FC6 03B9 +1FC8>1F72 +1FC9>03AD +1FCA>1F74 +1FCB>03AE +1FCC>03B7 03B9 +1FCD>0020 0313 0300 +1FCE>0020 0313 0301 +1FCF>0020 0313 0342 +1FD3>0390 +1FD8>1FD0 +1FD9>1FD1 +1FDA>1F76 +1FDB>03AF +1FDD>0020 0314 0300 +1FDE>0020 0314 0301 +1FDF>0020 0314 0342 +1FE3>03B0 +1FE8>1FE0 +1FE9>1FE1 +1FEA>1F7A +1FEB>03CD +1FEC>1FE5 +1FED>0020 0308 0300 +1FEE>0020 0308 0301 +1FEF>0060 +1FF2>1F7C 03B9 +1FF3>03C9 03B9 +1FF4>03CE 03B9 +1FF7>1FF6 03B9 +1FF8>1F78 +1FF9>03CC +1FFA>1F7C +1FFB>03CE +1FFC>03C9 03B9 +1FFD>0020 0301 +1FFE>0020 0314 +2000..200A>0020 +200B..200F> +2011>2010 +2017>0020 0333 +2024>002E +2025>002E 002E +2026>002E 002E 002E +202A..202E> +202F>0020 +2033>2032 2032 +2034>2032 2032 2032 +2036>2035 2035 +2037>2035 2035 2035 +203C>0021 0021 +203E>0020 0305 +2047>003F 003F +2048>003F 0021 +2049>0021 003F +2057>2032 2032 2032 2032 +205F>0020 +2060..2064> +2065..2069> +206A..206F> +2070>0030 +2071>0069 +2074>0034 +2075>0035 +2076>0036 +2077>0037 +2078>0038 +2079>0039 +207A>002B +207B>2212 +207C>003D +207D>0028 +207E>0029 +207F>006E +2080>0030 +2081>0031 +2082>0032 +2083>0033 +2084>0034 +2085>0035 +2086>0036 +2087>0037 +2088>0038 +2089>0039 +208A>002B +208B>2212 +208C>003D +208D>0028 +208E>0029 +2090>0061 +2091>0065 +2092>006F +2093>0078 +2094>0259 +20A8>0072 0073 +2100>0061 002F 0063 +2101>0061 002F 0073 +2102>0063 +2103>00B0 0063 +2105>0063 002F 006F +2106>0063 002F 0075 +2107>025B +2109>00B0 0066 +210A>0067 +210B..210E>0068 +210F>0127 +2110..2111>0069 +2112..2113>006C +2115>006E +2116>006E 006F +2119>0070 +211A>0071 +211B..211D>0072 +2120>0073 006D +2121>0074 0065 006C +2122>0074 006D +2124>007A +2126>03C9 +2128>007A +212A>006B +212B>00E5 +212C>0062 +212D>0063 +212F..2130>0065 +2131>0066 +2132>214E +2133>006D +2134>006F +2135>05D0 +2136>05D1 +2137>05D2 +2138>05D3 +2139>0069 +213B>0066 0061 0078 +213C>03C0 +213D..213E>03B3 +213F>03C0 +2140>2211 +2145..2146>0064 +2147>0065 +2148>0069 +2149>006A +2150>0031 2044 0037 +2151>0031 2044 0039 +2152>0031 2044 0031 0030 +2153>0031 2044 0033 +2154>0032 2044 0033 +2155>0031 2044 0035 +2156>0032 2044 0035 +2157>0033 2044 0035 +2158>0034 2044 0035 +2159>0031 2044 0036 +215A>0035 2044 0036 +215B>0031 2044 0038 +215C>0033 2044 0038 +215D>0035 2044 0038 +215E>0037 2044 0038 +215F>0031 2044 +2160>0069 +2161>0069 0069 +2162>0069 0069 0069 +2163>0069 0076 +2164>0076 +2165>0076 0069 +2166>0076 0069 0069 +2167>0076 0069 0069 0069 +2168>0069 0078 +2169>0078 +216A>0078 0069 +216B>0078 0069 0069 +216C>006C +216D>0063 +216E>0064 +216F>006D +2170>0069 +2171>0069 0069 +2172>0069 0069 0069 +2173>0069 0076 +2174>0076 +2175>0076 0069 +2176>0076 0069 0069 +2177>0076 0069 0069 0069 +2178>0069 0078 +2179>0078 +217A>0078 0069 +217B>0078 0069 0069 +217C>006C +217D>0063 +217E>0064 +217F>006D +2183>2184 +2189>0030 2044 0033 +222C>222B 222B +222D>222B 222B 222B +222F>222E 222E +2230>222E 222E 222E +2329>3008 +232A>3009 +2460>0031 +2461>0032 +2462>0033 +2463>0034 +2464>0035 +2465>0036 +2466>0037 +2467>0038 +2468>0039 +2469>0031 0030 +246A>0031 0031 +246B>0031 0032 +246C>0031 0033 +246D>0031 0034 +246E>0031 0035 +246F>0031 0036 +2470>0031 0037 +2471>0031 0038 +2472>0031 0039 +2473>0032 0030 +2474>0028 0031 0029 +2475>0028 0032 0029 +2476>0028 0033 0029 +2477>0028 0034 0029 +2478>0028 0035 0029 +2479>0028 0036 0029 +247A>0028 0037 0029 +247B>0028 0038 0029 +247C>0028 0039 0029 +247D>0028 0031 0030 0029 +247E>0028 0031 0031 0029 +247F>0028 0031 0032 0029 +2480>0028 0031 0033 0029 +2481>0028 0031 0034 0029 +2482>0028 0031 0035 0029 +2483>0028 0031 0036 0029 +2484>0028 0031 0037 0029 +2485>0028 0031 0038 0029 +2486>0028 0031 0039 0029 +2487>0028 0032 0030 0029 +2488>0031 002E +2489>0032 002E +248A>0033 002E +248B>0034 002E +248C>0035 002E +248D>0036 002E +248E>0037 002E +248F>0038 002E +2490>0039 002E +2491>0031 0030 002E +2492>0031 0031 002E +2493>0031 0032 002E +2494>0031 0033 002E +2495>0031 0034 002E +2496>0031 0035 002E +2497>0031 0036 002E +2498>0031 0037 002E +2499>0031 0038 002E +249A>0031 0039 002E +249B>0032 0030 002E +249C>0028 0061 0029 +249D>0028 0062 0029 +249E>0028 0063 0029 +249F>0028 0064 0029 +24A0>0028 0065 0029 +24A1>0028 0066 0029 +24A2>0028 0067 0029 +24A3>0028 0068 0029 +24A4>0028 0069 0029 +24A5>0028 006A 0029 +24A6>0028 006B 0029 +24A7>0028 006C 0029 +24A8>0028 006D 0029 +24A9>0028 006E 0029 +24AA>0028 006F 0029 +24AB>0028 0070 0029 +24AC>0028 0071 0029 +24AD>0028 0072 0029 +24AE>0028 0073 0029 +24AF>0028 0074 0029 +24B0>0028 0075 0029 +24B1>0028 0076 0029 +24B2>0028 0077 0029 +24B3>0028 0078 0029 +24B4>0028 0079 0029 +24B5>0028 007A 0029 +24B6>0061 +24B7>0062 +24B8>0063 +24B9>0064 +24BA>0065 +24BB>0066 +24BC>0067 +24BD>0068 +24BE>0069 +24BF>006A +24C0>006B +24C1>006C +24C2>006D +24C3>006E +24C4>006F +24C5>0070 +24C6>0071 +24C7>0072 +24C8>0073 +24C9>0074 +24CA>0075 +24CB>0076 +24CC>0077 +24CD>0078 +24CE>0079 +24CF>007A +24D0>0061 +24D1>0062 +24D2>0063 +24D3>0064 +24D4>0065 +24D5>0066 +24D6>0067 +24D7>0068 +24D8>0069 +24D9>006A +24DA>006B +24DB>006C +24DC>006D +24DD>006E +24DE>006F +24DF>0070 +24E0>0071 +24E1>0072 +24E2>0073 +24E3>0074 +24E4>0075 +24E5>0076 +24E6>0077 +24E7>0078 +24E8>0079 +24E9>007A +24EA>0030 +2A0C>222B 222B 222B 222B +2A74>003A 003A 003D +2A75>003D 003D +2A76>003D 003D 003D +2ADC>2ADD 0338 +2C00>2C30 +2C01>2C31 +2C02>2C32 +2C03>2C33 +2C04>2C34 +2C05>2C35 +2C06>2C36 +2C07>2C37 +2C08>2C38 +2C09>2C39 +2C0A>2C3A +2C0B>2C3B +2C0C>2C3C +2C0D>2C3D +2C0E>2C3E +2C0F>2C3F +2C10>2C40 +2C11>2C41 +2C12>2C42 +2C13>2C43 +2C14>2C44 +2C15>2C45 +2C16>2C46 +2C17>2C47 +2C18>2C48 +2C19>2C49 +2C1A>2C4A +2C1B>2C4B +2C1C>2C4C +2C1D>2C4D +2C1E>2C4E +2C1F>2C4F +2C20>2C50 +2C21>2C51 +2C22>2C52 +2C23>2C53 +2C24>2C54 +2C25>2C55 +2C26>2C56 +2C27>2C57 +2C28>2C58 +2C29>2C59 +2C2A>2C5A +2C2B>2C5B +2C2C>2C5C +2C2D>2C5D +2C2E>2C5E +2C60>2C61 +2C62>026B +2C63>1D7D +2C64>027D +2C67>2C68 +2C69>2C6A +2C6B>2C6C +2C6D>0251 +2C6E>0271 +2C6F>0250 +2C70>0252 +2C72>2C73 +2C75>2C76 +2C7C>006A +2C7D>0076 +2C7E>023F +2C7F>0240 +2C80>2C81 +2C82>2C83 +2C84>2C85 +2C86>2C87 +2C88>2C89 +2C8A>2C8B +2C8C>2C8D +2C8E>2C8F +2C90>2C91 +2C92>2C93 +2C94>2C95 +2C96>2C97 +2C98>2C99 +2C9A>2C9B +2C9C>2C9D +2C9E>2C9F +2CA0>2CA1 +2CA2>2CA3 +2CA4>2CA5 +2CA6>2CA7 +2CA8>2CA9 +2CAA>2CAB +2CAC>2CAD +2CAE>2CAF +2CB0>2CB1 +2CB2>2CB3 +2CB4>2CB5 +2CB6>2CB7 +2CB8>2CB9 +2CBA>2CBB +2CBC>2CBD +2CBE>2CBF +2CC0>2CC1 +2CC2>2CC3 +2CC4>2CC5 +2CC6>2CC7 +2CC8>2CC9 +2CCA>2CCB +2CCC>2CCD +2CCE>2CCF +2CD0>2CD1 +2CD2>2CD3 +2CD4>2CD5 +2CD6>2CD7 +2CD8>2CD9 +2CDA>2CDB +2CDC>2CDD +2CDE>2CDF +2CE0>2CE1 +2CE2>2CE3 +2CEB>2CEC +2CED>2CEE +2D6F>2D61 +2E9F>6BCD +2EF3>9F9F +2F00>4E00 +2F01>4E28 +2F02>4E36 +2F03>4E3F +2F04>4E59 +2F05>4E85 +2F06>4E8C +2F07>4EA0 +2F08>4EBA +2F09>513F +2F0A>5165 +2F0B>516B +2F0C>5182 +2F0D>5196 +2F0E>51AB +2F0F>51E0 +2F10>51F5 +2F11>5200 +2F12>529B +2F13>52F9 +2F14>5315 +2F15>531A +2F16>5338 +2F17>5341 +2F18>535C +2F19>5369 +2F1A>5382 +2F1B>53B6 +2F1C>53C8 +2F1D>53E3 +2F1E>56D7 +2F1F>571F +2F20>58EB +2F21>5902 +2F22>590A +2F23>5915 +2F24>5927 +2F25>5973 +2F26>5B50 +2F27>5B80 +2F28>5BF8 +2F29>5C0F +2F2A>5C22 +2F2B>5C38 +2F2C>5C6E +2F2D>5C71 +2F2E>5DDB +2F2F>5DE5 +2F30>5DF1 +2F31>5DFE +2F32>5E72 +2F33>5E7A +2F34>5E7F +2F35>5EF4 +2F36>5EFE +2F37>5F0B +2F38>5F13 +2F39>5F50 +2F3A>5F61 +2F3B>5F73 +2F3C>5FC3 +2F3D>6208 +2F3E>6236 +2F3F>624B +2F40>652F +2F41>6534 +2F42>6587 +2F43>6597 +2F44>65A4 +2F45>65B9 +2F46>65E0 +2F47>65E5 +2F48>66F0 +2F49>6708 +2F4A>6728 +2F4B>6B20 +2F4C>6B62 +2F4D>6B79 +2F4E>6BB3 +2F4F>6BCB +2F50>6BD4 +2F51>6BDB +2F52>6C0F +2F53>6C14 +2F54>6C34 +2F55>706B +2F56>722A +2F57>7236 +2F58>723B +2F59>723F +2F5A>7247 +2F5B>7259 +2F5C>725B +2F5D>72AC +2F5E>7384 +2F5F>7389 +2F60>74DC +2F61>74E6 +2F62>7518 +2F63>751F +2F64>7528 +2F65>7530 +2F66>758B +2F67>7592 +2F68>7676 +2F69>767D +2F6A>76AE +2F6B>76BF +2F6C>76EE +2F6D>77DB +2F6E>77E2 +2F6F>77F3 +2F70>793A +2F71>79B8 +2F72>79BE +2F73>7A74 +2F74>7ACB +2F75>7AF9 +2F76>7C73 +2F77>7CF8 +2F78>7F36 +2F79>7F51 +2F7A>7F8A +2F7B>7FBD +2F7C>8001 +2F7D>800C +2F7E>8012 +2F7F>8033 +2F80>807F +2F81>8089 +2F82>81E3 +2F83>81EA +2F84>81F3 +2F85>81FC +2F86>820C +2F87>821B +2F88>821F +2F89>826E +2F8A>8272 +2F8B>8278 +2F8C>864D +2F8D>866B +2F8E>8840 +2F8F>884C +2F90>8863 +2F91>897E +2F92>898B +2F93>89D2 +2F94>8A00 +2F95>8C37 +2F96>8C46 +2F97>8C55 +2F98>8C78 +2F99>8C9D +2F9A>8D64 +2F9B>8D70 +2F9C>8DB3 +2F9D>8EAB +2F9E>8ECA +2F9F>8F9B +2FA0>8FB0 +2FA1>8FB5 +2FA2>9091 +2FA3>9149 +2FA4>91C6 +2FA5>91CC +2FA6>91D1 +2FA7>9577 +2FA8>9580 +2FA9>961C +2FAA>96B6 +2FAB>96B9 +2FAC>96E8 +2FAD>9751 +2FAE>975E +2FAF>9762 +2FB0>9769 +2FB1>97CB +2FB2>97ED +2FB3>97F3 +2FB4>9801 +2FB5>98A8 +2FB6>98DB +2FB7>98DF +2FB8>9996 +2FB9>9999 +2FBA>99AC +2FBB>9AA8 +2FBC>9AD8 +2FBD>9ADF +2FBE>9B25 +2FBF>9B2F +2FC0>9B32 +2FC1>9B3C +2FC2>9B5A +2FC3>9CE5 +2FC4>9E75 +2FC5>9E7F +2FC6>9EA5 +2FC7>9EBB +2FC8>9EC3 +2FC9>9ECD +2FCA>9ED1 +2FCB>9EF9 +2FCC>9EFD +2FCD>9F0E +2FCE>9F13 +2FCF>9F20 +2FD0>9F3B +2FD1>9F4A +2FD2>9F52 +2FD3>9F8D +2FD4>9F9C +2FD5>9FA0 +3000>0020 +3036>3012 +3038>5341 +3039>5344 +303A>5345 +309B>0020 3099 +309C>0020 309A +309F>3088 308A +30FF>30B3 30C8 +3131>1100 +3132>1101 +3133>11AA +3134>1102 +3135>11AC +3136>11AD +3137>1103 +3138>1104 +3139>1105 +313A>11B0 +313B>11B1 +313C>11B2 +313D>11B3 +313E>11B4 +313F>11B5 +3140>111A +3141>1106 +3142>1107 +3143>1108 +3144>1121 +3145>1109 +3146>110A +3147>110B +3148>110C +3149>110D +314A>110E +314B>110F +314C>1110 +314D>1111 +314E>1112 +314F>1161 +3150>1162 +3151>1163 +3152>1164 +3153>1165 +3154>1166 +3155>1167 +3156>1168 +3157>1169 +3158>116A +3159>116B +315A>116C +315B>116D +315C>116E +315D>116F +315E>1170 +315F>1171 +3160>1172 +3161>1173 +3162>1174 +3163>1175 +3164> +3165>1114 +3166>1115 +3167>11C7 +3168>11C8 +3169>11CC +316A>11CE +316B>11D3 +316C>11D7 +316D>11D9 +316E>111C +316F>11DD +3170>11DF +3171>111D +3172>111E +3173>1120 +3174>1122 +3175>1123 +3176>1127 +3177>1129 +3178>112B +3179>112C +317A>112D +317B>112E +317C>112F +317D>1132 +317E>1136 +317F>1140 +3180>1147 +3181>114C +3182>11F1 +3183>11F2 +3184>1157 +3185>1158 +3186>1159 +3187>1184 +3188>1185 +3189>1188 +318A>1191 +318B>1192 +318C>1194 +318D>119E +318E>11A1 +3192>4E00 +3193>4E8C +3194>4E09 +3195>56DB +3196>4E0A +3197>4E2D +3198>4E0B +3199>7532 +319A>4E59 +319B>4E19 +319C>4E01 +319D>5929 +319E>5730 +319F>4EBA +3200>0028 1100 0029 +3201>0028 1102 0029 +3202>0028 1103 0029 +3203>0028 1105 0029 +3204>0028 1106 0029 +3205>0028 1107 0029 +3206>0028 1109 0029 +3207>0028 110B 0029 +3208>0028 110C 0029 +3209>0028 110E 0029 +320A>0028 110F 0029 +320B>0028 1110 0029 +320C>0028 1111 0029 +320D>0028 1112 0029 +320E>0028 AC00 0029 +320F>0028 B098 0029 +3210>0028 B2E4 0029 +3211>0028 B77C 0029 +3212>0028 B9C8 0029 +3213>0028 BC14 0029 +3214>0028 C0AC 0029 +3215>0028 C544 0029 +3216>0028 C790 0029 +3217>0028 CC28 0029 +3218>0028 CE74 0029 +3219>0028 D0C0 0029 +321A>0028 D30C 0029 +321B>0028 D558 0029 +321C>0028 C8FC 0029 +321D>0028 C624 C804 0029 +321E>0028 C624 D6C4 0029 +3220>0028 4E00 0029 +3221>0028 4E8C 0029 +3222>0028 4E09 0029 +3223>0028 56DB 0029 +3224>0028 4E94 0029 +3225>0028 516D 0029 +3226>0028 4E03 0029 +3227>0028 516B 0029 +3228>0028 4E5D 0029 +3229>0028 5341 0029 +322A>0028 6708 0029 +322B>0028 706B 0029 +322C>0028 6C34 0029 +322D>0028 6728 0029 +322E>0028 91D1 0029 +322F>0028 571F 0029 +3230>0028 65E5 0029 +3231>0028 682A 0029 +3232>0028 6709 0029 +3233>0028 793E 0029 +3234>0028 540D 0029 +3235>0028 7279 0029 +3236>0028 8CA1 0029 +3237>0028 795D 0029 +3238>0028 52B4 0029 +3239>0028 4EE3 0029 +323A>0028 547C 0029 +323B>0028 5B66 0029 +323C>0028 76E3 0029 +323D>0028 4F01 0029 +323E>0028 8CC7 0029 +323F>0028 5354 0029 +3240>0028 796D 0029 +3241>0028 4F11 0029 +3242>0028 81EA 0029 +3243>0028 81F3 0029 +3244>554F +3245>5E7C +3246>6587 +3247>7B8F +3250>0070 0074 0065 +3251>0032 0031 +3252>0032 0032 +3253>0032 0033 +3254>0032 0034 +3255>0032 0035 +3256>0032 0036 +3257>0032 0037 +3258>0032 0038 +3259>0032 0039 +325A>0033 0030 +325B>0033 0031 +325C>0033 0032 +325D>0033 0033 +325E>0033 0034 +325F>0033 0035 +3260>1100 +3261>1102 +3262>1103 +3263>1105 +3264>1106 +3265>1107 +3266>1109 +3267>110B +3268>110C +3269>110E +326A>110F +326B>1110 +326C>1111 +326D>1112 +326E>AC00 +326F>B098 +3270>B2E4 +3271>B77C +3272>B9C8 +3273>BC14 +3274>C0AC +3275>C544 +3276>C790 +3277>CC28 +3278>CE74 +3279>D0C0 +327A>D30C +327B>D558 +327C>CC38 ACE0 +327D>C8FC C758 +327E>C6B0 +3280>4E00 +3281>4E8C +3282>4E09 +3283>56DB +3284>4E94 +3285>516D +3286>4E03 +3287>516B +3288>4E5D +3289>5341 +328A>6708 +328B>706B +328C>6C34 +328D>6728 +328E>91D1 +328F>571F +3290>65E5 +3291>682A +3292>6709 +3293>793E +3294>540D +3295>7279 +3296>8CA1 +3297>795D +3298>52B4 +3299>79D8 +329A>7537 +329B>5973 +329C>9069 +329D>512A +329E>5370 +329F>6CE8 +32A0>9805 +32A1>4F11 +32A2>5199 +32A3>6B63 +32A4>4E0A +32A5>4E2D +32A6>4E0B +32A7>5DE6 +32A8>53F3 +32A9>533B +32AA>5B97 +32AB>5B66 +32AC>76E3 +32AD>4F01 +32AE>8CC7 +32AF>5354 +32B0>591C +32B1>0033 0036 +32B2>0033 0037 +32B3>0033 0038 +32B4>0033 0039 +32B5>0034 0030 +32B6>0034 0031 +32B7>0034 0032 +32B8>0034 0033 +32B9>0034 0034 +32BA>0034 0035 +32BB>0034 0036 +32BC>0034 0037 +32BD>0034 0038 +32BE>0034 0039 +32BF>0035 0030 +32C0>0031 6708 +32C1>0032 6708 +32C2>0033 6708 +32C3>0034 6708 +32C4>0035 6708 +32C5>0036 6708 +32C6>0037 6708 +32C7>0038 6708 +32C8>0039 6708 +32C9>0031 0030 6708 +32CA>0031 0031 6708 +32CB>0031 0032 6708 +32CC>0068 0067 +32CD>0065 0072 0067 +32CE>0065 0076 +32CF>006C 0074 0064 +32D0>30A2 +32D1>30A4 +32D2>30A6 +32D3>30A8 +32D4>30AA +32D5>30AB +32D6>30AD +32D7>30AF +32D8>30B1 +32D9>30B3 +32DA>30B5 +32DB>30B7 +32DC>30B9 +32DD>30BB +32DE>30BD +32DF>30BF +32E0>30C1 +32E1>30C4 +32E2>30C6 +32E3>30C8 +32E4>30CA +32E5>30CB +32E6>30CC +32E7>30CD +32E8>30CE +32E9>30CF +32EA>30D2 +32EB>30D5 +32EC>30D8 +32ED>30DB +32EE>30DE +32EF>30DF +32F0>30E0 +32F1>30E1 +32F2>30E2 +32F3>30E4 +32F4>30E6 +32F5>30E8 +32F6>30E9 +32F7>30EA +32F8>30EB +32F9>30EC +32FA>30ED +32FB>30EF +32FC>30F0 +32FD>30F1 +32FE>30F2 +3300>30A2 30D1 30FC 30C8 +3301>30A2 30EB 30D5 30A1 +3302>30A2 30F3 30DA 30A2 +3303>30A2 30FC 30EB +3304>30A4 30CB 30F3 30B0 +3305>30A4 30F3 30C1 +3306>30A6 30A9 30F3 +3307>30A8 30B9 30AF 30FC 30C9 +3308>30A8 30FC 30AB 30FC +3309>30AA 30F3 30B9 +330A>30AA 30FC 30E0 +330B>30AB 30A4 30EA +330C>30AB 30E9 30C3 30C8 +330D>30AB 30ED 30EA 30FC +330E>30AC 30ED 30F3 +330F>30AC 30F3 30DE +3310>30AE 30AC +3311>30AE 30CB 30FC +3312>30AD 30E5 30EA 30FC +3313>30AE 30EB 30C0 30FC +3314>30AD 30ED +3315>30AD 30ED 30B0 30E9 30E0 +3316>30AD 30ED 30E1 30FC 30C8 30EB +3317>30AD 30ED 30EF 30C3 30C8 +3318>30B0 30E9 30E0 +3319>30B0 30E9 30E0 30C8 30F3 +331A>30AF 30EB 30BC 30A4 30ED +331B>30AF 30ED 30FC 30CD +331C>30B1 30FC 30B9 +331D>30B3 30EB 30CA +331E>30B3 30FC 30DD +331F>30B5 30A4 30AF 30EB +3320>30B5 30F3 30C1 30FC 30E0 +3321>30B7 30EA 30F3 30B0 +3322>30BB 30F3 30C1 +3323>30BB 30F3 30C8 +3324>30C0 30FC 30B9 +3325>30C7 30B7 +3326>30C9 30EB +3327>30C8 30F3 +3328>30CA 30CE +3329>30CE 30C3 30C8 +332A>30CF 30A4 30C4 +332B>30D1 30FC 30BB 30F3 30C8 +332C>30D1 30FC 30C4 +332D>30D0 30FC 30EC 30EB +332E>30D4 30A2 30B9 30C8 30EB +332F>30D4 30AF 30EB +3330>30D4 30B3 +3331>30D3 30EB +3332>30D5 30A1 30E9 30C3 30C9 +3333>30D5 30A3 30FC 30C8 +3334>30D6 30C3 30B7 30A7 30EB +3335>30D5 30E9 30F3 +3336>30D8 30AF 30BF 30FC 30EB +3337>30DA 30BD +3338>30DA 30CB 30D2 +3339>30D8 30EB 30C4 +333A>30DA 30F3 30B9 +333B>30DA 30FC 30B8 +333C>30D9 30FC 30BF +333D>30DD 30A4 30F3 30C8 +333E>30DC 30EB 30C8 +333F>30DB 30F3 +3340>30DD 30F3 30C9 +3341>30DB 30FC 30EB +3342>30DB 30FC 30F3 +3343>30DE 30A4 30AF 30ED +3344>30DE 30A4 30EB +3345>30DE 30C3 30CF +3346>30DE 30EB 30AF +3347>30DE 30F3 30B7 30E7 30F3 +3348>30DF 30AF 30ED 30F3 +3349>30DF 30EA +334A>30DF 30EA 30D0 30FC 30EB +334B>30E1 30AC +334C>30E1 30AC 30C8 30F3 +334D>30E1 30FC 30C8 30EB +334E>30E4 30FC 30C9 +334F>30E4 30FC 30EB +3350>30E6 30A2 30F3 +3351>30EA 30C3 30C8 30EB +3352>30EA 30E9 +3353>30EB 30D4 30FC +3354>30EB 30FC 30D6 30EB +3355>30EC 30E0 +3356>30EC 30F3 30C8 30B2 30F3 +3357>30EF 30C3 30C8 +3358>0030 70B9 +3359>0031 70B9 +335A>0032 70B9 +335B>0033 70B9 +335C>0034 70B9 +335D>0035 70B9 +335E>0036 70B9 +335F>0037 70B9 +3360>0038 70B9 +3361>0039 70B9 +3362>0031 0030 70B9 +3363>0031 0031 70B9 +3364>0031 0032 70B9 +3365>0031 0033 70B9 +3366>0031 0034 70B9 +3367>0031 0035 70B9 +3368>0031 0036 70B9 +3369>0031 0037 70B9 +336A>0031 0038 70B9 +336B>0031 0039 70B9 +336C>0032 0030 70B9 +336D>0032 0031 70B9 +336E>0032 0032 70B9 +336F>0032 0033 70B9 +3370>0032 0034 70B9 +3371>0068 0070 0061 +3372>0064 0061 +3373>0061 0075 +3374>0062 0061 0072 +3375>006F 0076 +3376>0070 0063 +3377>0064 006D +3378>0064 006D 0032 +3379>0064 006D 0033 +337A>0069 0075 +337B>5E73 6210 +337C>662D 548C +337D>5927 6B63 +337E>660E 6CBB +337F>682A 5F0F 4F1A 793E +3380>0070 0061 +3381>006E 0061 +3382>03BC 0061 +3383>006D 0061 +3384>006B 0061 +3385>006B 0062 +3386>006D 0062 +3387>0067 0062 +3388>0063 0061 006C +3389>006B 0063 0061 006C +338A>0070 0066 +338B>006E 0066 +338C>03BC 0066 +338D>03BC 0067 +338E>006D 0067 +338F>006B 0067 +3390>0068 007A +3391>006B 0068 007A +3392>006D 0068 007A +3393>0067 0068 007A +3394>0074 0068 007A +3395>03BC 006C +3396>006D 006C +3397>0064 006C +3398>006B 006C +3399>0066 006D +339A>006E 006D +339B>03BC 006D +339C>006D 006D +339D>0063 006D +339E>006B 006D +339F>006D 006D 0032 +33A0>0063 006D 0032 +33A1>006D 0032 +33A2>006B 006D 0032 +33A3>006D 006D 0033 +33A4>0063 006D 0033 +33A5>006D 0033 +33A6>006B 006D 0033 +33A7>006D 2215 0073 +33A8>006D 2215 0073 0032 +33A9>0070 0061 +33AA>006B 0070 0061 +33AB>006D 0070 0061 +33AC>0067 0070 0061 +33AD>0072 0061 0064 +33AE>0072 0061 0064 2215 0073 +33AF>0072 0061 0064 2215 0073 0032 +33B0>0070 0073 +33B1>006E 0073 +33B2>03BC 0073 +33B3>006D 0073 +33B4>0070 0076 +33B5>006E 0076 +33B6>03BC 0076 +33B7>006D 0076 +33B8>006B 0076 +33B9>006D 0076 +33BA>0070 0077 +33BB>006E 0077 +33BC>03BC 0077 +33BD>006D 0077 +33BE>006B 0077 +33BF>006D 0077 +33C0>006B 03C9 +33C1>006D 03C9 +33C2>0061 002E 006D 002E +33C3>0062 0071 +33C4>0063 0063 +33C5>0063 0064 +33C6>0063 2215 006B 0067 +33C7>0063 006F 002E +33C8>0064 0062 +33C9>0067 0079 +33CA>0068 0061 +33CB>0068 0070 +33CC>0069 006E +33CD>006B 006B +33CE>006B 006D +33CF>006B 0074 +33D0>006C 006D +33D1>006C 006E +33D2>006C 006F 0067 +33D3>006C 0078 +33D4>006D 0062 +33D5>006D 0069 006C +33D6>006D 006F 006C +33D7>0070 0068 +33D8>0070 002E 006D 002E +33D9>0070 0070 006D +33DA>0070 0072 +33DB>0073 0072 +33DC>0073 0076 +33DD>0077 0062 +33DE>0076 2215 006D +33DF>0061 2215 006D +33E0>0031 65E5 +33E1>0032 65E5 +33E2>0033 65E5 +33E3>0034 65E5 +33E4>0035 65E5 +33E5>0036 65E5 +33E6>0037 65E5 +33E7>0038 65E5 +33E8>0039 65E5 +33E9>0031 0030 65E5 +33EA>0031 0031 65E5 +33EB>0031 0032 65E5 +33EC>0031 0033 65E5 +33ED>0031 0034 65E5 +33EE>0031 0035 65E5 +33EF>0031 0036 65E5 +33F0>0031 0037 65E5 +33F1>0031 0038 65E5 +33F2>0031 0039 65E5 +33F3>0032 0030 65E5 +33F4>0032 0031 65E5 +33F5>0032 0032 65E5 +33F6>0032 0033 65E5 +33F7>0032 0034 65E5 +33F8>0032 0035 65E5 +33F9>0032 0036 65E5 +33FA>0032 0037 65E5 +33FB>0032 0038 65E5 +33FC>0032 0039 65E5 +33FD>0033 0030 65E5 +33FE>0033 0031 65E5 +33FF>0067 0061 006C +A640>A641 +A642>A643 +A644>A645 +A646>A647 +A648>A649 +A64A>A64B +A64C>A64D +A64E>A64F +A650>A651 +A652>A653 +A654>A655 +A656>A657 +A658>A659 +A65A>A65B +A65C>A65D +A65E>A65F +A662>A663 +A664>A665 +A666>A667 +A668>A669 +A66A>A66B +A66C>A66D +A680>A681 +A682>A683 +A684>A685 +A686>A687 +A688>A689 +A68A>A68B +A68C>A68D +A68E>A68F +A690>A691 +A692>A693 +A694>A695 +A696>A697 +A722>A723 +A724>A725 +A726>A727 +A728>A729 +A72A>A72B +A72C>A72D +A72E>A72F +A732>A733 +A734>A735 +A736>A737 +A738>A739 +A73A>A73B +A73C>A73D +A73E>A73F +A740>A741 +A742>A743 +A744>A745 +A746>A747 +A748>A749 +A74A>A74B +A74C>A74D +A74E>A74F +A750>A751 +A752>A753 +A754>A755 +A756>A757 +A758>A759 +A75A>A75B +A75C>A75D +A75E>A75F +A760>A761 +A762>A763 +A764>A765 +A766>A767 +A768>A769 +A76A>A76B +A76C>A76D +A76E>A76F +A770>A76F +A779>A77A +A77B>A77C +A77D>1D79 +A77E>A77F +A780>A781 +A782>A783 +A784>A785 +A786>A787 +A78B>A78C +F900>8C48 +F901>66F4 +F902>8ECA +F903>8CC8 +F904>6ED1 +F905>4E32 +F906>53E5 +F907..F908>9F9C +F909>5951 +F90A>91D1 +F90B>5587 +F90C>5948 +F90D>61F6 +F90E>7669 +F90F>7F85 +F910>863F +F911>87BA +F912>88F8 +F913>908F +F914>6A02 +F915>6D1B +F916>70D9 +F917>73DE +F918>843D +F919>916A +F91A>99F1 +F91B>4E82 +F91C>5375 +F91D>6B04 +F91E>721B +F91F>862D +F920>9E1E +F921>5D50 +F922>6FEB +F923>85CD +F924>8964 +F925>62C9 +F926>81D8 +F927>881F +F928>5ECA +F929>6717 +F92A>6D6A +F92B>72FC +F92C>90CE +F92D>4F86 +F92E>51B7 +F92F>52DE +F930>64C4 +F931>6AD3 +F932>7210 +F933>76E7 +F934>8001 +F935>8606 +F936>865C +F937>8DEF +F938>9732 +F939>9B6F +F93A>9DFA +F93B>788C +F93C>797F +F93D>7DA0 +F93E>83C9 +F93F>9304 +F940>9E7F +F941>8AD6 +F942>58DF +F943>5F04 +F944>7C60 +F945>807E +F946>7262 +F947>78CA +F948>8CC2 +F949>96F7 +F94A>58D8 +F94B>5C62 +F94C>6A13 +F94D>6DDA +F94E>6F0F +F94F>7D2F +F950>7E37 +F951>964B +F952>52D2 +F953>808B +F954>51DC +F955>51CC +F956>7A1C +F957>7DBE +F958>83F1 +F959>9675 +F95A>8B80 +F95B>62CF +F95C>6A02 +F95D>8AFE +F95E>4E39 +F95F>5BE7 +F960>6012 +F961>7387 +F962>7570 +F963>5317 +F964>78FB +F965>4FBF +F966>5FA9 +F967>4E0D +F968>6CCC +F969>6578 +F96A>7D22 +F96B>53C3 +F96C>585E +F96D>7701 +F96E>8449 +F96F>8AAA +F970>6BBA +F971>8FB0 +F972>6C88 +F973>62FE +F974>82E5 +F975>63A0 +F976>7565 +F977>4EAE +F978>5169 +F979>51C9 +F97A>6881 +F97B>7CE7 +F97C>826F +F97D>8AD2 +F97E>91CF +F97F>52F5 +F980>5442 +F981>5973 +F982>5EEC +F983>65C5 +F984>6FFE +F985>792A +F986>95AD +F987>9A6A +F988>9E97 +F989>9ECE +F98A>529B +F98B>66C6 +F98C>6B77 +F98D>8F62 +F98E>5E74 +F98F>6190 +F990>6200 +F991>649A +F992>6F23 +F993>7149 +F994>7489 +F995>79CA +F996>7DF4 +F997>806F +F998>8F26 +F999>84EE +F99A>9023 +F99B>934A +F99C>5217 +F99D>52A3 +F99E>54BD +F99F>70C8 +F9A0>88C2 +F9A1>8AAA +F9A2>5EC9 +F9A3>5FF5 +F9A4>637B +F9A5>6BAE +F9A6>7C3E +F9A7>7375 +F9A8>4EE4 +F9A9>56F9 +F9AA>5BE7 +F9AB>5DBA +F9AC>601C +F9AD>73B2 +F9AE>7469 +F9AF>7F9A +F9B0>8046 +F9B1>9234 +F9B2>96F6 +F9B3>9748 +F9B4>9818 +F9B5>4F8B +F9B6>79AE +F9B7>91B4 +F9B8>96B8 +F9B9>60E1 +F9BA>4E86 +F9BB>50DA +F9BC>5BEE +F9BD>5C3F +F9BE>6599 +F9BF>6A02 +F9C0>71CE +F9C1>7642 +F9C2>84FC +F9C3>907C +F9C4>9F8D +F9C5>6688 +F9C6>962E +F9C7>5289 +F9C8>677B +F9C9>67F3 +F9CA>6D41 +F9CB>6E9C +F9CC>7409 +F9CD>7559 +F9CE>786B +F9CF>7D10 +F9D0>985E +F9D1>516D +F9D2>622E +F9D3>9678 +F9D4>502B +F9D5>5D19 +F9D6>6DEA +F9D7>8F2A +F9D8>5F8B +F9D9>6144 +F9DA>6817 +F9DB>7387 +F9DC>9686 +F9DD>5229 +F9DE>540F +F9DF>5C65 +F9E0>6613 +F9E1>674E +F9E2>68A8 +F9E3>6CE5 +F9E4>7406 +F9E5>75E2 +F9E6>7F79 +F9E7>88CF +F9E8>88E1 +F9E9>91CC +F9EA>96E2 +F9EB>533F +F9EC>6EBA +F9ED>541D +F9EE>71D0 +F9EF>7498 +F9F0>85FA +F9F1>96A3 +F9F2>9C57 +F9F3>9E9F +F9F4>6797 +F9F5>6DCB +F9F6>81E8 +F9F7>7ACB +F9F8>7B20 +F9F9>7C92 +F9FA>72C0 +F9FB>7099 +F9FC>8B58 +F9FD>4EC0 +F9FE>8336 +F9FF>523A +FA00>5207 +FA01>5EA6 +FA02>62D3 +FA03>7CD6 +FA04>5B85 +FA05>6D1E +FA06>66B4 +FA07>8F3B +FA08>884C +FA09>964D +FA0A>898B +FA0B>5ED3 +FA0C>5140 +FA0D>55C0 +FA10>585A +FA12>6674 +FA15>51DE +FA16>732A +FA17>76CA +FA18>793C +FA19>795E +FA1A>7965 +FA1B>798F +FA1C>9756 +FA1D>7CBE +FA1E>7FBD +FA20>8612 +FA22>8AF8 +FA25>9038 +FA26>90FD +FA2A>98EF +FA2B>98FC +FA2C>9928 +FA2D>9DB4 +FA30>4FAE +FA31>50E7 +FA32>514D +FA33>52C9 +FA34>52E4 +FA35>5351 +FA36>559D +FA37>5606 +FA38>5668 +FA39>5840 +FA3A>58A8 +FA3B>5C64 +FA3C>5C6E +FA3D>6094 +FA3E>6168 +FA3F>618E +FA40>61F2 +FA41>654F +FA42>65E2 +FA43>6691 +FA44>6885 +FA45>6D77 +FA46>6E1A +FA47>6F22 +FA48>716E +FA49>722B +FA4A>7422 +FA4B>7891 +FA4C>793E +FA4D>7949 +FA4E>7948 +FA4F>7950 +FA50>7956 +FA51>795D +FA52>798D +FA53>798E +FA54>7A40 +FA55>7A81 +FA56>7BC0 +FA57>7DF4 +FA58>7E09 +FA59>7E41 +FA5A>7F72 +FA5B>8005 +FA5C>81ED +FA5D..FA5E>8279 +FA5F>8457 +FA60>8910 +FA61>8996 +FA62>8B01 +FA63>8B39 +FA64>8CD3 +FA65>8D08 +FA66>8FB6 +FA67>9038 +FA68>96E3 +FA69>97FF +FA6A>983B +FA6B>6075 +FA6C>242EE +FA6D>8218 +FA70>4E26 +FA71>51B5 +FA72>5168 +FA73>4F80 +FA74>5145 +FA75>5180 +FA76>52C7 +FA77>52FA +FA78>559D +FA79>5555 +FA7A>5599 +FA7B>55E2 +FA7C>585A +FA7D>58B3 +FA7E>5944 +FA7F>5954 +FA80>5A62 +FA81>5B28 +FA82>5ED2 +FA83>5ED9 +FA84>5F69 +FA85>5FAD +FA86>60D8 +FA87>614E +FA88>6108 +FA89>618E +FA8A>6160 +FA8B>61F2 +FA8C>6234 +FA8D>63C4 +FA8E>641C +FA8F>6452 +FA90>6556 +FA91>6674 +FA92>6717 +FA93>671B +FA94>6756 +FA95>6B79 +FA96>6BBA +FA97>6D41 +FA98>6EDB +FA99>6ECB +FA9A>6F22 +FA9B>701E +FA9C>716E +FA9D>77A7 +FA9E>7235 +FA9F>72AF +FAA0>732A +FAA1>7471 +FAA2>7506 +FAA3>753B +FAA4>761D +FAA5>761F +FAA6>76CA +FAA7>76DB +FAA8>76F4 +FAA9>774A +FAAA>7740 +FAAB>78CC +FAAC>7AB1 +FAAD>7BC0 +FAAE>7C7B +FAAF>7D5B +FAB0>7DF4 +FAB1>7F3E +FAB2>8005 +FAB3>8352 +FAB4>83EF +FAB5>8779 +FAB6>8941 +FAB7>8986 +FAB8>8996 +FAB9>8ABF +FABA>8AF8 +FABB>8ACB +FABC>8B01 +FABD>8AFE +FABE>8AED +FABF>8B39 +FAC0>8B8A +FAC1>8D08 +FAC2>8F38 +FAC3>9072 +FAC4>9199 +FAC5>9276 +FAC6>967C +FAC7>96E3 +FAC8>9756 +FAC9>97DB +FACA>97FF +FACB>980B +FACC>983B +FACD>9B12 +FACE>9F9C +FACF>2284A +FAD0>22844 +FAD1>233D5 +FAD2>3B9D +FAD3>4018 +FAD4>4039 +FAD5>25249 +FAD6>25CD0 +FAD7>27ED3 +FAD8>9F43 +FAD9>9F8E +FB00>0066 0066 +FB01>0066 0069 +FB02>0066 006C +FB03>0066 0066 0069 +FB04>0066 0066 006C +FB05..FB06>0073 0074 +FB13>0574 0576 +FB14>0574 0565 +FB15>0574 056B +FB16>057E 0576 +FB17>0574 056D +FB1D>05D9 05B4 +FB1F>05F2 05B7 +FB20>05E2 +FB21>05D0 +FB22>05D3 +FB23>05D4 +FB24>05DB +FB25>05DC +FB26>05DD +FB27>05E8 +FB28>05EA +FB29>002B +FB2A>05E9 05C1 +FB2B>05E9 05C2 +FB2C>05E9 05BC 05C1 +FB2D>05E9 05BC 05C2 +FB2E>05D0 05B7 +FB2F>05D0 05B8 +FB30>05D0 05BC +FB31>05D1 05BC +FB32>05D2 05BC +FB33>05D3 05BC +FB34>05D4 05BC +FB35>05D5 05BC +FB36>05D6 05BC +FB38>05D8 05BC +FB39>05D9 05BC +FB3A>05DA 05BC +FB3B>05DB 05BC +FB3C>05DC 05BC +FB3E>05DE 05BC +FB40>05E0 05BC +FB41>05E1 05BC +FB43>05E3 05BC +FB44>05E4 05BC +FB46>05E6 05BC +FB47>05E7 05BC +FB48>05E8 05BC +FB49>05E9 05BC +FB4A>05EA 05BC +FB4B>05D5 05B9 +FB4C>05D1 05BF +FB4D>05DB 05BF +FB4E>05E4 05BF +FB4F>05D0 05DC +FB50..FB51>0671 +FB52..FB55>067B +FB56..FB59>067E +FB5A..FB5D>0680 +FB5E..FB61>067A +FB62..FB65>067F +FB66..FB69>0679 +FB6A..FB6D>06A4 +FB6E..FB71>06A6 +FB72..FB75>0684 +FB76..FB79>0683 +FB7A..FB7D>0686 +FB7E..FB81>0687 +FB82..FB83>068D +FB84..FB85>068C +FB86..FB87>068E +FB88..FB89>0688 +FB8A..FB8B>0698 +FB8C..FB8D>0691 +FB8E..FB91>06A9 +FB92..FB95>06AF +FB96..FB99>06B3 +FB9A..FB9D>06B1 +FB9E..FB9F>06BA +FBA0..FBA3>06BB +FBA4..FBA5>06C0 +FBA6..FBA9>06C1 +FBAA..FBAD>06BE +FBAE..FBAF>06D2 +FBB0..FBB1>06D3 +FBD3..FBD6>06AD +FBD7..FBD8>06C7 +FBD9..FBDA>06C6 +FBDB..FBDC>06C8 +FBDD>06C7 0674 +FBDE..FBDF>06CB +FBE0..FBE1>06C5 +FBE2..FBE3>06C9 +FBE4..FBE7>06D0 +FBE8..FBE9>0649 +FBEA..FBEB>0626 0627 +FBEC..FBED>0626 06D5 +FBEE..FBEF>0626 0648 +FBF0..FBF1>0626 06C7 +FBF2..FBF3>0626 06C6 +FBF4..FBF5>0626 06C8 +FBF6..FBF8>0626 06D0 +FBF9..FBFB>0626 0649 +FBFC..FBFF>06CC +FC00>0626 062C +FC01>0626 062D +FC02>0626 0645 +FC03>0626 0649 +FC04>0626 064A +FC05>0628 062C +FC06>0628 062D +FC07>0628 062E +FC08>0628 0645 +FC09>0628 0649 +FC0A>0628 064A +FC0B>062A 062C +FC0C>062A 062D +FC0D>062A 062E +FC0E>062A 0645 +FC0F>062A 0649 +FC10>062A 064A +FC11>062B 062C +FC12>062B 0645 +FC13>062B 0649 +FC14>062B 064A +FC15>062C 062D +FC16>062C 0645 +FC17>062D 062C +FC18>062D 0645 +FC19>062E 062C +FC1A>062E 062D +FC1B>062E 0645 +FC1C>0633 062C +FC1D>0633 062D +FC1E>0633 062E +FC1F>0633 0645 +FC20>0635 062D +FC21>0635 0645 +FC22>0636 062C +FC23>0636 062D +FC24>0636 062E +FC25>0636 0645 +FC26>0637 062D +FC27>0637 0645 +FC28>0638 0645 +FC29>0639 062C +FC2A>0639 0645 +FC2B>063A 062C +FC2C>063A 0645 +FC2D>0641 062C +FC2E>0641 062D +FC2F>0641 062E +FC30>0641 0645 +FC31>0641 0649 +FC32>0641 064A +FC33>0642 062D +FC34>0642 0645 +FC35>0642 0649 +FC36>0642 064A +FC37>0643 0627 +FC38>0643 062C +FC39>0643 062D +FC3A>0643 062E +FC3B>0643 0644 +FC3C>0643 0645 +FC3D>0643 0649 +FC3E>0643 064A +FC3F>0644 062C +FC40>0644 062D +FC41>0644 062E +FC42>0644 0645 +FC43>0644 0649 +FC44>0644 064A +FC45>0645 062C +FC46>0645 062D +FC47>0645 062E +FC48>0645 0645 +FC49>0645 0649 +FC4A>0645 064A +FC4B>0646 062C +FC4C>0646 062D +FC4D>0646 062E +FC4E>0646 0645 +FC4F>0646 0649 +FC50>0646 064A +FC51>0647 062C +FC52>0647 0645 +FC53>0647 0649 +FC54>0647 064A +FC55>064A 062C +FC56>064A 062D +FC57>064A 062E +FC58>064A 0645 +FC59>064A 0649 +FC5A>064A 064A +FC5B>0630 0670 +FC5C>0631 0670 +FC5D>0649 0670 +FC5E>0020 064C 0651 +FC5F>0020 064D 0651 +FC60>0020 064E 0651 +FC61>0020 064F 0651 +FC62>0020 0650 0651 +FC63>0020 0651 0670 +FC64>0626 0631 +FC65>0626 0632 +FC66>0626 0645 +FC67>0626 0646 +FC68>0626 0649 +FC69>0626 064A +FC6A>0628 0631 +FC6B>0628 0632 +FC6C>0628 0645 +FC6D>0628 0646 +FC6E>0628 0649 +FC6F>0628 064A +FC70>062A 0631 +FC71>062A 0632 +FC72>062A 0645 +FC73>062A 0646 +FC74>062A 0649 +FC75>062A 064A +FC76>062B 0631 +FC77>062B 0632 +FC78>062B 0645 +FC79>062B 0646 +FC7A>062B 0649 +FC7B>062B 064A +FC7C>0641 0649 +FC7D>0641 064A +FC7E>0642 0649 +FC7F>0642 064A +FC80>0643 0627 +FC81>0643 0644 +FC82>0643 0645 +FC83>0643 0649 +FC84>0643 064A +FC85>0644 0645 +FC86>0644 0649 +FC87>0644 064A +FC88>0645 0627 +FC89>0645 0645 +FC8A>0646 0631 +FC8B>0646 0632 +FC8C>0646 0645 +FC8D>0646 0646 +FC8E>0646 0649 +FC8F>0646 064A +FC90>0649 0670 +FC91>064A 0631 +FC92>064A 0632 +FC93>064A 0645 +FC94>064A 0646 +FC95>064A 0649 +FC96>064A 064A +FC97>0626 062C +FC98>0626 062D +FC99>0626 062E +FC9A>0626 0645 +FC9B>0626 0647 +FC9C>0628 062C +FC9D>0628 062D +FC9E>0628 062E +FC9F>0628 0645 +FCA0>0628 0647 +FCA1>062A 062C +FCA2>062A 062D +FCA3>062A 062E +FCA4>062A 0645 +FCA5>062A 0647 +FCA6>062B 0645 +FCA7>062C 062D +FCA8>062C 0645 +FCA9>062D 062C +FCAA>062D 0645 +FCAB>062E 062C +FCAC>062E 0645 +FCAD>0633 062C +FCAE>0633 062D +FCAF>0633 062E +FCB0>0633 0645 +FCB1>0635 062D +FCB2>0635 062E +FCB3>0635 0645 +FCB4>0636 062C +FCB5>0636 062D +FCB6>0636 062E +FCB7>0636 0645 +FCB8>0637 062D +FCB9>0638 0645 +FCBA>0639 062C +FCBB>0639 0645 +FCBC>063A 062C +FCBD>063A 0645 +FCBE>0641 062C +FCBF>0641 062D +FCC0>0641 062E +FCC1>0641 0645 +FCC2>0642 062D +FCC3>0642 0645 +FCC4>0643 062C +FCC5>0643 062D +FCC6>0643 062E +FCC7>0643 0644 +FCC8>0643 0645 +FCC9>0644 062C +FCCA>0644 062D +FCCB>0644 062E +FCCC>0644 0645 +FCCD>0644 0647 +FCCE>0645 062C +FCCF>0645 062D +FCD0>0645 062E +FCD1>0645 0645 +FCD2>0646 062C +FCD3>0646 062D +FCD4>0646 062E +FCD5>0646 0645 +FCD6>0646 0647 +FCD7>0647 062C +FCD8>0647 0645 +FCD9>0647 0670 +FCDA>064A 062C +FCDB>064A 062D +FCDC>064A 062E +FCDD>064A 0645 +FCDE>064A 0647 +FCDF>0626 0645 +FCE0>0626 0647 +FCE1>0628 0645 +FCE2>0628 0647 +FCE3>062A 0645 +FCE4>062A 0647 +FCE5>062B 0645 +FCE6>062B 0647 +FCE7>0633 0645 +FCE8>0633 0647 +FCE9>0634 0645 +FCEA>0634 0647 +FCEB>0643 0644 +FCEC>0643 0645 +FCED>0644 0645 +FCEE>0646 0645 +FCEF>0646 0647 +FCF0>064A 0645 +FCF1>064A 0647 +FCF2>0640 064E 0651 +FCF3>0640 064F 0651 +FCF4>0640 0650 0651 +FCF5>0637 0649 +FCF6>0637 064A +FCF7>0639 0649 +FCF8>0639 064A +FCF9>063A 0649 +FCFA>063A 064A +FCFB>0633 0649 +FCFC>0633 064A +FCFD>0634 0649 +FCFE>0634 064A +FCFF>062D 0649 +FD00>062D 064A +FD01>062C 0649 +FD02>062C 064A +FD03>062E 0649 +FD04>062E 064A +FD05>0635 0649 +FD06>0635 064A +FD07>0636 0649 +FD08>0636 064A +FD09>0634 062C +FD0A>0634 062D +FD0B>0634 062E +FD0C>0634 0645 +FD0D>0634 0631 +FD0E>0633 0631 +FD0F>0635 0631 +FD10>0636 0631 +FD11>0637 0649 +FD12>0637 064A +FD13>0639 0649 +FD14>0639 064A +FD15>063A 0649 +FD16>063A 064A +FD17>0633 0649 +FD18>0633 064A +FD19>0634 0649 +FD1A>0634 064A +FD1B>062D 0649 +FD1C>062D 064A +FD1D>062C 0649 +FD1E>062C 064A +FD1F>062E 0649 +FD20>062E 064A +FD21>0635 0649 +FD22>0635 064A +FD23>0636 0649 +FD24>0636 064A +FD25>0634 062C +FD26>0634 062D +FD27>0634 062E +FD28>0634 0645 +FD29>0634 0631 +FD2A>0633 0631 +FD2B>0635 0631 +FD2C>0636 0631 +FD2D>0634 062C +FD2E>0634 062D +FD2F>0634 062E +FD30>0634 0645 +FD31>0633 0647 +FD32>0634 0647 +FD33>0637 0645 +FD34>0633 062C +FD35>0633 062D +FD36>0633 062E +FD37>0634 062C +FD38>0634 062D +FD39>0634 062E +FD3A>0637 0645 +FD3B>0638 0645 +FD3C..FD3D>0627 064B +FD50>062A 062C 0645 +FD51..FD52>062A 062D 062C +FD53>062A 062D 0645 +FD54>062A 062E 0645 +FD55>062A 0645 062C +FD56>062A 0645 062D +FD57>062A 0645 062E +FD58..FD59>062C 0645 062D +FD5A>062D 0645 064A +FD5B>062D 0645 0649 +FD5C>0633 062D 062C +FD5D>0633 062C 062D +FD5E>0633 062C 0649 +FD5F..FD60>0633 0645 062D +FD61>0633 0645 062C +FD62..FD63>0633 0645 0645 +FD64..FD65>0635 062D 062D +FD66>0635 0645 0645 +FD67..FD68>0634 062D 0645 +FD69>0634 062C 064A +FD6A..FD6B>0634 0645 062E +FD6C..FD6D>0634 0645 0645 +FD6E>0636 062D 0649 +FD6F..FD70>0636 062E 0645 +FD71..FD72>0637 0645 062D +FD73>0637 0645 0645 +FD74>0637 0645 064A +FD75>0639 062C 0645 +FD76..FD77>0639 0645 0645 +FD78>0639 0645 0649 +FD79>063A 0645 0645 +FD7A>063A 0645 064A +FD7B>063A 0645 0649 +FD7C..FD7D>0641 062E 0645 +FD7E>0642 0645 062D +FD7F>0642 0645 0645 +FD80>0644 062D 0645 +FD81>0644 062D 064A +FD82>0644 062D 0649 +FD83..FD84>0644 062C 062C +FD85..FD86>0644 062E 0645 +FD87..FD88>0644 0645 062D +FD89>0645 062D 062C +FD8A>0645 062D 0645 +FD8B>0645 062D 064A +FD8C>0645 062C 062D +FD8D>0645 062C 0645 +FD8E>0645 062E 062C +FD8F>0645 062E 0645 +FD92>0645 062C 062E +FD93>0647 0645 062C +FD94>0647 0645 0645 +FD95>0646 062D 0645 +FD96>0646 062D 0649 +FD97..FD98>0646 062C 0645 +FD99>0646 062C 0649 +FD9A>0646 0645 064A +FD9B>0646 0645 0649 +FD9C..FD9D>064A 0645 0645 +FD9E>0628 062E 064A +FD9F>062A 062C 064A +FDA0>062A 062C 0649 +FDA1>062A 062E 064A +FDA2>062A 062E 0649 +FDA3>062A 0645 064A +FDA4>062A 0645 0649 +FDA5>062C 0645 064A +FDA6>062C 062D 0649 +FDA7>062C 0645 0649 +FDA8>0633 062E 0649 +FDA9>0635 062D 064A +FDAA>0634 062D 064A +FDAB>0636 062D 064A +FDAC>0644 062C 064A +FDAD>0644 0645 064A +FDAE>064A 062D 064A +FDAF>064A 062C 064A +FDB0>064A 0645 064A +FDB1>0645 0645 064A +FDB2>0642 0645 064A +FDB3>0646 062D 064A +FDB4>0642 0645 062D +FDB5>0644 062D 0645 +FDB6>0639 0645 064A +FDB7>0643 0645 064A +FDB8>0646 062C 062D +FDB9>0645 062E 064A +FDBA>0644 062C 0645 +FDBB>0643 0645 0645 +FDBC>0644 062C 0645 +FDBD>0646 062C 062D +FDBE>062C 062D 064A +FDBF>062D 062C 064A +FDC0>0645 062C 064A +FDC1>0641 0645 064A +FDC2>0628 062D 064A +FDC3>0643 0645 0645 +FDC4>0639 062C 0645 +FDC5>0635 0645 0645 +FDC6>0633 062E 064A +FDC7>0646 062C 064A +FDF0>0635 0644 06D2 +FDF1>0642 0644 06D2 +FDF2>0627 0644 0644 0647 +FDF3>0627 0643 0628 0631 +FDF4>0645 062D 0645 062F +FDF5>0635 0644 0639 0645 +FDF6>0631 0633 0648 0644 +FDF7>0639 0644 064A 0647 +FDF8>0648 0633 0644 0645 +FDF9>0635 0644 0649 +FDFA>0635 0644 0649 0020 0627 0644 0644 0647 0020 0639 0644 064A 0647 0020 0648 0633 0644 0645 +FDFB>062C 0644 0020 062C 0644 0627 0644 0647 +FDFC>0631 06CC 0627 0644 +FE00..FE0F> +FE10>002C +FE11>3001 +FE12>3002 +FE13>003A +FE14>003B +FE15>0021 +FE16>003F +FE17>3016 +FE18>3017 +FE19>002E 002E 002E +FE30>002E 002E +FE31>2014 +FE32>2013 +FE33..FE34>005F +FE35>0028 +FE36>0029 +FE37>007B +FE38>007D +FE39>3014 +FE3A>3015 +FE3B>3010 +FE3C>3011 +FE3D>300A +FE3E>300B +FE3F>3008 +FE40>3009 +FE41>300C +FE42>300D +FE43>300E +FE44>300F +FE47>005B +FE48>005D +FE49..FE4C>0020 0305 +FE4D..FE4F>005F +FE50>002C +FE51>3001 +FE52>002E +FE54>003B +FE55>003A +FE56>003F +FE57>0021 +FE58>2014 +FE59>0028 +FE5A>0029 +FE5B>007B +FE5C>007D +FE5D>3014 +FE5E>3015 +FE5F>0023 +FE60>0026 +FE61>002A +FE62>002B +FE63>002D +FE64>003C +FE65>003E +FE66>003D +FE68>005C +FE69>0024 +FE6A>0025 +FE6B>0040 +FE70>0020 064B +FE71>0640 064B +FE72>0020 064C +FE74>0020 064D +FE76>0020 064E +FE77>0640 064E +FE78>0020 064F +FE79>0640 064F +FE7A>0020 0650 +FE7B>0640 0650 +FE7C>0020 0651 +FE7D>0640 0651 +FE7E>0020 0652 +FE7F>0640 0652 +FE80>0621 +FE81..FE82>0622 +FE83..FE84>0623 +FE85..FE86>0624 +FE87..FE88>0625 +FE89..FE8C>0626 +FE8D..FE8E>0627 +FE8F..FE92>0628 +FE93..FE94>0629 +FE95..FE98>062A +FE99..FE9C>062B +FE9D..FEA0>062C +FEA1..FEA4>062D +FEA5..FEA8>062E +FEA9..FEAA>062F +FEAB..FEAC>0630 +FEAD..FEAE>0631 +FEAF..FEB0>0632 +FEB1..FEB4>0633 +FEB5..FEB8>0634 +FEB9..FEBC>0635 +FEBD..FEC0>0636 +FEC1..FEC4>0637 +FEC5..FEC8>0638 +FEC9..FECC>0639 +FECD..FED0>063A +FED1..FED4>0641 +FED5..FED8>0642 +FED9..FEDC>0643 +FEDD..FEE0>0644 +FEE1..FEE4>0645 +FEE5..FEE8>0646 +FEE9..FEEC>0647 +FEED..FEEE>0648 +FEEF..FEF0>0649 +FEF1..FEF4>064A +FEF5..FEF6>0644 0622 +FEF7..FEF8>0644 0623 +FEF9..FEFA>0644 0625 +FEFB..FEFC>0644 0627 +FEFF> +FF01>0021 +FF02>0022 +FF03>0023 +FF04>0024 +FF05>0025 +FF06>0026 +FF07>0027 +FF08>0028 +FF09>0029 +FF0A>002A +FF0B>002B +FF0C>002C +FF0D>002D +FF0E>002E +FF0F>002F +FF10>0030 +FF11>0031 +FF12>0032 +FF13>0033 +FF14>0034 +FF15>0035 +FF16>0036 +FF17>0037 +FF18>0038 +FF19>0039 +FF1A>003A +FF1B>003B +FF1C>003C +FF1D>003D +FF1E>003E +FF1F>003F +FF20>0040 +FF21>0061 +FF22>0062 +FF23>0063 +FF24>0064 +FF25>0065 +FF26>0066 +FF27>0067 +FF28>0068 +FF29>0069 +FF2A>006A +FF2B>006B +FF2C>006C +FF2D>006D +FF2E>006E +FF2F>006F +FF30>0070 +FF31>0071 +FF32>0072 +FF33>0073 +FF34>0074 +FF35>0075 +FF36>0076 +FF37>0077 +FF38>0078 +FF39>0079 +FF3A>007A +FF3B>005B +FF3C>005C +FF3D>005D +FF3E>005E +FF3F>005F +FF40>0060 +FF41>0061 +FF42>0062 +FF43>0063 +FF44>0064 +FF45>0065 +FF46>0066 +FF47>0067 +FF48>0068 +FF49>0069 +FF4A>006A +FF4B>006B +FF4C>006C +FF4D>006D +FF4E>006E +FF4F>006F +FF50>0070 +FF51>0071 +FF52>0072 +FF53>0073 +FF54>0074 +FF55>0075 +FF56>0076 +FF57>0077 +FF58>0078 +FF59>0079 +FF5A>007A +FF5B>007B +FF5C>007C +FF5D>007D +FF5E>007E +FF5F>2985 +FF60>2986 +FF61>3002 +FF62>300C +FF63>300D +FF64>3001 +FF65>30FB +FF66>30F2 +FF67>30A1 +FF68>30A3 +FF69>30A5 +FF6A>30A7 +FF6B>30A9 +FF6C>30E3 +FF6D>30E5 +FF6E>30E7 +FF6F>30C3 +FF70>30FC +FF71>30A2 +FF72>30A4 +FF73>30A6 +FF74>30A8 +FF75>30AA +FF76>30AB +FF77>30AD +FF78>30AF +FF79>30B1 +FF7A>30B3 +FF7B>30B5 +FF7C>30B7 +FF7D>30B9 +FF7E>30BB +FF7F>30BD +FF80>30BF +FF81>30C1 +FF82>30C4 +FF83>30C6 +FF84>30C8 +FF85>30CA +FF86>30CB +FF87>30CC +FF88>30CD +FF89>30CE +FF8A>30CF +FF8B>30D2 +FF8C>30D5 +FF8D>30D8 +FF8E>30DB +FF8F>30DE +FF90>30DF +FF91>30E0 +FF92>30E1 +FF93>30E2 +FF94>30E4 +FF95>30E6 +FF96>30E8 +FF97>30E9 +FF98>30EA +FF99>30EB +FF9A>30EC +FF9B>30ED +FF9C>30EF +FF9D>30F3 +FF9E>3099 +FF9F>309A +FFA0> +FFA1>1100 +FFA2>1101 +FFA3>11AA +FFA4>1102 +FFA5>11AC +FFA6>11AD +FFA7>1103 +FFA8>1104 +FFA9>1105 +FFAA>11B0 +FFAB>11B1 +FFAC>11B2 +FFAD>11B3 +FFAE>11B4 +FFAF>11B5 +FFB0>111A +FFB1>1106 +FFB2>1107 +FFB3>1108 +FFB4>1121 +FFB5>1109 +FFB6>110A +FFB7>110B +FFB8>110C +FFB9>110D +FFBA>110E +FFBB>110F +FFBC>1110 +FFBD>1111 +FFBE>1112 +FFC2>1161 +FFC3>1162 +FFC4>1163 +FFC5>1164 +FFC6>1165 +FFC7>1166 +FFCA>1167 +FFCB>1168 +FFCC>1169 +FFCD>116A +FFCE>116B +FFCF>116C +FFD2>116D +FFD3>116E +FFD4>116F +FFD5>1170 +FFD6>1171 +FFD7>1172 +FFDA>1173 +FFDB>1174 +FFDC>1175 +FFE0>00A2 +FFE1>00A3 +FFE2>00AC +FFE3>0020 0304 +FFE4>00A6 +FFE5>00A5 +FFE6>20A9 +FFE8>2502 +FFE9>2190 +FFEA>2191 +FFEB>2192 +FFEC>2193 +FFED>25A0 +FFEE>25CB +FFF0..FFF8> +10400>10428 +10401>10429 +10402>1042A +10403>1042B +10404>1042C +10405>1042D +10406>1042E +10407>1042F +10408>10430 +10409>10431 +1040A>10432 +1040B>10433 +1040C>10434 +1040D>10435 +1040E>10436 +1040F>10437 +10410>10438 +10411>10439 +10412>1043A +10413>1043B +10414>1043C +10415>1043D +10416>1043E +10417>1043F +10418>10440 +10419>10441 +1041A>10442 +1041B>10443 +1041C>10444 +1041D>10445 +1041E>10446 +1041F>10447 +10420>10448 +10421>10449 +10422>1044A +10423>1044B +10424>1044C +10425>1044D +10426>1044E +10427>1044F +1D15E>1D157 1D165 +1D15F>1D158 1D165 +1D160>1D158 1D165 1D16E +1D161>1D158 1D165 1D16F +1D162>1D158 1D165 1D170 +1D163>1D158 1D165 1D171 +1D164>1D158 1D165 1D172 +1D173..1D17A> +1D1BB>1D1B9 1D165 +1D1BC>1D1BA 1D165 +1D1BD>1D1B9 1D165 1D16E +1D1BE>1D1BA 1D165 1D16E +1D1BF>1D1B9 1D165 1D16F +1D1C0>1D1BA 1D165 1D16F +1D400>0061 +1D401>0062 +1D402>0063 +1D403>0064 +1D404>0065 +1D405>0066 +1D406>0067 +1D407>0068 +1D408>0069 +1D409>006A +1D40A>006B +1D40B>006C +1D40C>006D +1D40D>006E +1D40E>006F +1D40F>0070 +1D410>0071 +1D411>0072 +1D412>0073 +1D413>0074 +1D414>0075 +1D415>0076 +1D416>0077 +1D417>0078 +1D418>0079 +1D419>007A +1D41A>0061 +1D41B>0062 +1D41C>0063 +1D41D>0064 +1D41E>0065 +1D41F>0066 +1D420>0067 +1D421>0068 +1D422>0069 +1D423>006A +1D424>006B +1D425>006C +1D426>006D +1D427>006E +1D428>006F +1D429>0070 +1D42A>0071 +1D42B>0072 +1D42C>0073 +1D42D>0074 +1D42E>0075 +1D42F>0076 +1D430>0077 +1D431>0078 +1D432>0079 +1D433>007A +1D434>0061 +1D435>0062 +1D436>0063 +1D437>0064 +1D438>0065 +1D439>0066 +1D43A>0067 +1D43B>0068 +1D43C>0069 +1D43D>006A +1D43E>006B +1D43F>006C +1D440>006D +1D441>006E +1D442>006F +1D443>0070 +1D444>0071 +1D445>0072 +1D446>0073 +1D447>0074 +1D448>0075 +1D449>0076 +1D44A>0077 +1D44B>0078 +1D44C>0079 +1D44D>007A +1D44E>0061 +1D44F>0062 +1D450>0063 +1D451>0064 +1D452>0065 +1D453>0066 +1D454>0067 +1D456>0069 +1D457>006A +1D458>006B +1D459>006C +1D45A>006D +1D45B>006E +1D45C>006F +1D45D>0070 +1D45E>0071 +1D45F>0072 +1D460>0073 +1D461>0074 +1D462>0075 +1D463>0076 +1D464>0077 +1D465>0078 +1D466>0079 +1D467>007A +1D468>0061 +1D469>0062 +1D46A>0063 +1D46B>0064 +1D46C>0065 +1D46D>0066 +1D46E>0067 +1D46F>0068 +1D470>0069 +1D471>006A +1D472>006B +1D473>006C +1D474>006D +1D475>006E +1D476>006F +1D477>0070 +1D478>0071 +1D479>0072 +1D47A>0073 +1D47B>0074 +1D47C>0075 +1D47D>0076 +1D47E>0077 +1D47F>0078 +1D480>0079 +1D481>007A +1D482>0061 +1D483>0062 +1D484>0063 +1D485>0064 +1D486>0065 +1D487>0066 +1D488>0067 +1D489>0068 +1D48A>0069 +1D48B>006A +1D48C>006B +1D48D>006C +1D48E>006D +1D48F>006E +1D490>006F +1D491>0070 +1D492>0071 +1D493>0072 +1D494>0073 +1D495>0074 +1D496>0075 +1D497>0076 +1D498>0077 +1D499>0078 +1D49A>0079 +1D49B>007A +1D49C>0061 +1D49E>0063 +1D49F>0064 +1D4A2>0067 +1D4A5>006A +1D4A6>006B +1D4A9>006E +1D4AA>006F +1D4AB>0070 +1D4AC>0071 +1D4AE>0073 +1D4AF>0074 +1D4B0>0075 +1D4B1>0076 +1D4B2>0077 +1D4B3>0078 +1D4B4>0079 +1D4B5>007A +1D4B6>0061 +1D4B7>0062 +1D4B8>0063 +1D4B9>0064 +1D4BB>0066 +1D4BD>0068 +1D4BE>0069 +1D4BF>006A +1D4C0>006B +1D4C1>006C +1D4C2>006D +1D4C3>006E +1D4C5>0070 +1D4C6>0071 +1D4C7>0072 +1D4C8>0073 +1D4C9>0074 +1D4CA>0075 +1D4CB>0076 +1D4CC>0077 +1D4CD>0078 +1D4CE>0079 +1D4CF>007A +1D4D0>0061 +1D4D1>0062 +1D4D2>0063 +1D4D3>0064 +1D4D4>0065 +1D4D5>0066 +1D4D6>0067 +1D4D7>0068 +1D4D8>0069 +1D4D9>006A +1D4DA>006B +1D4DB>006C +1D4DC>006D +1D4DD>006E +1D4DE>006F +1D4DF>0070 +1D4E0>0071 +1D4E1>0072 +1D4E2>0073 +1D4E3>0074 +1D4E4>0075 +1D4E5>0076 +1D4E6>0077 +1D4E7>0078 +1D4E8>0079 +1D4E9>007A +1D4EA>0061 +1D4EB>0062 +1D4EC>0063 +1D4ED>0064 +1D4EE>0065 +1D4EF>0066 +1D4F0>0067 +1D4F1>0068 +1D4F2>0069 +1D4F3>006A +1D4F4>006B +1D4F5>006C +1D4F6>006D +1D4F7>006E +1D4F8>006F +1D4F9>0070 +1D4FA>0071 +1D4FB>0072 +1D4FC>0073 +1D4FD>0074 +1D4FE>0075 +1D4FF>0076 +1D500>0077 +1D501>0078 +1D502>0079 +1D503>007A +1D504>0061 +1D505>0062 +1D507>0064 +1D508>0065 +1D509>0066 +1D50A>0067 +1D50D>006A +1D50E>006B +1D50F>006C +1D510>006D +1D511>006E +1D512>006F +1D513>0070 +1D514>0071 +1D516>0073 +1D517>0074 +1D518>0075 +1D519>0076 +1D51A>0077 +1D51B>0078 +1D51C>0079 +1D51E>0061 +1D51F>0062 +1D520>0063 +1D521>0064 +1D522>0065 +1D523>0066 +1D524>0067 +1D525>0068 +1D526>0069 +1D527>006A +1D528>006B +1D529>006C +1D52A>006D +1D52B>006E +1D52C>006F +1D52D>0070 +1D52E>0071 +1D52F>0072 +1D530>0073 +1D531>0074 +1D532>0075 +1D533>0076 +1D534>0077 +1D535>0078 +1D536>0079 +1D537>007A +1D538>0061 +1D539>0062 +1D53B>0064 +1D53C>0065 +1D53D>0066 +1D53E>0067 +1D540>0069 +1D541>006A +1D542>006B +1D543>006C +1D544>006D +1D546>006F +1D54A>0073 +1D54B>0074 +1D54C>0075 +1D54D>0076 +1D54E>0077 +1D54F>0078 +1D550>0079 +1D552>0061 +1D553>0062 +1D554>0063 +1D555>0064 +1D556>0065 +1D557>0066 +1D558>0067 +1D559>0068 +1D55A>0069 +1D55B>006A +1D55C>006B +1D55D>006C +1D55E>006D +1D55F>006E +1D560>006F +1D561>0070 +1D562>0071 +1D563>0072 +1D564>0073 +1D565>0074 +1D566>0075 +1D567>0076 +1D568>0077 +1D569>0078 +1D56A>0079 +1D56B>007A +1D56C>0061 +1D56D>0062 +1D56E>0063 +1D56F>0064 +1D570>0065 +1D571>0066 +1D572>0067 +1D573>0068 +1D574>0069 +1D575>006A +1D576>006B +1D577>006C +1D578>006D +1D579>006E +1D57A>006F +1D57B>0070 +1D57C>0071 +1D57D>0072 +1D57E>0073 +1D57F>0074 +1D580>0075 +1D581>0076 +1D582>0077 +1D583>0078 +1D584>0079 +1D585>007A +1D586>0061 +1D587>0062 +1D588>0063 +1D589>0064 +1D58A>0065 +1D58B>0066 +1D58C>0067 +1D58D>0068 +1D58E>0069 +1D58F>006A +1D590>006B +1D591>006C +1D592>006D +1D593>006E +1D594>006F +1D595>0070 +1D596>0071 +1D597>0072 +1D598>0073 +1D599>0074 +1D59A>0075 +1D59B>0076 +1D59C>0077 +1D59D>0078 +1D59E>0079 +1D59F>007A +1D5A0>0061 +1D5A1>0062 +1D5A2>0063 +1D5A3>0064 +1D5A4>0065 +1D5A5>0066 +1D5A6>0067 +1D5A7>0068 +1D5A8>0069 +1D5A9>006A +1D5AA>006B +1D5AB>006C +1D5AC>006D +1D5AD>006E +1D5AE>006F +1D5AF>0070 +1D5B0>0071 +1D5B1>0072 +1D5B2>0073 +1D5B3>0074 +1D5B4>0075 +1D5B5>0076 +1D5B6>0077 +1D5B7>0078 +1D5B8>0079 +1D5B9>007A +1D5BA>0061 +1D5BB>0062 +1D5BC>0063 +1D5BD>0064 +1D5BE>0065 +1D5BF>0066 +1D5C0>0067 +1D5C1>0068 +1D5C2>0069 +1D5C3>006A +1D5C4>006B +1D5C5>006C +1D5C6>006D +1D5C7>006E +1D5C8>006F +1D5C9>0070 +1D5CA>0071 +1D5CB>0072 +1D5CC>0073 +1D5CD>0074 +1D5CE>0075 +1D5CF>0076 +1D5D0>0077 +1D5D1>0078 +1D5D2>0079 +1D5D3>007A +1D5D4>0061 +1D5D5>0062 +1D5D6>0063 +1D5D7>0064 +1D5D8>0065 +1D5D9>0066 +1D5DA>0067 +1D5DB>0068 +1D5DC>0069 +1D5DD>006A +1D5DE>006B +1D5DF>006C +1D5E0>006D +1D5E1>006E +1D5E2>006F +1D5E3>0070 +1D5E4>0071 +1D5E5>0072 +1D5E6>0073 +1D5E7>0074 +1D5E8>0075 +1D5E9>0076 +1D5EA>0077 +1D5EB>0078 +1D5EC>0079 +1D5ED>007A +1D5EE>0061 +1D5EF>0062 +1D5F0>0063 +1D5F1>0064 +1D5F2>0065 +1D5F3>0066 +1D5F4>0067 +1D5F5>0068 +1D5F6>0069 +1D5F7>006A +1D5F8>006B +1D5F9>006C +1D5FA>006D +1D5FB>006E +1D5FC>006F +1D5FD>0070 +1D5FE>0071 +1D5FF>0072 +1D600>0073 +1D601>0074 +1D602>0075 +1D603>0076 +1D604>0077 +1D605>0078 +1D606>0079 +1D607>007A +1D608>0061 +1D609>0062 +1D60A>0063 +1D60B>0064 +1D60C>0065 +1D60D>0066 +1D60E>0067 +1D60F>0068 +1D610>0069 +1D611>006A +1D612>006B +1D613>006C +1D614>006D +1D615>006E +1D616>006F +1D617>0070 +1D618>0071 +1D619>0072 +1D61A>0073 +1D61B>0074 +1D61C>0075 +1D61D>0076 +1D61E>0077 +1D61F>0078 +1D620>0079 +1D621>007A +1D622>0061 +1D623>0062 +1D624>0063 +1D625>0064 +1D626>0065 +1D627>0066 +1D628>0067 +1D629>0068 +1D62A>0069 +1D62B>006A +1D62C>006B +1D62D>006C +1D62E>006D +1D62F>006E +1D630>006F +1D631>0070 +1D632>0071 +1D633>0072 +1D634>0073 +1D635>0074 +1D636>0075 +1D637>0076 +1D638>0077 +1D639>0078 +1D63A>0079 +1D63B>007A +1D63C>0061 +1D63D>0062 +1D63E>0063 +1D63F>0064 +1D640>0065 +1D641>0066 +1D642>0067 +1D643>0068 +1D644>0069 +1D645>006A +1D646>006B +1D647>006C +1D648>006D +1D649>006E +1D64A>006F +1D64B>0070 +1D64C>0071 +1D64D>0072 +1D64E>0073 +1D64F>0074 +1D650>0075 +1D651>0076 +1D652>0077 +1D653>0078 +1D654>0079 +1D655>007A +1D656>0061 +1D657>0062 +1D658>0063 +1D659>0064 +1D65A>0065 +1D65B>0066 +1D65C>0067 +1D65D>0068 +1D65E>0069 +1D65F>006A +1D660>006B +1D661>006C +1D662>006D +1D663>006E +1D664>006F +1D665>0070 +1D666>0071 +1D667>0072 +1D668>0073 +1D669>0074 +1D66A>0075 +1D66B>0076 +1D66C>0077 +1D66D>0078 +1D66E>0079 +1D66F>007A +1D670>0061 +1D671>0062 +1D672>0063 +1D673>0064 +1D674>0065 +1D675>0066 +1D676>0067 +1D677>0068 +1D678>0069 +1D679>006A +1D67A>006B +1D67B>006C +1D67C>006D +1D67D>006E +1D67E>006F +1D67F>0070 +1D680>0071 +1D681>0072 +1D682>0073 +1D683>0074 +1D684>0075 +1D685>0076 +1D686>0077 +1D687>0078 +1D688>0079 +1D689>007A +1D68A>0061 +1D68B>0062 +1D68C>0063 +1D68D>0064 +1D68E>0065 +1D68F>0066 +1D690>0067 +1D691>0068 +1D692>0069 +1D693>006A +1D694>006B +1D695>006C +1D696>006D +1D697>006E +1D698>006F +1D699>0070 +1D69A>0071 +1D69B>0072 +1D69C>0073 +1D69D>0074 +1D69E>0075 +1D69F>0076 +1D6A0>0077 +1D6A1>0078 +1D6A2>0079 +1D6A3>007A +1D6A4>0131 +1D6A5>0237 +1D6A8>03B1 +1D6A9>03B2 +1D6AA>03B3 +1D6AB>03B4 +1D6AC>03B5 +1D6AD>03B6 +1D6AE>03B7 +1D6AF>03B8 +1D6B0>03B9 +1D6B1>03BA +1D6B2>03BB +1D6B3>03BC +1D6B4>03BD +1D6B5>03BE +1D6B6>03BF +1D6B7>03C0 +1D6B8>03C1 +1D6B9>03B8 +1D6BA>03C3 +1D6BB>03C4 +1D6BC>03C5 +1D6BD>03C6 +1D6BE>03C7 +1D6BF>03C8 +1D6C0>03C9 +1D6C1>2207 +1D6C2>03B1 +1D6C3>03B2 +1D6C4>03B3 +1D6C5>03B4 +1D6C6>03B5 +1D6C7>03B6 +1D6C8>03B7 +1D6C9>03B8 +1D6CA>03B9 +1D6CB>03BA +1D6CC>03BB +1D6CD>03BC +1D6CE>03BD +1D6CF>03BE +1D6D0>03BF +1D6D1>03C0 +1D6D2>03C1 +1D6D3..1D6D4>03C3 +1D6D5>03C4 +1D6D6>03C5 +1D6D7>03C6 +1D6D8>03C7 +1D6D9>03C8 +1D6DA>03C9 +1D6DB>2202 +1D6DC>03B5 +1D6DD>03B8 +1D6DE>03BA +1D6DF>03C6 +1D6E0>03C1 +1D6E1>03C0 +1D6E2>03B1 +1D6E3>03B2 +1D6E4>03B3 +1D6E5>03B4 +1D6E6>03B5 +1D6E7>03B6 +1D6E8>03B7 +1D6E9>03B8 +1D6EA>03B9 +1D6EB>03BA +1D6EC>03BB +1D6ED>03BC +1D6EE>03BD +1D6EF>03BE +1D6F0>03BF +1D6F1>03C0 +1D6F2>03C1 +1D6F3>03B8 +1D6F4>03C3 +1D6F5>03C4 +1D6F6>03C5 +1D6F7>03C6 +1D6F8>03C7 +1D6F9>03C8 +1D6FA>03C9 +1D6FB>2207 +1D6FC>03B1 +1D6FD>03B2 +1D6FE>03B3 +1D6FF>03B4 +1D700>03B5 +1D701>03B6 +1D702>03B7 +1D703>03B8 +1D704>03B9 +1D705>03BA +1D706>03BB +1D707>03BC +1D708>03BD +1D709>03BE +1D70A>03BF +1D70B>03C0 +1D70C>03C1 +1D70D..1D70E>03C3 +1D70F>03C4 +1D710>03C5 +1D711>03C6 +1D712>03C7 +1D713>03C8 +1D714>03C9 +1D715>2202 +1D716>03B5 +1D717>03B8 +1D718>03BA +1D719>03C6 +1D71A>03C1 +1D71B>03C0 +1D71C>03B1 +1D71D>03B2 +1D71E>03B3 +1D71F>03B4 +1D720>03B5 +1D721>03B6 +1D722>03B7 +1D723>03B8 +1D724>03B9 +1D725>03BA +1D726>03BB +1D727>03BC +1D728>03BD +1D729>03BE +1D72A>03BF +1D72B>03C0 +1D72C>03C1 +1D72D>03B8 +1D72E>03C3 +1D72F>03C4 +1D730>03C5 +1D731>03C6 +1D732>03C7 +1D733>03C8 +1D734>03C9 +1D735>2207 +1D736>03B1 +1D737>03B2 +1D738>03B3 +1D739>03B4 +1D73A>03B5 +1D73B>03B6 +1D73C>03B7 +1D73D>03B8 +1D73E>03B9 +1D73F>03BA +1D740>03BB +1D741>03BC +1D742>03BD +1D743>03BE +1D744>03BF +1D745>03C0 +1D746>03C1 +1D747..1D748>03C3 +1D749>03C4 +1D74A>03C5 +1D74B>03C6 +1D74C>03C7 +1D74D>03C8 +1D74E>03C9 +1D74F>2202 +1D750>03B5 +1D751>03B8 +1D752>03BA +1D753>03C6 +1D754>03C1 +1D755>03C0 +1D756>03B1 +1D757>03B2 +1D758>03B3 +1D759>03B4 +1D75A>03B5 +1D75B>03B6 +1D75C>03B7 +1D75D>03B8 +1D75E>03B9 +1D75F>03BA +1D760>03BB +1D761>03BC +1D762>03BD +1D763>03BE +1D764>03BF +1D765>03C0 +1D766>03C1 +1D767>03B8 +1D768>03C3 +1D769>03C4 +1D76A>03C5 +1D76B>03C6 +1D76C>03C7 +1D76D>03C8 +1D76E>03C9 +1D76F>2207 +1D770>03B1 +1D771>03B2 +1D772>03B3 +1D773>03B4 +1D774>03B5 +1D775>03B6 +1D776>03B7 +1D777>03B8 +1D778>03B9 +1D779>03BA +1D77A>03BB +1D77B>03BC +1D77C>03BD +1D77D>03BE +1D77E>03BF +1D77F>03C0 +1D780>03C1 +1D781..1D782>03C3 +1D783>03C4 +1D784>03C5 +1D785>03C6 +1D786>03C7 +1D787>03C8 +1D788>03C9 +1D789>2202 +1D78A>03B5 +1D78B>03B8 +1D78C>03BA +1D78D>03C6 +1D78E>03C1 +1D78F>03C0 +1D790>03B1 +1D791>03B2 +1D792>03B3 +1D793>03B4 +1D794>03B5 +1D795>03B6 +1D796>03B7 +1D797>03B8 +1D798>03B9 +1D799>03BA +1D79A>03BB +1D79B>03BC +1D79C>03BD +1D79D>03BE +1D79E>03BF +1D79F>03C0 +1D7A0>03C1 +1D7A1>03B8 +1D7A2>03C3 +1D7A3>03C4 +1D7A4>03C5 +1D7A5>03C6 +1D7A6>03C7 +1D7A7>03C8 +1D7A8>03C9 +1D7A9>2207 +1D7AA>03B1 +1D7AB>03B2 +1D7AC>03B3 +1D7AD>03B4 +1D7AE>03B5 +1D7AF>03B6 +1D7B0>03B7 +1D7B1>03B8 +1D7B2>03B9 +1D7B3>03BA +1D7B4>03BB +1D7B5>03BC +1D7B6>03BD +1D7B7>03BE +1D7B8>03BF +1D7B9>03C0 +1D7BA>03C1 +1D7BB..1D7BC>03C3 +1D7BD>03C4 +1D7BE>03C5 +1D7BF>03C6 +1D7C0>03C7 +1D7C1>03C8 +1D7C2>03C9 +1D7C3>2202 +1D7C4>03B5 +1D7C5>03B8 +1D7C6>03BA +1D7C7>03C6 +1D7C8>03C1 +1D7C9>03C0 +1D7CA..1D7CB>03DD +1D7CE>0030 +1D7CF>0031 +1D7D0>0032 +1D7D1>0033 +1D7D2>0034 +1D7D3>0035 +1D7D4>0036 +1D7D5>0037 +1D7D6>0038 +1D7D7>0039 +1D7D8>0030 +1D7D9>0031 +1D7DA>0032 +1D7DB>0033 +1D7DC>0034 +1D7DD>0035 +1D7DE>0036 +1D7DF>0037 +1D7E0>0038 +1D7E1>0039 +1D7E2>0030 +1D7E3>0031 +1D7E4>0032 +1D7E5>0033 +1D7E6>0034 +1D7E7>0035 +1D7E8>0036 +1D7E9>0037 +1D7EA>0038 +1D7EB>0039 +1D7EC>0030 +1D7ED>0031 +1D7EE>0032 +1D7EF>0033 +1D7F0>0034 +1D7F1>0035 +1D7F2>0036 +1D7F3>0037 +1D7F4>0038 +1D7F5>0039 +1D7F6>0030 +1D7F7>0031 +1D7F8>0032 +1D7F9>0033 +1D7FA>0034 +1D7FB>0035 +1D7FC>0036 +1D7FD>0037 +1D7FE>0038 +1D7FF>0039 +1F100>0030 002E +1F101>0030 002C +1F102>0031 002C +1F103>0032 002C +1F104>0033 002C +1F105>0034 002C +1F106>0035 002C +1F107>0036 002C +1F108>0037 002C +1F109>0038 002C +1F10A>0039 002C +1F110>0028 0061 0029 +1F111>0028 0062 0029 +1F112>0028 0063 0029 +1F113>0028 0064 0029 +1F114>0028 0065 0029 +1F115>0028 0066 0029 +1F116>0028 0067 0029 +1F117>0028 0068 0029 +1F118>0028 0069 0029 +1F119>0028 006A 0029 +1F11A>0028 006B 0029 +1F11B>0028 006C 0029 +1F11C>0028 006D 0029 +1F11D>0028 006E 0029 +1F11E>0028 006F 0029 +1F11F>0028 0070 0029 +1F120>0028 0071 0029 +1F121>0028 0072 0029 +1F122>0028 0073 0029 +1F123>0028 0074 0029 +1F124>0028 0075 0029 +1F125>0028 0076 0029 +1F126>0028 0077 0029 +1F127>0028 0078 0029 +1F128>0028 0079 0029 +1F129>0028 007A 0029 +1F12A>3014 0073 3015 +1F12B>0063 +1F12C>0072 +1F12D>0063 0064 +1F12E>0077 007A +1F131>0062 +1F13D>006E +1F13F>0070 +1F142>0073 +1F146>0077 +1F14A>0068 0076 +1F14B>006D 0076 +1F14C>0073 0064 +1F14D>0073 0073 +1F14E>0070 0070 0076 +1F190>0064 006A +1F200>307B 304B +1F210>624B +1F211>5B57 +1F212>53CC +1F213>30C7 +1F214>4E8C +1F215>591A +1F216>89E3 +1F217>5929 +1F218>4EA4 +1F219>6620 +1F21A>7121 +1F21B>6599 +1F21C>524D +1F21D>5F8C +1F21E>518D +1F21F>65B0 +1F220>521D +1F221>7D42 +1F222>751F +1F223>8CA9 +1F224>58F0 +1F225>5439 +1F226>6F14 +1F227>6295 +1F228>6355 +1F229>4E00 +1F22A>4E09 +1F22B>904A +1F22C>5DE6 +1F22D>4E2D +1F22E>53F3 +1F22F>6307 +1F230>8D70 +1F231>6253 +1F240>3014 672C 3015 +1F241>3014 4E09 3015 +1F242>3014 4E8C 3015 +1F243>3014 5B89 3015 +1F244>3014 70B9 3015 +1F245>3014 6253 3015 +1F246>3014 76D7 3015 +1F247>3014 52DD 3015 +1F248>3014 6557 3015 +2F800>4E3D +2F801>4E38 +2F802>4E41 +2F803>20122 +2F804>4F60 +2F805>4FAE +2F806>4FBB +2F807>5002 +2F808>507A +2F809>5099 +2F80A>50E7 +2F80B>50CF +2F80C>349E +2F80D>2063A +2F80E>514D +2F80F>5154 +2F810>5164 +2F811>5177 +2F812>2051C +2F813>34B9 +2F814>5167 +2F815>518D +2F816>2054B +2F817>5197 +2F818>51A4 +2F819>4ECC +2F81A>51AC +2F81B>51B5 +2F81C>291DF +2F81D>51F5 +2F81E>5203 +2F81F>34DF +2F820>523B +2F821>5246 +2F822>5272 +2F823>5277 +2F824>3515 +2F825>52C7 +2F826>52C9 +2F827>52E4 +2F828>52FA +2F829>5305 +2F82A>5306 +2F82B>5317 +2F82C>5349 +2F82D>5351 +2F82E>535A +2F82F>5373 +2F830>537D +2F831..2F833>537F +2F834>20A2C +2F835>7070 +2F836>53CA +2F837>53DF +2F838>20B63 +2F839>53EB +2F83A>53F1 +2F83B>5406 +2F83C>549E +2F83D>5438 +2F83E>5448 +2F83F>5468 +2F840>54A2 +2F841>54F6 +2F842>5510 +2F843>5553 +2F844>5563 +2F845..2F846>5584 +2F847>5599 +2F848>55AB +2F849>55B3 +2F84A>55C2 +2F84B>5716 +2F84C>5606 +2F84D>5717 +2F84E>5651 +2F84F>5674 +2F850>5207 +2F851>58EE +2F852>57CE +2F853>57F4 +2F854>580D +2F855>578B +2F856>5832 +2F857>5831 +2F858>58AC +2F859>214E4 +2F85A>58F2 +2F85B>58F7 +2F85C>5906 +2F85D>591A +2F85E>5922 +2F85F>5962 +2F860>216A8 +2F861>216EA +2F862>59EC +2F863>5A1B +2F864>5A27 +2F865>59D8 +2F866>5A66 +2F867>36EE +2F868>36FC +2F869>5B08 +2F86A..2F86B>5B3E +2F86C>219C8 +2F86D>5BC3 +2F86E>5BD8 +2F86F>5BE7 +2F870>5BF3 +2F871>21B18 +2F872>5BFF +2F873>5C06 +2F874>5F53 +2F875>5C22 +2F876>3781 +2F877>5C60 +2F878>5C6E +2F879>5CC0 +2F87A>5C8D +2F87B>21DE4 +2F87C>5D43 +2F87D>21DE6 +2F87E>5D6E +2F87F>5D6B +2F880>5D7C +2F881>5DE1 +2F882>5DE2 +2F883>382F +2F884>5DFD +2F885>5E28 +2F886>5E3D +2F887>5E69 +2F888>3862 +2F889>22183 +2F88A>387C +2F88B>5EB0 +2F88C>5EB3 +2F88D>5EB6 +2F88E>5ECA +2F88F>2A392 +2F890>5EFE +2F891..2F892>22331 +2F893>8201 +2F894..2F895>5F22 +2F896>38C7 +2F897>232B8 +2F898>261DA +2F899>5F62 +2F89A>5F6B +2F89B>38E3 +2F89C>5F9A +2F89D>5FCD +2F89E>5FD7 +2F89F>5FF9 +2F8A0>6081 +2F8A1>393A +2F8A2>391C +2F8A3>6094 +2F8A4>226D4 +2F8A5>60C7 +2F8A6>6148 +2F8A7>614C +2F8A8>614E +2F8A9>614C +2F8AA>617A +2F8AB>618E +2F8AC>61B2 +2F8AD>61A4 +2F8AE>61AF +2F8AF>61DE +2F8B0>61F2 +2F8B1>61F6 +2F8B2>6210 +2F8B3>621B +2F8B4>625D +2F8B5>62B1 +2F8B6>62D4 +2F8B7>6350 +2F8B8>22B0C +2F8B9>633D +2F8BA>62FC +2F8BB>6368 +2F8BC>6383 +2F8BD>63E4 +2F8BE>22BF1 +2F8BF>6422 +2F8C0>63C5 +2F8C1>63A9 +2F8C2>3A2E +2F8C3>6469 +2F8C4>647E +2F8C5>649D +2F8C6>6477 +2F8C7>3A6C +2F8C8>654F +2F8C9>656C +2F8CA>2300A +2F8CB>65E3 +2F8CC>66F8 +2F8CD>6649 +2F8CE>3B19 +2F8CF>6691 +2F8D0>3B08 +2F8D1>3AE4 +2F8D2>5192 +2F8D3>5195 +2F8D4>6700 +2F8D5>669C +2F8D6>80AD +2F8D7>43D9 +2F8D8>6717 +2F8D9>671B +2F8DA>6721 +2F8DB>675E +2F8DC>6753 +2F8DD>233C3 +2F8DE>3B49 +2F8DF>67FA +2F8E0>6785 +2F8E1>6852 +2F8E2>6885 +2F8E3>2346D +2F8E4>688E +2F8E5>681F +2F8E6>6914 +2F8E7>3B9D +2F8E8>6942 +2F8E9>69A3 +2F8EA>69EA +2F8EB>6AA8 +2F8EC>236A3 +2F8ED>6ADB +2F8EE>3C18 +2F8EF>6B21 +2F8F0>238A7 +2F8F1>6B54 +2F8F2>3C4E +2F8F3>6B72 +2F8F4>6B9F +2F8F5>6BBA +2F8F6>6BBB +2F8F7>23A8D +2F8F8>21D0B +2F8F9>23AFA +2F8FA>6C4E +2F8FB>23CBC +2F8FC>6CBF +2F8FD>6CCD +2F8FE>6C67 +2F8FF>6D16 +2F900>6D3E +2F901>6D77 +2F902>6D41 +2F903>6D69 +2F904>6D78 +2F905>6D85 +2F906>23D1E +2F907>6D34 +2F908>6E2F +2F909>6E6E +2F90A>3D33 +2F90B>6ECB +2F90C>6EC7 +2F90D>23ED1 +2F90E>6DF9 +2F90F>6F6E +2F910>23F5E +2F911>23F8E +2F912>6FC6 +2F913>7039 +2F914>701E +2F915>701B +2F916>3D96 +2F917>704A +2F918>707D +2F919>7077 +2F91A>70AD +2F91B>20525 +2F91C>7145 +2F91D>24263 +2F91E>719C +2F91F>243AB +2F920>7228 +2F921>7235 +2F922>7250 +2F923>24608 +2F924>7280 +2F925>7295 +2F926>24735 +2F927>24814 +2F928>737A +2F929>738B +2F92A>3EAC +2F92B>73A5 +2F92C..2F92D>3EB8 +2F92E>7447 +2F92F>745C +2F930>7471 +2F931>7485 +2F932>74CA +2F933>3F1B +2F934>7524 +2F935>24C36 +2F936>753E +2F937>24C92 +2F938>7570 +2F939>2219F +2F93A>7610 +2F93B>24FA1 +2F93C>24FB8 +2F93D>25044 +2F93E>3FFC +2F93F>4008 +2F940>76F4 +2F941>250F3 +2F942>250F2 +2F943>25119 +2F944>25133 +2F945>771E +2F946..2F947>771F +2F948>774A +2F949>4039 +2F94A>778B +2F94B>4046 +2F94C>4096 +2F94D>2541D +2F94E>784E +2F94F>788C +2F950>78CC +2F951>40E3 +2F952>25626 +2F953>7956 +2F954>2569A +2F955>256C5 +2F956>798F +2F957>79EB +2F958>412F +2F959>7A40 +2F95A>7A4A +2F95B>7A4F +2F95C>2597C +2F95D..2F95E>25AA7 +2F95F>7AEE +2F960>4202 +2F961>25BAB +2F962>7BC6 +2F963>7BC9 +2F964>4227 +2F965>25C80 +2F966>7CD2 +2F967>42A0 +2F968>7CE8 +2F969>7CE3 +2F96A>7D00 +2F96B>25F86 +2F96C>7D63 +2F96D>4301 +2F96E>7DC7 +2F96F>7E02 +2F970>7E45 +2F971>4334 +2F972>26228 +2F973>26247 +2F974>4359 +2F975>262D9 +2F976>7F7A +2F977>2633E +2F978>7F95 +2F979>7FFA +2F97A>8005 +2F97B>264DA +2F97C>26523 +2F97D>8060 +2F97E>265A8 +2F97F>8070 +2F980>2335F +2F981>43D5 +2F982>80B2 +2F983>8103 +2F984>440B +2F985>813E +2F986>5AB5 +2F987>267A7 +2F988>267B5 +2F989>23393 +2F98A>2339C +2F98B>8201 +2F98C>8204 +2F98D>8F9E +2F98E>446B +2F98F>8291 +2F990>828B +2F991>829D +2F992>52B3 +2F993>82B1 +2F994>82B3 +2F995>82BD +2F996>82E6 +2F997>26B3C +2F998>82E5 +2F999>831D +2F99A>8363 +2F99B>83AD +2F99C>8323 +2F99D>83BD +2F99E>83E7 +2F99F>8457 +2F9A0>8353 +2F9A1>83CA +2F9A2>83CC +2F9A3>83DC +2F9A4>26C36 +2F9A5>26D6B +2F9A6>26CD5 +2F9A7>452B +2F9A8>84F1 +2F9A9>84F3 +2F9AA>8516 +2F9AB>273CA +2F9AC>8564 +2F9AD>26F2C +2F9AE>455D +2F9AF>4561 +2F9B0>26FB1 +2F9B1>270D2 +2F9B2>456B +2F9B3>8650 +2F9B4>865C +2F9B5>8667 +2F9B6>8669 +2F9B7>86A9 +2F9B8>8688 +2F9B9>870E +2F9BA>86E2 +2F9BB>8779 +2F9BC>8728 +2F9BD>876B +2F9BE>8786 +2F9BF>45D7 +2F9C0>87E1 +2F9C1>8801 +2F9C2>45F9 +2F9C3>8860 +2F9C4>8863 +2F9C5>27667 +2F9C6>88D7 +2F9C7>88DE +2F9C8>4635 +2F9C9>88FA +2F9CA>34BB +2F9CB>278AE +2F9CC>27966 +2F9CD>46BE +2F9CE>46C7 +2F9CF>8AA0 +2F9D0>8AED +2F9D1>8B8A +2F9D2>8C55 +2F9D3>27CA8 +2F9D4>8CAB +2F9D5>8CC1 +2F9D6>8D1B +2F9D7>8D77 +2F9D8>27F2F +2F9D9>20804 +2F9DA>8DCB +2F9DB>8DBC +2F9DC>8DF0 +2F9DD>208DE +2F9DE>8ED4 +2F9DF>8F38 +2F9E0>285D2 +2F9E1>285ED +2F9E2>9094 +2F9E3>90F1 +2F9E4>9111 +2F9E5>2872E +2F9E6>911B +2F9E7>9238 +2F9E8>92D7 +2F9E9>92D8 +2F9EA>927C +2F9EB>93F9 +2F9EC>9415 +2F9ED>28BFA +2F9EE>958B +2F9EF>4995 +2F9F0>95B7 +2F9F1>28D77 +2F9F2>49E6 +2F9F3>96C3 +2F9F4>5DB2 +2F9F5>9723 +2F9F6>29145 +2F9F7>2921A +2F9F8>4A6E +2F9F9>4A76 +2F9FA>97E0 +2F9FB>2940A +2F9FC>4AB2 +2F9FD>29496 +2F9FE..2F9FF>980B +2FA00>9829 +2FA01>295B6 +2FA02>98E2 +2FA03>4B33 +2FA04>9929 +2FA05>99A7 +2FA06>99C2 +2FA07>99FE +2FA08>4BCE +2FA09>29B30 +2FA0A>9B12 +2FA0B>9C40 +2FA0C>9CFD +2FA0D>4CCE +2FA0E>4CED +2FA0F>9D67 +2FA10>2A0CE +2FA11>4CF8 +2FA12>2A105 +2FA13>2A20E +2FA14>2A291 +2FA15>9EBB +2FA16>4D56 +2FA17>9EF9 +2FA18>9EFE +2FA19>9F05 +2FA1A>9F0F +2FA1B>9F16 +2FA1C>9F3B +2FA1D>2A600 +E0000> +E0001> +E0002..E001F> +E0020..E007F> +E0080..E00FF> +E0100..E01EF> +E01F0..E0FFF> + +# Total code points: 9740 diff --git a/mailnews/extensions/fts3/public/moz.build b/mailnews/extensions/fts3/public/moz.build new file mode 100644 index 000000000..9d38f0177 --- /dev/null +++ b/mailnews/extensions/fts3/public/moz.build @@ -0,0 +1,11 @@ +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +XPIDL_SOURCES += [ + 'nsIFts3Tokenizer.idl', +] + +XPIDL_MODULE = 'fts3tok' + diff --git a/mailnews/extensions/fts3/public/nsIFts3Tokenizer.idl b/mailnews/extensions/fts3/public/nsIFts3Tokenizer.idl new file mode 100644 index 000000000..c2bb7d435 --- /dev/null +++ b/mailnews/extensions/fts3/public/nsIFts3Tokenizer.idl @@ -0,0 +1,15 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" + +interface mozIStorageConnection; + +[scriptable, uuid(136c88ea-7003-4fe8-8835-333fd18e598c)] +interface nsIFts3Tokenizer : nsISupports { + // register FTS3 tokenizer module for "mozporter" tokenizer + // mozporter is based by porter tokenizer with bi-gram tokenizer for CJK + void registerTokenizer(in mozIStorageConnection connection); +}; diff --git a/mailnews/extensions/fts3/src/Normalize.c b/mailnews/extensions/fts3/src/Normalize.c new file mode 100644 index 000000000..92ac400e2 --- /dev/null +++ b/mailnews/extensions/fts3/src/Normalize.c @@ -0,0 +1,1929 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* THIS FILE IS GENERATED BY generate_table.py. DON'T EDIT THIS */ + +static const unsigned short gNormalizeTable0040[] = { + /* U+0040 */ + 0x0040, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + }; + +static const unsigned short gNormalizeTable0080[] = { + /* U+0080 */ + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x0020, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, + 0x0020, 0x00a9, 0x0061, 0x00ab, 0x00ac, 0x0020, 0x00ae, 0x0020, + 0x00b0, 0x00b1, 0x0032, 0x0033, 0x0020, 0x03bc, 0x00b6, 0x00b7, + 0x0020, 0x0031, 0x006f, 0x00bb, 0x0031, 0x0031, 0x0033, 0x00bf, + }; + +static const unsigned short gNormalizeTable00c0[] = { + /* U+00c0 */ + 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x00e6, 0x0063, + 0x0065, 0x0065, 0x0065, 0x0065, 0x0069, 0x0069, 0x0069, 0x0069, + 0x00f0, 0x006e, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x00d7, + 0x00f8, 0x0075, 0x0075, 0x0075, 0x0075, 0x0079, 0x00fe, 0x0073, + 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x00e6, 0x0063, + 0x0065, 0x0065, 0x0065, 0x0065, 0x0069, 0x0069, 0x0069, 0x0069, + 0x00f0, 0x006e, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x00f7, + 0x00f8, 0x0075, 0x0075, 0x0075, 0x0075, 0x0079, 0x00fe, 0x0079, + }; + +static const unsigned short gNormalizeTable0100[] = { + /* U+0100 */ + 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0063, 0x0063, + 0x0063, 0x0063, 0x0063, 0x0063, 0x0063, 0x0063, 0x0064, 0x0064, + 0x0111, 0x0111, 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, + 0x0065, 0x0065, 0x0065, 0x0065, 0x0067, 0x0067, 0x0067, 0x0067, + 0x0067, 0x0067, 0x0067, 0x0067, 0x0068, 0x0068, 0x0127, 0x0127, + 0x0069, 0x0069, 0x0069, 0x0069, 0x0069, 0x0069, 0x0069, 0x0069, + 0x0069, 0x0131, 0x0069, 0x0069, 0x006a, 0x006a, 0x006b, 0x006b, + 0x0138, 0x006c, 0x006c, 0x006c, 0x006c, 0x006c, 0x006c, 0x006c, + }; + +static const unsigned short gNormalizeTable0140[] = { + /* U+0140 */ + 0x006c, 0x0142, 0x0142, 0x006e, 0x006e, 0x006e, 0x006e, 0x006e, + 0x006e, 0x02bc, 0x014b, 0x014b, 0x006f, 0x006f, 0x006f, 0x006f, + 0x006f, 0x006f, 0x0153, 0x0153, 0x0072, 0x0072, 0x0072, 0x0072, + 0x0072, 0x0072, 0x0073, 0x0073, 0x0073, 0x0073, 0x0073, 0x0073, + 0x0073, 0x0073, 0x0074, 0x0074, 0x0074, 0x0074, 0x0167, 0x0167, + 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, + 0x0075, 0x0075, 0x0075, 0x0075, 0x0077, 0x0077, 0x0079, 0x0079, + 0x0079, 0x007a, 0x007a, 0x007a, 0x007a, 0x007a, 0x007a, 0x0073, + }; + +static const unsigned short gNormalizeTable0180[] = { + /* U+0180 */ + 0x0180, 0x0253, 0x0183, 0x0183, 0x0185, 0x0185, 0x0254, 0x0188, + 0x0188, 0x0256, 0x0257, 0x018c, 0x018c, 0x018d, 0x01dd, 0x0259, + 0x025b, 0x0192, 0x0192, 0x0260, 0x0263, 0x0195, 0x0269, 0x0268, + 0x0199, 0x0199, 0x019a, 0x019b, 0x026f, 0x0272, 0x019e, 0x0275, + 0x006f, 0x006f, 0x01a3, 0x01a3, 0x01a5, 0x01a5, 0x0280, 0x01a8, + 0x01a8, 0x0283, 0x01aa, 0x01ab, 0x01ad, 0x01ad, 0x0288, 0x0075, + 0x0075, 0x028a, 0x028b, 0x01b4, 0x01b4, 0x01b6, 0x01b6, 0x0292, + 0x01b9, 0x01b9, 0x01ba, 0x01bb, 0x01bd, 0x01bd, 0x01be, 0x01bf, + }; + +static const unsigned short gNormalizeTable01c0[] = { + /* U+01c0 */ + 0x01c0, 0x01c1, 0x01c2, 0x01c3, 0x0064, 0x0064, 0x0064, 0x006c, + 0x006c, 0x006c, 0x006e, 0x006e, 0x006e, 0x0061, 0x0061, 0x0069, + 0x0069, 0x006f, 0x006f, 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, + 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, 0x01dd, 0x0061, 0x0061, + 0x0061, 0x0061, 0x00e6, 0x00e6, 0x01e5, 0x01e5, 0x0067, 0x0067, + 0x006b, 0x006b, 0x006f, 0x006f, 0x006f, 0x006f, 0x0292, 0x0292, + 0x006a, 0x0064, 0x0064, 0x0064, 0x0067, 0x0067, 0x0195, 0x01bf, + 0x006e, 0x006e, 0x0061, 0x0061, 0x00e6, 0x00e6, 0x00f8, 0x00f8, + }; + +static const unsigned short gNormalizeTable0200[] = { + /* U+0200 */ + 0x0061, 0x0061, 0x0061, 0x0061, 0x0065, 0x0065, 0x0065, 0x0065, + 0x0069, 0x0069, 0x0069, 0x0069, 0x006f, 0x006f, 0x006f, 0x006f, + 0x0072, 0x0072, 0x0072, 0x0072, 0x0075, 0x0075, 0x0075, 0x0075, + 0x0073, 0x0073, 0x0074, 0x0074, 0x021d, 0x021d, 0x0068, 0x0068, + 0x019e, 0x0221, 0x0223, 0x0223, 0x0225, 0x0225, 0x0061, 0x0061, + 0x0065, 0x0065, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, + 0x006f, 0x006f, 0x0079, 0x0079, 0x0234, 0x0235, 0x0236, 0x0237, + 0x0238, 0x0239, 0x2c65, 0x023c, 0x023c, 0x019a, 0x2c66, 0x023f, + }; + +static const unsigned short gNormalizeTable0240[] = { + /* U+0240 */ + 0x0240, 0x0242, 0x0242, 0x0180, 0x0289, 0x028c, 0x0247, 0x0247, + 0x0249, 0x0249, 0x024b, 0x024b, 0x024d, 0x024d, 0x024f, 0x024f, + 0x0250, 0x0251, 0x0252, 0x0253, 0x0254, 0x0255, 0x0256, 0x0257, + 0x0258, 0x0259, 0x025a, 0x025b, 0x025c, 0x025d, 0x025e, 0x025f, + 0x0260, 0x0261, 0x0262, 0x0263, 0x0264, 0x0265, 0x0266, 0x0267, + 0x0268, 0x0269, 0x026a, 0x026b, 0x026c, 0x026d, 0x026e, 0x026f, + 0x0270, 0x0271, 0x0272, 0x0273, 0x0274, 0x0275, 0x0276, 0x0277, + 0x0278, 0x0279, 0x027a, 0x027b, 0x027c, 0x027d, 0x027e, 0x027f, + }; + +static const unsigned short gNormalizeTable0280[] = { + /* U+0280 */ + 0x0280, 0x0281, 0x0282, 0x0283, 0x0284, 0x0285, 0x0286, 0x0287, + 0x0288, 0x0289, 0x028a, 0x028b, 0x028c, 0x028d, 0x028e, 0x028f, + 0x0290, 0x0291, 0x0292, 0x0293, 0x0294, 0x0295, 0x0296, 0x0297, + 0x0298, 0x0299, 0x029a, 0x029b, 0x029c, 0x029d, 0x029e, 0x029f, + 0x02a0, 0x02a1, 0x02a2, 0x02a3, 0x02a4, 0x02a5, 0x02a6, 0x02a7, + 0x02a8, 0x02a9, 0x02aa, 0x02ab, 0x02ac, 0x02ad, 0x02ae, 0x02af, + 0x0068, 0x0266, 0x006a, 0x0072, 0x0279, 0x027b, 0x0281, 0x0077, + 0x0079, 0x02b9, 0x02ba, 0x02bb, 0x02bc, 0x02bd, 0x02be, 0x02bf, + }; + +static const unsigned short gNormalizeTable02c0[] = { + /* U+02c0 */ + 0x02c0, 0x02c1, 0x02c2, 0x02c3, 0x02c4, 0x02c5, 0x02c6, 0x02c7, + 0x02c8, 0x02c9, 0x02ca, 0x02cb, 0x02cc, 0x02cd, 0x02ce, 0x02cf, + 0x02d0, 0x02d1, 0x02d2, 0x02d3, 0x02d4, 0x02d5, 0x02d6, 0x02d7, + 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x02de, 0x02df, + 0x0263, 0x006c, 0x0073, 0x0078, 0x0295, 0x02e5, 0x02e6, 0x02e7, + 0x02e8, 0x02e9, 0x02ea, 0x02eb, 0x02ec, 0x02ed, 0x02ee, 0x02ef, + 0x02f0, 0x02f1, 0x02f2, 0x02f3, 0x02f4, 0x02f5, 0x02f6, 0x02f7, + 0x02f8, 0x02f9, 0x02fa, 0x02fb, 0x02fc, 0x02fd, 0x02fe, 0x02ff, + }; + +static const unsigned short gNormalizeTable0340[] = { + /* U+0340 */ + 0x0300, 0x0301, 0x0342, 0x0313, 0x0308, 0x03b9, 0x0346, 0x0347, + 0x0348, 0x0349, 0x034a, 0x034b, 0x034c, 0x034d, 0x034e, 0x0020, + 0x0350, 0x0351, 0x0352, 0x0353, 0x0354, 0x0355, 0x0356, 0x0357, + 0x0358, 0x0359, 0x035a, 0x035b, 0x035c, 0x035d, 0x035e, 0x035f, + 0x0360, 0x0361, 0x0362, 0x0363, 0x0364, 0x0365, 0x0366, 0x0367, + 0x0368, 0x0369, 0x036a, 0x036b, 0x036c, 0x036d, 0x036e, 0x036f, + 0x0371, 0x0371, 0x0373, 0x0373, 0x02b9, 0x0375, 0x0377, 0x0377, + 0x0378, 0x0379, 0x0020, 0x037b, 0x037c, 0x037d, 0x003b, 0x037f, + }; + +static const unsigned short gNormalizeTable0380[] = { + /* U+0380 */ + 0x0380, 0x0381, 0x0382, 0x0383, 0x0020, 0x0020, 0x03b1, 0x00b7, + 0x03b5, 0x03b7, 0x03b9, 0x038b, 0x03bf, 0x038d, 0x03c5, 0x03c9, + 0x03b9, 0x03b1, 0x03b2, 0x03b3, 0x03b4, 0x03b5, 0x03b6, 0x03b7, + 0x03b8, 0x03b9, 0x03ba, 0x03bb, 0x03bc, 0x03bd, 0x03be, 0x03bf, + 0x03c0, 0x03c1, 0x03a2, 0x03c3, 0x03c4, 0x03c5, 0x03c6, 0x03c7, + 0x03c8, 0x03c9, 0x03b9, 0x03c5, 0x03b1, 0x03b5, 0x03b7, 0x03b9, + 0x03c5, 0x03b1, 0x03b2, 0x03b3, 0x03b4, 0x03b5, 0x03b6, 0x03b7, + 0x03b8, 0x03b9, 0x03ba, 0x03bb, 0x03bc, 0x03bd, 0x03be, 0x03bf, + }; + +static const unsigned short gNormalizeTable03c0[] = { + /* U+03c0 */ + 0x03c0, 0x03c1, 0x03c3, 0x03c3, 0x03c4, 0x03c5, 0x03c6, 0x03c7, + 0x03c8, 0x03c9, 0x03b9, 0x03c5, 0x03bf, 0x03c5, 0x03c9, 0x03d7, + 0x03b2, 0x03b8, 0x03c5, 0x03c5, 0x03c5, 0x03c6, 0x03c0, 0x03d7, + 0x03d9, 0x03d9, 0x03db, 0x03db, 0x03dd, 0x03dd, 0x03df, 0x03df, + 0x03e1, 0x03e1, 0x03e3, 0x03e3, 0x03e5, 0x03e5, 0x03e7, 0x03e7, + 0x03e9, 0x03e9, 0x03eb, 0x03eb, 0x03ed, 0x03ed, 0x03ef, 0x03ef, + 0x03ba, 0x03c1, 0x03c3, 0x03f3, 0x03b8, 0x03b5, 0x03f6, 0x03f8, + 0x03f8, 0x03c3, 0x03fb, 0x03fb, 0x03fc, 0x037b, 0x037c, 0x037d, + }; + +static const unsigned short gNormalizeTable0400[] = { + /* U+0400 */ + 0x0435, 0x0435, 0x0452, 0x0433, 0x0454, 0x0455, 0x0456, 0x0456, + 0x0458, 0x0459, 0x045a, 0x045b, 0x043a, 0x0438, 0x0443, 0x045f, + 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, + 0x0438, 0x0438, 0x043a, 0x043b, 0x043c, 0x043d, 0x043e, 0x043f, + 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, + 0x0448, 0x0449, 0x044a, 0x044b, 0x044c, 0x044d, 0x044e, 0x044f, + 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, + 0x0438, 0x0438, 0x043a, 0x043b, 0x043c, 0x043d, 0x043e, 0x043f, + }; + +static const unsigned short gNormalizeTable0440[] = { + /* U+0440 */ + 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, + 0x0448, 0x0449, 0x044a, 0x044b, 0x044c, 0x044d, 0x044e, 0x044f, + 0x0435, 0x0435, 0x0452, 0x0433, 0x0454, 0x0455, 0x0456, 0x0456, + 0x0458, 0x0459, 0x045a, 0x045b, 0x043a, 0x0438, 0x0443, 0x045f, + 0x0461, 0x0461, 0x0463, 0x0463, 0x0465, 0x0465, 0x0467, 0x0467, + 0x0469, 0x0469, 0x046b, 0x046b, 0x046d, 0x046d, 0x046f, 0x046f, + 0x0471, 0x0471, 0x0473, 0x0473, 0x0475, 0x0475, 0x0475, 0x0475, + 0x0479, 0x0479, 0x047b, 0x047b, 0x047d, 0x047d, 0x047f, 0x047f, + }; + +static const unsigned short gNormalizeTable0480[] = { + /* U+0480 */ + 0x0481, 0x0481, 0x0482, 0x0483, 0x0484, 0x0485, 0x0486, 0x0487, + 0x0488, 0x0489, 0x048b, 0x048b, 0x048d, 0x048d, 0x048f, 0x048f, + 0x0491, 0x0491, 0x0493, 0x0493, 0x0495, 0x0495, 0x0497, 0x0497, + 0x0499, 0x0499, 0x049b, 0x049b, 0x049d, 0x049d, 0x049f, 0x049f, + 0x04a1, 0x04a1, 0x04a3, 0x04a3, 0x04a5, 0x04a5, 0x04a7, 0x04a7, + 0x04a9, 0x04a9, 0x04ab, 0x04ab, 0x04ad, 0x04ad, 0x04af, 0x04af, + 0x04b1, 0x04b1, 0x04b3, 0x04b3, 0x04b5, 0x04b5, 0x04b7, 0x04b7, + 0x04b9, 0x04b9, 0x04bb, 0x04bb, 0x04bd, 0x04bd, 0x04bf, 0x04bf, + }; + +static const unsigned short gNormalizeTable04c0[] = { + /* U+04c0 */ + 0x04cf, 0x0436, 0x0436, 0x04c4, 0x04c4, 0x04c6, 0x04c6, 0x04c8, + 0x04c8, 0x04ca, 0x04ca, 0x04cc, 0x04cc, 0x04ce, 0x04ce, 0x04cf, + 0x0430, 0x0430, 0x0430, 0x0430, 0x04d5, 0x04d5, 0x0435, 0x0435, + 0x04d9, 0x04d9, 0x04d9, 0x04d9, 0x0436, 0x0436, 0x0437, 0x0437, + 0x04e1, 0x04e1, 0x0438, 0x0438, 0x0438, 0x0438, 0x043e, 0x043e, + 0x04e9, 0x04e9, 0x04e9, 0x04e9, 0x044d, 0x044d, 0x0443, 0x0443, + 0x0443, 0x0443, 0x0443, 0x0443, 0x0447, 0x0447, 0x04f7, 0x04f7, + 0x044b, 0x044b, 0x04fb, 0x04fb, 0x04fd, 0x04fd, 0x04ff, 0x04ff, + }; + +static const unsigned short gNormalizeTable0500[] = { + /* U+0500 */ + 0x0501, 0x0501, 0x0503, 0x0503, 0x0505, 0x0505, 0x0507, 0x0507, + 0x0509, 0x0509, 0x050b, 0x050b, 0x050d, 0x050d, 0x050f, 0x050f, + 0x0511, 0x0511, 0x0513, 0x0513, 0x0515, 0x0515, 0x0517, 0x0517, + 0x0519, 0x0519, 0x051b, 0x051b, 0x051d, 0x051d, 0x051f, 0x051f, + 0x0521, 0x0521, 0x0523, 0x0523, 0x0525, 0x0525, 0x0526, 0x0527, + 0x0528, 0x0529, 0x052a, 0x052b, 0x052c, 0x052d, 0x052e, 0x052f, + 0x0530, 0x0561, 0x0562, 0x0563, 0x0564, 0x0565, 0x0566, 0x0567, + 0x0568, 0x0569, 0x056a, 0x056b, 0x056c, 0x056d, 0x056e, 0x056f, + }; + +static const unsigned short gNormalizeTable0540[] = { + /* U+0540 */ + 0x0570, 0x0571, 0x0572, 0x0573, 0x0574, 0x0575, 0x0576, 0x0577, + 0x0578, 0x0579, 0x057a, 0x057b, 0x057c, 0x057d, 0x057e, 0x057f, + 0x0580, 0x0581, 0x0582, 0x0583, 0x0584, 0x0585, 0x0586, 0x0557, + 0x0558, 0x0559, 0x055a, 0x055b, 0x055c, 0x055d, 0x055e, 0x055f, + 0x0560, 0x0561, 0x0562, 0x0563, 0x0564, 0x0565, 0x0566, 0x0567, + 0x0568, 0x0569, 0x056a, 0x056b, 0x056c, 0x056d, 0x056e, 0x056f, + 0x0570, 0x0571, 0x0572, 0x0573, 0x0574, 0x0575, 0x0576, 0x0577, + 0x0578, 0x0579, 0x057a, 0x057b, 0x057c, 0x057d, 0x057e, 0x057f, + }; + +static const unsigned short gNormalizeTable0580[] = { + /* U+0580 */ + 0x0580, 0x0581, 0x0582, 0x0583, 0x0584, 0x0585, 0x0586, 0x0565, + 0x0588, 0x0589, 0x058a, 0x058b, 0x058c, 0x058d, 0x058e, 0x058f, + 0x0590, 0x0591, 0x0592, 0x0593, 0x0594, 0x0595, 0x0596, 0x0597, + 0x0598, 0x0599, 0x059a, 0x059b, 0x059c, 0x059d, 0x059e, 0x059f, + 0x05a0, 0x05a1, 0x05a2, 0x05a3, 0x05a4, 0x05a5, 0x05a6, 0x05a7, + 0x05a8, 0x05a9, 0x05aa, 0x05ab, 0x05ac, 0x05ad, 0x05ae, 0x05af, + 0x05b0, 0x05b1, 0x05b2, 0x05b3, 0x05b4, 0x05b5, 0x05b6, 0x05b7, + 0x05b8, 0x05b9, 0x05ba, 0x05bb, 0x05bc, 0x05bd, 0x05be, 0x05bf, + }; + +static const unsigned short gNormalizeTable0600[] = { + /* U+0600 */ + 0x0600, 0x0601, 0x0602, 0x0603, 0x0604, 0x0605, 0x0606, 0x0607, + 0x0608, 0x0609, 0x060a, 0x060b, 0x060c, 0x060d, 0x060e, 0x060f, + 0x0610, 0x0611, 0x0612, 0x0613, 0x0614, 0x0615, 0x0616, 0x0617, + 0x0618, 0x0619, 0x061a, 0x061b, 0x061c, 0x061d, 0x061e, 0x061f, + 0x0620, 0x0621, 0x0627, 0x0627, 0x0648, 0x0627, 0x064a, 0x0627, + 0x0628, 0x0629, 0x062a, 0x062b, 0x062c, 0x062d, 0x062e, 0x062f, + 0x0630, 0x0631, 0x0632, 0x0633, 0x0634, 0x0635, 0x0636, 0x0637, + 0x0638, 0x0639, 0x063a, 0x063b, 0x063c, 0x063d, 0x063e, 0x063f, + }; + +static const unsigned short gNormalizeTable0640[] = { + /* U+0640 */ + 0x0640, 0x0641, 0x0642, 0x0643, 0x0644, 0x0645, 0x0646, 0x0647, + 0x0648, 0x0649, 0x064a, 0x064b, 0x064c, 0x064d, 0x064e, 0x064f, + 0x0650, 0x0651, 0x0652, 0x0653, 0x0654, 0x0655, 0x0656, 0x0657, + 0x0658, 0x0659, 0x065a, 0x065b, 0x065c, 0x065d, 0x065e, 0x065f, + 0x0660, 0x0661, 0x0662, 0x0663, 0x0664, 0x0665, 0x0666, 0x0667, + 0x0668, 0x0669, 0x066a, 0x066b, 0x066c, 0x066d, 0x066e, 0x066f, + 0x0670, 0x0671, 0x0672, 0x0673, 0x0674, 0x0627, 0x0648, 0x06c7, + 0x064a, 0x0679, 0x067a, 0x067b, 0x067c, 0x067d, 0x067e, 0x067f, + }; + +static const unsigned short gNormalizeTable06c0[] = { + /* U+06c0 */ + 0x06d5, 0x06c1, 0x06c1, 0x06c3, 0x06c4, 0x06c5, 0x06c6, 0x06c7, + 0x06c8, 0x06c9, 0x06ca, 0x06cb, 0x06cc, 0x06cd, 0x06ce, 0x06cf, + 0x06d0, 0x06d1, 0x06d2, 0x06d2, 0x06d4, 0x06d5, 0x06d6, 0x06d7, + 0x06d8, 0x06d9, 0x06da, 0x06db, 0x06dc, 0x06dd, 0x06de, 0x06df, + 0x06e0, 0x06e1, 0x06e2, 0x06e3, 0x06e4, 0x06e5, 0x06e6, 0x06e7, + 0x06e8, 0x06e9, 0x06ea, 0x06eb, 0x06ec, 0x06ed, 0x06ee, 0x06ef, + 0x06f0, 0x06f1, 0x06f2, 0x06f3, 0x06f4, 0x06f5, 0x06f6, 0x06f7, + 0x06f8, 0x06f9, 0x06fa, 0x06fb, 0x06fc, 0x06fd, 0x06fe, 0x06ff, + }; + +static const unsigned short gNormalizeTable0900[] = { + /* U+0900 */ + 0x0900, 0x0901, 0x0902, 0x0903, 0x0904, 0x0905, 0x0906, 0x0907, + 0x0908, 0x0909, 0x090a, 0x090b, 0x090c, 0x090d, 0x090e, 0x090f, + 0x0910, 0x0911, 0x0912, 0x0913, 0x0914, 0x0915, 0x0916, 0x0917, + 0x0918, 0x0919, 0x091a, 0x091b, 0x091c, 0x091d, 0x091e, 0x091f, + 0x0920, 0x0921, 0x0922, 0x0923, 0x0924, 0x0925, 0x0926, 0x0927, + 0x0928, 0x0928, 0x092a, 0x092b, 0x092c, 0x092d, 0x092e, 0x092f, + 0x0930, 0x0930, 0x0932, 0x0933, 0x0933, 0x0935, 0x0936, 0x0937, + 0x0938, 0x0939, 0x093a, 0x093b, 0x093c, 0x093d, 0x093e, 0x093f, + }; + +static const unsigned short gNormalizeTable0940[] = { + /* U+0940 */ + 0x0940, 0x0941, 0x0942, 0x0943, 0x0944, 0x0945, 0x0946, 0x0947, + 0x0948, 0x0949, 0x094a, 0x094b, 0x094c, 0x094d, 0x094e, 0x094f, + 0x0950, 0x0951, 0x0952, 0x0953, 0x0954, 0x0955, 0x0956, 0x0957, + 0x0915, 0x0916, 0x0917, 0x091c, 0x0921, 0x0922, 0x092b, 0x092f, + 0x0960, 0x0961, 0x0962, 0x0963, 0x0964, 0x0965, 0x0966, 0x0967, + 0x0968, 0x0969, 0x096a, 0x096b, 0x096c, 0x096d, 0x096e, 0x096f, + 0x0970, 0x0971, 0x0972, 0x0973, 0x0974, 0x0975, 0x0976, 0x0977, + 0x0978, 0x0979, 0x097a, 0x097b, 0x097c, 0x097d, 0x097e, 0x097f, + }; + +static const unsigned short gNormalizeTable09c0[] = { + /* U+09c0 */ + 0x09c0, 0x09c1, 0x09c2, 0x09c3, 0x09c4, 0x09c5, 0x09c6, 0x09c7, + 0x09c8, 0x09c9, 0x09ca, 0x09c7, 0x09c7, 0x09cd, 0x09ce, 0x09cf, + 0x09d0, 0x09d1, 0x09d2, 0x09d3, 0x09d4, 0x09d5, 0x09d6, 0x09d7, + 0x09d8, 0x09d9, 0x09da, 0x09db, 0x09a1, 0x09a2, 0x09de, 0x09af, + 0x09e0, 0x09e1, 0x09e2, 0x09e3, 0x09e4, 0x09e5, 0x09e6, 0x09e7, + 0x09e8, 0x09e9, 0x09ea, 0x09eb, 0x09ec, 0x09ed, 0x09ee, 0x09ef, + 0x09f0, 0x09f1, 0x09f2, 0x09f3, 0x09f4, 0x09f5, 0x09f6, 0x09f7, + 0x09f8, 0x09f9, 0x09fa, 0x09fb, 0x09fc, 0x09fd, 0x09fe, 0x09ff, + }; + +static const unsigned short gNormalizeTable0a00[] = { + /* U+0a00 */ + 0x0a00, 0x0a01, 0x0a02, 0x0a03, 0x0a04, 0x0a05, 0x0a06, 0x0a07, + 0x0a08, 0x0a09, 0x0a0a, 0x0a0b, 0x0a0c, 0x0a0d, 0x0a0e, 0x0a0f, + 0x0a10, 0x0a11, 0x0a12, 0x0a13, 0x0a14, 0x0a15, 0x0a16, 0x0a17, + 0x0a18, 0x0a19, 0x0a1a, 0x0a1b, 0x0a1c, 0x0a1d, 0x0a1e, 0x0a1f, + 0x0a20, 0x0a21, 0x0a22, 0x0a23, 0x0a24, 0x0a25, 0x0a26, 0x0a27, + 0x0a28, 0x0a29, 0x0a2a, 0x0a2b, 0x0a2c, 0x0a2d, 0x0a2e, 0x0a2f, + 0x0a30, 0x0a31, 0x0a32, 0x0a32, 0x0a34, 0x0a35, 0x0a38, 0x0a37, + 0x0a38, 0x0a39, 0x0a3a, 0x0a3b, 0x0a3c, 0x0a3d, 0x0a3e, 0x0a3f, + }; + +static const unsigned short gNormalizeTable0a40[] = { + /* U+0a40 */ + 0x0a40, 0x0a41, 0x0a42, 0x0a43, 0x0a44, 0x0a45, 0x0a46, 0x0a47, + 0x0a48, 0x0a49, 0x0a4a, 0x0a4b, 0x0a4c, 0x0a4d, 0x0a4e, 0x0a4f, + 0x0a50, 0x0a51, 0x0a52, 0x0a53, 0x0a54, 0x0a55, 0x0a56, 0x0a57, + 0x0a58, 0x0a16, 0x0a17, 0x0a1c, 0x0a5c, 0x0a5d, 0x0a2b, 0x0a5f, + 0x0a60, 0x0a61, 0x0a62, 0x0a63, 0x0a64, 0x0a65, 0x0a66, 0x0a67, + 0x0a68, 0x0a69, 0x0a6a, 0x0a6b, 0x0a6c, 0x0a6d, 0x0a6e, 0x0a6f, + 0x0a70, 0x0a71, 0x0a72, 0x0a73, 0x0a74, 0x0a75, 0x0a76, 0x0a77, + 0x0a78, 0x0a79, 0x0a7a, 0x0a7b, 0x0a7c, 0x0a7d, 0x0a7e, 0x0a7f, + }; + +static const unsigned short gNormalizeTable0b40[] = { + /* U+0b40 */ + 0x0b40, 0x0b41, 0x0b42, 0x0b43, 0x0b44, 0x0b45, 0x0b46, 0x0b47, + 0x0b47, 0x0b49, 0x0b4a, 0x0b47, 0x0b47, 0x0b4d, 0x0b4e, 0x0b4f, + 0x0b50, 0x0b51, 0x0b52, 0x0b53, 0x0b54, 0x0b55, 0x0b56, 0x0b57, + 0x0b58, 0x0b59, 0x0b5a, 0x0b5b, 0x0b21, 0x0b22, 0x0b5e, 0x0b5f, + 0x0b60, 0x0b61, 0x0b62, 0x0b63, 0x0b64, 0x0b65, 0x0b66, 0x0b67, + 0x0b68, 0x0b69, 0x0b6a, 0x0b6b, 0x0b6c, 0x0b6d, 0x0b6e, 0x0b6f, + 0x0b70, 0x0b71, 0x0b72, 0x0b73, 0x0b74, 0x0b75, 0x0b76, 0x0b77, + 0x0b78, 0x0b79, 0x0b7a, 0x0b7b, 0x0b7c, 0x0b7d, 0x0b7e, 0x0b7f, + }; + +static const unsigned short gNormalizeTable0b80[] = { + /* U+0b80 */ + 0x0b80, 0x0b81, 0x0b82, 0x0b83, 0x0b84, 0x0b85, 0x0b86, 0x0b87, + 0x0b88, 0x0b89, 0x0b8a, 0x0b8b, 0x0b8c, 0x0b8d, 0x0b8e, 0x0b8f, + 0x0b90, 0x0b91, 0x0b92, 0x0b93, 0x0b92, 0x0b95, 0x0b96, 0x0b97, + 0x0b98, 0x0b99, 0x0b9a, 0x0b9b, 0x0b9c, 0x0b9d, 0x0b9e, 0x0b9f, + 0x0ba0, 0x0ba1, 0x0ba2, 0x0ba3, 0x0ba4, 0x0ba5, 0x0ba6, 0x0ba7, + 0x0ba8, 0x0ba9, 0x0baa, 0x0bab, 0x0bac, 0x0bad, 0x0bae, 0x0baf, + 0x0bb0, 0x0bb1, 0x0bb2, 0x0bb3, 0x0bb4, 0x0bb5, 0x0bb6, 0x0bb7, + 0x0bb8, 0x0bb9, 0x0bba, 0x0bbb, 0x0bbc, 0x0bbd, 0x0bbe, 0x0bbf, + }; + +static const unsigned short gNormalizeTable0bc0[] = { + /* U+0bc0 */ + 0x0bc0, 0x0bc1, 0x0bc2, 0x0bc3, 0x0bc4, 0x0bc5, 0x0bc6, 0x0bc7, + 0x0bc8, 0x0bc9, 0x0bc6, 0x0bc7, 0x0bc6, 0x0bcd, 0x0bce, 0x0bcf, + 0x0bd0, 0x0bd1, 0x0bd2, 0x0bd3, 0x0bd4, 0x0bd5, 0x0bd6, 0x0bd7, + 0x0bd8, 0x0bd9, 0x0bda, 0x0bdb, 0x0bdc, 0x0bdd, 0x0bde, 0x0bdf, + 0x0be0, 0x0be1, 0x0be2, 0x0be3, 0x0be4, 0x0be5, 0x0be6, 0x0be7, + 0x0be8, 0x0be9, 0x0bea, 0x0beb, 0x0bec, 0x0bed, 0x0bee, 0x0bef, + 0x0bf0, 0x0bf1, 0x0bf2, 0x0bf3, 0x0bf4, 0x0bf5, 0x0bf6, 0x0bf7, + 0x0bf8, 0x0bf9, 0x0bfa, 0x0bfb, 0x0bfc, 0x0bfd, 0x0bfe, 0x0bff, + }; + +static const unsigned short gNormalizeTable0c40[] = { + /* U+0c40 */ + 0x0c40, 0x0c41, 0x0c42, 0x0c43, 0x0c44, 0x0c45, 0x0c46, 0x0c47, + 0x0c46, 0x0c49, 0x0c4a, 0x0c4b, 0x0c4c, 0x0c4d, 0x0c4e, 0x0c4f, + 0x0c50, 0x0c51, 0x0c52, 0x0c53, 0x0c54, 0x0c55, 0x0c56, 0x0c57, + 0x0c58, 0x0c59, 0x0c5a, 0x0c5b, 0x0c5c, 0x0c5d, 0x0c5e, 0x0c5f, + 0x0c60, 0x0c61, 0x0c62, 0x0c63, 0x0c64, 0x0c65, 0x0c66, 0x0c67, + 0x0c68, 0x0c69, 0x0c6a, 0x0c6b, 0x0c6c, 0x0c6d, 0x0c6e, 0x0c6f, + 0x0c70, 0x0c71, 0x0c72, 0x0c73, 0x0c74, 0x0c75, 0x0c76, 0x0c77, + 0x0c78, 0x0c79, 0x0c7a, 0x0c7b, 0x0c7c, 0x0c7d, 0x0c7e, 0x0c7f, + }; + +static const unsigned short gNormalizeTable0cc0[] = { + /* U+0cc0 */ + 0x0cbf, 0x0cc1, 0x0cc2, 0x0cc3, 0x0cc4, 0x0cc5, 0x0cc6, 0x0cc6, + 0x0cc6, 0x0cc9, 0x0cc6, 0x0cc6, 0x0ccc, 0x0ccd, 0x0cce, 0x0ccf, + 0x0cd0, 0x0cd1, 0x0cd2, 0x0cd3, 0x0cd4, 0x0cd5, 0x0cd6, 0x0cd7, + 0x0cd8, 0x0cd9, 0x0cda, 0x0cdb, 0x0cdc, 0x0cdd, 0x0cde, 0x0cdf, + 0x0ce0, 0x0ce1, 0x0ce2, 0x0ce3, 0x0ce4, 0x0ce5, 0x0ce6, 0x0ce7, + 0x0ce8, 0x0ce9, 0x0cea, 0x0ceb, 0x0cec, 0x0ced, 0x0cee, 0x0cef, + 0x0cf0, 0x0cf1, 0x0cf2, 0x0cf3, 0x0cf4, 0x0cf5, 0x0cf6, 0x0cf7, + 0x0cf8, 0x0cf9, 0x0cfa, 0x0cfb, 0x0cfc, 0x0cfd, 0x0cfe, 0x0cff, + }; + +static const unsigned short gNormalizeTable0d40[] = { + /* U+0d40 */ + 0x0d40, 0x0d41, 0x0d42, 0x0d43, 0x0d44, 0x0d45, 0x0d46, 0x0d47, + 0x0d48, 0x0d49, 0x0d46, 0x0d47, 0x0d46, 0x0d4d, 0x0d4e, 0x0d4f, + 0x0d50, 0x0d51, 0x0d52, 0x0d53, 0x0d54, 0x0d55, 0x0d56, 0x0d57, + 0x0d58, 0x0d59, 0x0d5a, 0x0d5b, 0x0d5c, 0x0d5d, 0x0d5e, 0x0d5f, + 0x0d60, 0x0d61, 0x0d62, 0x0d63, 0x0d64, 0x0d65, 0x0d66, 0x0d67, + 0x0d68, 0x0d69, 0x0d6a, 0x0d6b, 0x0d6c, 0x0d6d, 0x0d6e, 0x0d6f, + 0x0d70, 0x0d71, 0x0d72, 0x0d73, 0x0d74, 0x0d75, 0x0d76, 0x0d77, + 0x0d78, 0x0d79, 0x0d7a, 0x0d7b, 0x0d7c, 0x0d7d, 0x0d7e, 0x0d7f, + }; + +static const unsigned short gNormalizeTable0dc0[] = { + /* U+0dc0 */ + 0x0dc0, 0x0dc1, 0x0dc2, 0x0dc3, 0x0dc4, 0x0dc5, 0x0dc6, 0x0dc7, + 0x0dc8, 0x0dc9, 0x0dca, 0x0dcb, 0x0dcc, 0x0dcd, 0x0dce, 0x0dcf, + 0x0dd0, 0x0dd1, 0x0dd2, 0x0dd3, 0x0dd4, 0x0dd5, 0x0dd6, 0x0dd7, + 0x0dd8, 0x0dd9, 0x0dd9, 0x0ddb, 0x0dd9, 0x0dd9, 0x0dd9, 0x0ddf, + 0x0de0, 0x0de1, 0x0de2, 0x0de3, 0x0de4, 0x0de5, 0x0de6, 0x0de7, + 0x0de8, 0x0de9, 0x0dea, 0x0deb, 0x0dec, 0x0ded, 0x0dee, 0x0def, + 0x0df0, 0x0df1, 0x0df2, 0x0df3, 0x0df4, 0x0df5, 0x0df6, 0x0df7, + 0x0df8, 0x0df9, 0x0dfa, 0x0dfb, 0x0dfc, 0x0dfd, 0x0dfe, 0x0dff, + }; + +static const unsigned short gNormalizeTable0e00[] = { + /* U+0e00 */ + 0x0e00, 0x0e01, 0x0e02, 0x0e03, 0x0e04, 0x0e05, 0x0e06, 0x0e07, + 0x0e08, 0x0e09, 0x0e0a, 0x0e0b, 0x0e0c, 0x0e0d, 0x0e0e, 0x0e0f, + 0x0e10, 0x0e11, 0x0e12, 0x0e13, 0x0e14, 0x0e15, 0x0e16, 0x0e17, + 0x0e18, 0x0e19, 0x0e1a, 0x0e1b, 0x0e1c, 0x0e1d, 0x0e1e, 0x0e1f, + 0x0e20, 0x0e21, 0x0e22, 0x0e23, 0x0e24, 0x0e25, 0x0e26, 0x0e27, + 0x0e28, 0x0e29, 0x0e2a, 0x0e2b, 0x0e2c, 0x0e2d, 0x0e2e, 0x0e2f, + 0x0e30, 0x0e31, 0x0e32, 0x0e4d, 0x0e34, 0x0e35, 0x0e36, 0x0e37, + 0x0e38, 0x0e39, 0x0e3a, 0x0e3b, 0x0e3c, 0x0e3d, 0x0e3e, 0x0e3f, + }; + +static const unsigned short gNormalizeTable0e80[] = { + /* U+0e80 */ + 0x0e80, 0x0e81, 0x0e82, 0x0e83, 0x0e84, 0x0e85, 0x0e86, 0x0e87, + 0x0e88, 0x0e89, 0x0e8a, 0x0e8b, 0x0e8c, 0x0e8d, 0x0e8e, 0x0e8f, + 0x0e90, 0x0e91, 0x0e92, 0x0e93, 0x0e94, 0x0e95, 0x0e96, 0x0e97, + 0x0e98, 0x0e99, 0x0e9a, 0x0e9b, 0x0e9c, 0x0e9d, 0x0e9e, 0x0e9f, + 0x0ea0, 0x0ea1, 0x0ea2, 0x0ea3, 0x0ea4, 0x0ea5, 0x0ea6, 0x0ea7, + 0x0ea8, 0x0ea9, 0x0eaa, 0x0eab, 0x0eac, 0x0ead, 0x0eae, 0x0eaf, + 0x0eb0, 0x0eb1, 0x0eb2, 0x0ecd, 0x0eb4, 0x0eb5, 0x0eb6, 0x0eb7, + 0x0eb8, 0x0eb9, 0x0eba, 0x0ebb, 0x0ebc, 0x0ebd, 0x0ebe, 0x0ebf, + }; + +static const unsigned short gNormalizeTable0ec0[] = { + /* U+0ec0 */ + 0x0ec0, 0x0ec1, 0x0ec2, 0x0ec3, 0x0ec4, 0x0ec5, 0x0ec6, 0x0ec7, + 0x0ec8, 0x0ec9, 0x0eca, 0x0ecb, 0x0ecc, 0x0ecd, 0x0ece, 0x0ecf, + 0x0ed0, 0x0ed1, 0x0ed2, 0x0ed3, 0x0ed4, 0x0ed5, 0x0ed6, 0x0ed7, + 0x0ed8, 0x0ed9, 0x0eda, 0x0edb, 0x0eab, 0x0eab, 0x0ede, 0x0edf, + 0x0ee0, 0x0ee1, 0x0ee2, 0x0ee3, 0x0ee4, 0x0ee5, 0x0ee6, 0x0ee7, + 0x0ee8, 0x0ee9, 0x0eea, 0x0eeb, 0x0eec, 0x0eed, 0x0eee, 0x0eef, + 0x0ef0, 0x0ef1, 0x0ef2, 0x0ef3, 0x0ef4, 0x0ef5, 0x0ef6, 0x0ef7, + 0x0ef8, 0x0ef9, 0x0efa, 0x0efb, 0x0efc, 0x0efd, 0x0efe, 0x0eff, + }; + +static const unsigned short gNormalizeTable0f00[] = { + /* U+0f00 */ + 0x0f00, 0x0f01, 0x0f02, 0x0f03, 0x0f04, 0x0f05, 0x0f06, 0x0f07, + 0x0f08, 0x0f09, 0x0f0a, 0x0f0b, 0x0f0b, 0x0f0d, 0x0f0e, 0x0f0f, + 0x0f10, 0x0f11, 0x0f12, 0x0f13, 0x0f14, 0x0f15, 0x0f16, 0x0f17, + 0x0f18, 0x0f19, 0x0f1a, 0x0f1b, 0x0f1c, 0x0f1d, 0x0f1e, 0x0f1f, + 0x0f20, 0x0f21, 0x0f22, 0x0f23, 0x0f24, 0x0f25, 0x0f26, 0x0f27, + 0x0f28, 0x0f29, 0x0f2a, 0x0f2b, 0x0f2c, 0x0f2d, 0x0f2e, 0x0f2f, + 0x0f30, 0x0f31, 0x0f32, 0x0f33, 0x0f34, 0x0f35, 0x0f36, 0x0f37, + 0x0f38, 0x0f39, 0x0f3a, 0x0f3b, 0x0f3c, 0x0f3d, 0x0f3e, 0x0f3f, + }; + +static const unsigned short gNormalizeTable0f40[] = { + /* U+0f40 */ + 0x0f40, 0x0f41, 0x0f42, 0x0f42, 0x0f44, 0x0f45, 0x0f46, 0x0f47, + 0x0f48, 0x0f49, 0x0f4a, 0x0f4b, 0x0f4c, 0x0f4c, 0x0f4e, 0x0f4f, + 0x0f50, 0x0f51, 0x0f51, 0x0f53, 0x0f54, 0x0f55, 0x0f56, 0x0f56, + 0x0f58, 0x0f59, 0x0f5a, 0x0f5b, 0x0f5b, 0x0f5d, 0x0f5e, 0x0f5f, + 0x0f60, 0x0f61, 0x0f62, 0x0f63, 0x0f64, 0x0f65, 0x0f66, 0x0f67, + 0x0f68, 0x0f40, 0x0f6a, 0x0f6b, 0x0f6c, 0x0f6d, 0x0f6e, 0x0f6f, + 0x0f70, 0x0f71, 0x0f72, 0x0f71, 0x0f74, 0x0f71, 0x0fb2, 0x0fb2, + 0x0fb3, 0x0fb3, 0x0f7a, 0x0f7b, 0x0f7c, 0x0f7d, 0x0f7e, 0x0f7f, + }; + +static const unsigned short gNormalizeTable0f80[] = { + /* U+0f80 */ + 0x0f80, 0x0f71, 0x0f82, 0x0f83, 0x0f84, 0x0f85, 0x0f86, 0x0f87, + 0x0f88, 0x0f89, 0x0f8a, 0x0f8b, 0x0f8c, 0x0f8d, 0x0f8e, 0x0f8f, + 0x0f90, 0x0f91, 0x0f92, 0x0f92, 0x0f94, 0x0f95, 0x0f96, 0x0f97, + 0x0f98, 0x0f99, 0x0f9a, 0x0f9b, 0x0f9c, 0x0f9c, 0x0f9e, 0x0f9f, + 0x0fa0, 0x0fa1, 0x0fa1, 0x0fa3, 0x0fa4, 0x0fa5, 0x0fa6, 0x0fa6, + 0x0fa8, 0x0fa9, 0x0faa, 0x0fab, 0x0fab, 0x0fad, 0x0fae, 0x0faf, + 0x0fb0, 0x0fb1, 0x0fb2, 0x0fb3, 0x0fb4, 0x0fb5, 0x0fb6, 0x0fb7, + 0x0fb8, 0x0f90, 0x0fba, 0x0fbb, 0x0fbc, 0x0fbd, 0x0fbe, 0x0fbf, + }; + +static const unsigned short gNormalizeTable1000[] = { + /* U+1000 */ + 0x1000, 0x1001, 0x1002, 0x1003, 0x1004, 0x1005, 0x1006, 0x1007, + 0x1008, 0x1009, 0x100a, 0x100b, 0x100c, 0x100d, 0x100e, 0x100f, + 0x1010, 0x1011, 0x1012, 0x1013, 0x1014, 0x1015, 0x1016, 0x1017, + 0x1018, 0x1019, 0x101a, 0x101b, 0x101c, 0x101d, 0x101e, 0x101f, + 0x1020, 0x1021, 0x1022, 0x1023, 0x1024, 0x1025, 0x1025, 0x1027, + 0x1028, 0x1029, 0x102a, 0x102b, 0x102c, 0x102d, 0x102e, 0x102f, + 0x1030, 0x1031, 0x1032, 0x1033, 0x1034, 0x1035, 0x1036, 0x1037, + 0x1038, 0x1039, 0x103a, 0x103b, 0x103c, 0x103d, 0x103e, 0x103f, + }; + +static const unsigned short gNormalizeTable1080[] = { + /* U+1080 */ + 0x1080, 0x1081, 0x1082, 0x1083, 0x1084, 0x1085, 0x1086, 0x1087, + 0x1088, 0x1089, 0x108a, 0x108b, 0x108c, 0x108d, 0x108e, 0x108f, + 0x1090, 0x1091, 0x1092, 0x1093, 0x1094, 0x1095, 0x1096, 0x1097, + 0x1098, 0x1099, 0x109a, 0x109b, 0x109c, 0x109d, 0x109e, 0x109f, + 0x2d00, 0x2d01, 0x2d02, 0x2d03, 0x2d04, 0x2d05, 0x2d06, 0x2d07, + 0x2d08, 0x2d09, 0x2d0a, 0x2d0b, 0x2d0c, 0x2d0d, 0x2d0e, 0x2d0f, + 0x2d10, 0x2d11, 0x2d12, 0x2d13, 0x2d14, 0x2d15, 0x2d16, 0x2d17, + 0x2d18, 0x2d19, 0x2d1a, 0x2d1b, 0x2d1c, 0x2d1d, 0x2d1e, 0x2d1f, + }; + +static const unsigned short gNormalizeTable10c0[] = { + /* U+10c0 */ + 0x2d20, 0x2d21, 0x2d22, 0x2d23, 0x2d24, 0x2d25, 0x10c6, 0x10c7, + 0x10c8, 0x10c9, 0x10ca, 0x10cb, 0x10cc, 0x10cd, 0x10ce, 0x10cf, + 0x10d0, 0x10d1, 0x10d2, 0x10d3, 0x10d4, 0x10d5, 0x10d6, 0x10d7, + 0x10d8, 0x10d9, 0x10da, 0x10db, 0x10dc, 0x10dd, 0x10de, 0x10df, + 0x10e0, 0x10e1, 0x10e2, 0x10e3, 0x10e4, 0x10e5, 0x10e6, 0x10e7, + 0x10e8, 0x10e9, 0x10ea, 0x10eb, 0x10ec, 0x10ed, 0x10ee, 0x10ef, + 0x10f0, 0x10f1, 0x10f2, 0x10f3, 0x10f4, 0x10f5, 0x10f6, 0x10f7, + 0x10f8, 0x10f9, 0x10fa, 0x10fb, 0x10dc, 0x10fd, 0x10fe, 0x10ff, + }; + +static const unsigned short gNormalizeTable1140[] = { + /* U+1140 */ + 0x1140, 0x1141, 0x1142, 0x1143, 0x1144, 0x1145, 0x1146, 0x1147, + 0x1148, 0x1149, 0x114a, 0x114b, 0x114c, 0x114d, 0x114e, 0x114f, + 0x1150, 0x1151, 0x1152, 0x1153, 0x1154, 0x1155, 0x1156, 0x1157, + 0x1158, 0x1159, 0x115a, 0x115b, 0x115c, 0x115d, 0x115e, 0x0020, + 0x0020, 0x1161, 0x1162, 0x1163, 0x1164, 0x1165, 0x1166, 0x1167, + 0x1168, 0x1169, 0x116a, 0x116b, 0x116c, 0x116d, 0x116e, 0x116f, + 0x1170, 0x1171, 0x1172, 0x1173, 0x1174, 0x1175, 0x1176, 0x1177, + 0x1178, 0x1179, 0x117a, 0x117b, 0x117c, 0x117d, 0x117e, 0x117f, + }; + +static const unsigned short gNormalizeTable1780[] = { + /* U+1780 */ + 0x1780, 0x1781, 0x1782, 0x1783, 0x1784, 0x1785, 0x1786, 0x1787, + 0x1788, 0x1789, 0x178a, 0x178b, 0x178c, 0x178d, 0x178e, 0x178f, + 0x1790, 0x1791, 0x1792, 0x1793, 0x1794, 0x1795, 0x1796, 0x1797, + 0x1798, 0x1799, 0x179a, 0x179b, 0x179c, 0x179d, 0x179e, 0x179f, + 0x17a0, 0x17a1, 0x17a2, 0x17a3, 0x17a4, 0x17a5, 0x17a6, 0x17a7, + 0x17a8, 0x17a9, 0x17aa, 0x17ab, 0x17ac, 0x17ad, 0x17ae, 0x17af, + 0x17b0, 0x17b1, 0x17b2, 0x17b3, 0x0020, 0x0020, 0x17b6, 0x17b7, + 0x17b8, 0x17b9, 0x17ba, 0x17bb, 0x17bc, 0x17bd, 0x17be, 0x17bf, + }; + +static const unsigned short gNormalizeTable1800[] = { + /* U+1800 */ + 0x1800, 0x1801, 0x1802, 0x1803, 0x1804, 0x1805, 0x1806, 0x1807, + 0x1808, 0x1809, 0x180a, 0x0020, 0x0020, 0x0020, 0x180e, 0x180f, + 0x1810, 0x1811, 0x1812, 0x1813, 0x1814, 0x1815, 0x1816, 0x1817, + 0x1818, 0x1819, 0x181a, 0x181b, 0x181c, 0x181d, 0x181e, 0x181f, + 0x1820, 0x1821, 0x1822, 0x1823, 0x1824, 0x1825, 0x1826, 0x1827, + 0x1828, 0x1829, 0x182a, 0x182b, 0x182c, 0x182d, 0x182e, 0x182f, + 0x1830, 0x1831, 0x1832, 0x1833, 0x1834, 0x1835, 0x1836, 0x1837, + 0x1838, 0x1839, 0x183a, 0x183b, 0x183c, 0x183d, 0x183e, 0x183f, + }; + +static const unsigned short gNormalizeTable1b00[] = { + /* U+1b00 */ + 0x1b00, 0x1b01, 0x1b02, 0x1b03, 0x1b04, 0x1b05, 0x1b05, 0x1b07, + 0x1b07, 0x1b09, 0x1b09, 0x1b0b, 0x1b0b, 0x1b0d, 0x1b0d, 0x1b0f, + 0x1b10, 0x1b11, 0x1b11, 0x1b13, 0x1b14, 0x1b15, 0x1b16, 0x1b17, + 0x1b18, 0x1b19, 0x1b1a, 0x1b1b, 0x1b1c, 0x1b1d, 0x1b1e, 0x1b1f, + 0x1b20, 0x1b21, 0x1b22, 0x1b23, 0x1b24, 0x1b25, 0x1b26, 0x1b27, + 0x1b28, 0x1b29, 0x1b2a, 0x1b2b, 0x1b2c, 0x1b2d, 0x1b2e, 0x1b2f, + 0x1b30, 0x1b31, 0x1b32, 0x1b33, 0x1b34, 0x1b35, 0x1b36, 0x1b37, + 0x1b38, 0x1b39, 0x1b3a, 0x1b3a, 0x1b3c, 0x1b3c, 0x1b3e, 0x1b3f, + }; + +static const unsigned short gNormalizeTable1b40[] = { + /* U+1b40 */ + 0x1b3e, 0x1b3f, 0x1b42, 0x1b42, 0x1b44, 0x1b45, 0x1b46, 0x1b47, + 0x1b48, 0x1b49, 0x1b4a, 0x1b4b, 0x1b4c, 0x1b4d, 0x1b4e, 0x1b4f, + 0x1b50, 0x1b51, 0x1b52, 0x1b53, 0x1b54, 0x1b55, 0x1b56, 0x1b57, + 0x1b58, 0x1b59, 0x1b5a, 0x1b5b, 0x1b5c, 0x1b5d, 0x1b5e, 0x1b5f, + 0x1b60, 0x1b61, 0x1b62, 0x1b63, 0x1b64, 0x1b65, 0x1b66, 0x1b67, + 0x1b68, 0x1b69, 0x1b6a, 0x1b6b, 0x1b6c, 0x1b6d, 0x1b6e, 0x1b6f, + 0x1b70, 0x1b71, 0x1b72, 0x1b73, 0x1b74, 0x1b75, 0x1b76, 0x1b77, + 0x1b78, 0x1b79, 0x1b7a, 0x1b7b, 0x1b7c, 0x1b7d, 0x1b7e, 0x1b7f, + }; + +static const unsigned short gNormalizeTable1d00[] = { + /* U+1d00 */ + 0x1d00, 0x1d01, 0x1d02, 0x1d03, 0x1d04, 0x1d05, 0x1d06, 0x1d07, + 0x1d08, 0x1d09, 0x1d0a, 0x1d0b, 0x1d0c, 0x1d0d, 0x1d0e, 0x1d0f, + 0x1d10, 0x1d11, 0x1d12, 0x1d13, 0x1d14, 0x1d15, 0x1d16, 0x1d17, + 0x1d18, 0x1d19, 0x1d1a, 0x1d1b, 0x1d1c, 0x1d1d, 0x1d1e, 0x1d1f, + 0x1d20, 0x1d21, 0x1d22, 0x1d23, 0x1d24, 0x1d25, 0x1d26, 0x1d27, + 0x1d28, 0x1d29, 0x1d2a, 0x1d2b, 0x0061, 0x00e6, 0x0062, 0x1d2f, + 0x0064, 0x0065, 0x01dd, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x1d3b, 0x006f, 0x0223, 0x0070, 0x0072, + }; + +static const unsigned short gNormalizeTable1d40[] = { + /* U+1d40 */ + 0x0074, 0x0075, 0x0077, 0x0061, 0x0250, 0x0251, 0x1d02, 0x0062, + 0x0064, 0x0065, 0x0259, 0x025b, 0x025c, 0x0067, 0x1d4e, 0x006b, + 0x006d, 0x014b, 0x006f, 0x0254, 0x1d16, 0x1d17, 0x0070, 0x0074, + 0x0075, 0x1d1d, 0x026f, 0x0076, 0x1d25, 0x03b2, 0x03b3, 0x03b4, + 0x03c6, 0x03c7, 0x0069, 0x0072, 0x0075, 0x0076, 0x03b2, 0x03b3, + 0x03c1, 0x03c6, 0x03c7, 0x1d6b, 0x1d6c, 0x1d6d, 0x1d6e, 0x1d6f, + 0x1d70, 0x1d71, 0x1d72, 0x1d73, 0x1d74, 0x1d75, 0x1d76, 0x1d77, + 0x043d, 0x1d79, 0x1d7a, 0x1d7b, 0x1d7c, 0x1d7d, 0x1d7e, 0x1d7f, + }; + +static const unsigned short gNormalizeTable1d80[] = { + /* U+1d80 */ + 0x1d80, 0x1d81, 0x1d82, 0x1d83, 0x1d84, 0x1d85, 0x1d86, 0x1d87, + 0x1d88, 0x1d89, 0x1d8a, 0x1d8b, 0x1d8c, 0x1d8d, 0x1d8e, 0x1d8f, + 0x1d90, 0x1d91, 0x1d92, 0x1d93, 0x1d94, 0x1d95, 0x1d96, 0x1d97, + 0x1d98, 0x1d99, 0x1d9a, 0x0252, 0x0063, 0x0255, 0x00f0, 0x025c, + 0x0066, 0x025f, 0x0261, 0x0265, 0x0268, 0x0269, 0x026a, 0x1d7b, + 0x029d, 0x026d, 0x1d85, 0x029f, 0x0271, 0x0270, 0x0272, 0x0273, + 0x0274, 0x0275, 0x0278, 0x0282, 0x0283, 0x01ab, 0x0289, 0x028a, + 0x1d1c, 0x028b, 0x028c, 0x007a, 0x0290, 0x0291, 0x0292, 0x03b8, + }; + +static const unsigned short gNormalizeTable1e00[] = { + /* U+1e00 */ + 0x0061, 0x0061, 0x0062, 0x0062, 0x0062, 0x0062, 0x0062, 0x0062, + 0x0063, 0x0063, 0x0064, 0x0064, 0x0064, 0x0064, 0x0064, 0x0064, + 0x0064, 0x0064, 0x0064, 0x0064, 0x0065, 0x0065, 0x0065, 0x0065, + 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, 0x0066, 0x0066, + 0x0067, 0x0067, 0x0068, 0x0068, 0x0068, 0x0068, 0x0068, 0x0068, + 0x0068, 0x0068, 0x0068, 0x0068, 0x0069, 0x0069, 0x0069, 0x0069, + 0x006b, 0x006b, 0x006b, 0x006b, 0x006b, 0x006b, 0x006c, 0x006c, + 0x006c, 0x006c, 0x006c, 0x006c, 0x006c, 0x006c, 0x006d, 0x006d, + }; + +static const unsigned short gNormalizeTable1e40[] = { + /* U+1e40 */ + 0x006d, 0x006d, 0x006d, 0x006d, 0x006e, 0x006e, 0x006e, 0x006e, + 0x006e, 0x006e, 0x006e, 0x006e, 0x006f, 0x006f, 0x006f, 0x006f, + 0x006f, 0x006f, 0x006f, 0x006f, 0x0070, 0x0070, 0x0070, 0x0070, + 0x0072, 0x0072, 0x0072, 0x0072, 0x0072, 0x0072, 0x0072, 0x0072, + 0x0073, 0x0073, 0x0073, 0x0073, 0x0073, 0x0073, 0x0073, 0x0073, + 0x0073, 0x0073, 0x0074, 0x0074, 0x0074, 0x0074, 0x0074, 0x0074, + 0x0074, 0x0074, 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, + 0x0075, 0x0075, 0x0075, 0x0075, 0x0076, 0x0076, 0x0076, 0x0076, + }; + +static const unsigned short gNormalizeTable1e80[] = { + /* U+1e80 */ + 0x0077, 0x0077, 0x0077, 0x0077, 0x0077, 0x0077, 0x0077, 0x0077, + 0x0077, 0x0077, 0x0078, 0x0078, 0x0078, 0x0078, 0x0079, 0x0079, + 0x007a, 0x007a, 0x007a, 0x007a, 0x007a, 0x007a, 0x0068, 0x0074, + 0x0077, 0x0079, 0x0061, 0x0073, 0x1e9c, 0x1e9d, 0x0073, 0x1e9f, + 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, + 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, + 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, + 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, + }; + +static const unsigned short gNormalizeTable1ec0[] = { + /* U+1ec0 */ + 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, + 0x0069, 0x0069, 0x0069, 0x0069, 0x006f, 0x006f, 0x006f, 0x006f, + 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, + 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, + 0x006f, 0x006f, 0x006f, 0x006f, 0x0075, 0x0075, 0x0075, 0x0075, + 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, + 0x0075, 0x0075, 0x0079, 0x0079, 0x0079, 0x0079, 0x0079, 0x0079, + 0x0079, 0x0079, 0x1efb, 0x1efb, 0x1efd, 0x1efd, 0x1eff, 0x1eff, + }; + +static const unsigned short gNormalizeTable1f00[] = { + /* U+1f00 */ + 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, + 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, + 0x03b5, 0x03b5, 0x03b5, 0x03b5, 0x03b5, 0x03b5, 0x1f16, 0x1f17, + 0x03b5, 0x03b5, 0x03b5, 0x03b5, 0x03b5, 0x03b5, 0x1f1e, 0x1f1f, + 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, + 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, + 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x03b9, + 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x03b9, + }; + +static const unsigned short gNormalizeTable1f40[] = { + /* U+1f40 */ + 0x03bf, 0x03bf, 0x03bf, 0x03bf, 0x03bf, 0x03bf, 0x1f46, 0x1f47, + 0x03bf, 0x03bf, 0x03bf, 0x03bf, 0x03bf, 0x03bf, 0x1f4e, 0x1f4f, + 0x03c5, 0x03c5, 0x03c5, 0x03c5, 0x03c5, 0x03c5, 0x03c5, 0x03c5, + 0x1f58, 0x03c5, 0x1f5a, 0x03c5, 0x1f5c, 0x03c5, 0x1f5e, 0x03c5, + 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, + 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, + 0x03b1, 0x03b1, 0x03b5, 0x03b5, 0x03b7, 0x03b7, 0x03b9, 0x03b9, + 0x03bf, 0x03bf, 0x03c5, 0x03c5, 0x03c9, 0x03c9, 0x1f7e, 0x1f7f, + }; + +static const unsigned short gNormalizeTable1f80[] = { + /* U+1f80 */ + 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, + 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, + 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, + 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, + 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, + 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, + 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x1fb5, 0x03b1, 0x03b1, + 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x0020, 0x03b9, 0x0020, + }; + +static const unsigned short gNormalizeTable1fc0[] = { + /* U+1fc0 */ + 0x0020, 0x0020, 0x03b7, 0x03b7, 0x03b7, 0x1fc5, 0x03b7, 0x03b7, + 0x03b5, 0x03b5, 0x03b7, 0x03b7, 0x03b7, 0x0020, 0x0020, 0x0020, + 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x1fd4, 0x1fd5, 0x03b9, 0x03b9, + 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x1fdc, 0x0020, 0x0020, 0x0020, + 0x03c5, 0x03c5, 0x03c5, 0x03c5, 0x03c1, 0x03c1, 0x03c5, 0x03c5, + 0x03c5, 0x03c5, 0x03c5, 0x03c5, 0x03c1, 0x0020, 0x0020, 0x0060, + 0x1ff0, 0x1ff1, 0x03c9, 0x03c9, 0x03c9, 0x1ff5, 0x03c9, 0x03c9, + 0x03bf, 0x03bf, 0x03c9, 0x03c9, 0x03c9, 0x0020, 0x0020, 0x1fff, + }; + +static const unsigned short gNormalizeTable2000[] = { + /* U+2000 */ + 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, + 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, + 0x2010, 0x2010, 0x2012, 0x2013, 0x2014, 0x2015, 0x2016, 0x0020, + 0x2018, 0x2019, 0x201a, 0x201b, 0x201c, 0x201d, 0x201e, 0x201f, + 0x2020, 0x2021, 0x2022, 0x2023, 0x002e, 0x002e, 0x002e, 0x2027, + 0x2028, 0x2029, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, + 0x2030, 0x2031, 0x2032, 0x2032, 0x2032, 0x2035, 0x2035, 0x2035, + 0x2038, 0x2039, 0x203a, 0x203b, 0x0021, 0x203d, 0x0020, 0x203f, + }; + +static const unsigned short gNormalizeTable2040[] = { + /* U+2040 */ + 0x2040, 0x2041, 0x2042, 0x2043, 0x2044, 0x2045, 0x2046, 0x003f, + 0x003f, 0x0021, 0x204a, 0x204b, 0x204c, 0x204d, 0x204e, 0x204f, + 0x2050, 0x2051, 0x2052, 0x2053, 0x2054, 0x2055, 0x2056, 0x2032, + 0x2058, 0x2059, 0x205a, 0x205b, 0x205c, 0x205d, 0x205e, 0x0020, + 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, + 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, + 0x0030, 0x0069, 0x2072, 0x2073, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x002b, 0x2212, 0x003d, 0x0028, 0x0029, 0x006e, + }; + +static const unsigned short gNormalizeTable2080[] = { + /* U+2080 */ + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x002b, 0x2212, 0x003d, 0x0028, 0x0029, 0x208f, + 0x0061, 0x0065, 0x006f, 0x0078, 0x0259, 0x2095, 0x2096, 0x2097, + 0x2098, 0x2099, 0x209a, 0x209b, 0x209c, 0x209d, 0x209e, 0x209f, + 0x20a0, 0x20a1, 0x20a2, 0x20a3, 0x20a4, 0x20a5, 0x20a6, 0x20a7, + 0x0072, 0x20a9, 0x20aa, 0x20ab, 0x20ac, 0x20ad, 0x20ae, 0x20af, + 0x20b0, 0x20b1, 0x20b2, 0x20b3, 0x20b4, 0x20b5, 0x20b6, 0x20b7, + 0x20b8, 0x20b9, 0x20ba, 0x20bb, 0x20bc, 0x20bd, 0x20be, 0x20bf, + }; + +static const unsigned short gNormalizeTable2100[] = { + /* U+2100 */ + 0x0061, 0x0061, 0x0063, 0x00b0, 0x2104, 0x0063, 0x0063, 0x025b, + 0x2108, 0x00b0, 0x0067, 0x0068, 0x0068, 0x0068, 0x0068, 0x0127, + 0x0069, 0x0069, 0x006c, 0x006c, 0x2114, 0x006e, 0x006e, 0x2117, + 0x2118, 0x0070, 0x0071, 0x0072, 0x0072, 0x0072, 0x211e, 0x211f, + 0x0073, 0x0074, 0x0074, 0x2123, 0x007a, 0x2125, 0x03c9, 0x2127, + 0x007a, 0x2129, 0x006b, 0x0061, 0x0062, 0x0063, 0x212e, 0x0065, + 0x0065, 0x0066, 0x214e, 0x006d, 0x006f, 0x05d0, 0x05d1, 0x05d2, + 0x05d3, 0x0069, 0x213a, 0x0066, 0x03c0, 0x03b3, 0x03b3, 0x03c0, + }; + +static const unsigned short gNormalizeTable2140[] = { + /* U+2140 */ + 0x2211, 0x2141, 0x2142, 0x2143, 0x2144, 0x0064, 0x0064, 0x0065, + 0x0069, 0x006a, 0x214a, 0x214b, 0x214c, 0x214d, 0x214e, 0x214f, + 0x0031, 0x0031, 0x0031, 0x0031, 0x0032, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0031, 0x0035, 0x0031, 0x0033, 0x0035, 0x0037, 0x0031, + 0x0069, 0x0069, 0x0069, 0x0069, 0x0076, 0x0076, 0x0076, 0x0076, + 0x0069, 0x0078, 0x0078, 0x0078, 0x006c, 0x0063, 0x0064, 0x006d, + 0x0069, 0x0069, 0x0069, 0x0069, 0x0076, 0x0076, 0x0076, 0x0076, + 0x0069, 0x0078, 0x0078, 0x0078, 0x006c, 0x0063, 0x0064, 0x006d, + }; + +static const unsigned short gNormalizeTable2180[] = { + /* U+2180 */ + 0x2180, 0x2181, 0x2182, 0x2184, 0x2184, 0x2185, 0x2186, 0x2187, + 0x2188, 0x0030, 0x218a, 0x218b, 0x218c, 0x218d, 0x218e, 0x218f, + 0x2190, 0x2191, 0x2192, 0x2193, 0x2194, 0x2195, 0x2196, 0x2197, + 0x2198, 0x2199, 0x2190, 0x2192, 0x219c, 0x219d, 0x219e, 0x219f, + 0x21a0, 0x21a1, 0x21a2, 0x21a3, 0x21a4, 0x21a5, 0x21a6, 0x21a7, + 0x21a8, 0x21a9, 0x21aa, 0x21ab, 0x21ac, 0x21ad, 0x2194, 0x21af, + 0x21b0, 0x21b1, 0x21b2, 0x21b3, 0x21b4, 0x21b5, 0x21b6, 0x21b7, + 0x21b8, 0x21b9, 0x21ba, 0x21bb, 0x21bc, 0x21bd, 0x21be, 0x21bf, + }; + +static const unsigned short gNormalizeTable21c0[] = { + /* U+21c0 */ + 0x21c0, 0x21c1, 0x21c2, 0x21c3, 0x21c4, 0x21c5, 0x21c6, 0x21c7, + 0x21c8, 0x21c9, 0x21ca, 0x21cb, 0x21cc, 0x21d0, 0x21d4, 0x21d2, + 0x21d0, 0x21d1, 0x21d2, 0x21d3, 0x21d4, 0x21d5, 0x21d6, 0x21d7, + 0x21d8, 0x21d9, 0x21da, 0x21db, 0x21dc, 0x21dd, 0x21de, 0x21df, + 0x21e0, 0x21e1, 0x21e2, 0x21e3, 0x21e4, 0x21e5, 0x21e6, 0x21e7, + 0x21e8, 0x21e9, 0x21ea, 0x21eb, 0x21ec, 0x21ed, 0x21ee, 0x21ef, + 0x21f0, 0x21f1, 0x21f2, 0x21f3, 0x21f4, 0x21f5, 0x21f6, 0x21f7, + 0x21f8, 0x21f9, 0x21fa, 0x21fb, 0x21fc, 0x21fd, 0x21fe, 0x21ff, + }; + +static const unsigned short gNormalizeTable2200[] = { + /* U+2200 */ + 0x2200, 0x2201, 0x2202, 0x2203, 0x2203, 0x2205, 0x2206, 0x2207, + 0x2208, 0x2208, 0x220a, 0x220b, 0x220b, 0x220d, 0x220e, 0x220f, + 0x2210, 0x2211, 0x2212, 0x2213, 0x2214, 0x2215, 0x2216, 0x2217, + 0x2218, 0x2219, 0x221a, 0x221b, 0x221c, 0x221d, 0x221e, 0x221f, + 0x2220, 0x2221, 0x2222, 0x2223, 0x2223, 0x2225, 0x2225, 0x2227, + 0x2228, 0x2229, 0x222a, 0x222b, 0x222b, 0x222b, 0x222e, 0x222e, + 0x222e, 0x2231, 0x2232, 0x2233, 0x2234, 0x2235, 0x2236, 0x2237, + 0x2238, 0x2239, 0x223a, 0x223b, 0x223c, 0x223d, 0x223e, 0x223f, + }; + +static const unsigned short gNormalizeTable2240[] = { + /* U+2240 */ + 0x2240, 0x223c, 0x2242, 0x2243, 0x2243, 0x2245, 0x2246, 0x2245, + 0x2248, 0x2248, 0x224a, 0x224b, 0x224c, 0x224d, 0x224e, 0x224f, + 0x2250, 0x2251, 0x2252, 0x2253, 0x2254, 0x2255, 0x2256, 0x2257, + 0x2258, 0x2259, 0x225a, 0x225b, 0x225c, 0x225d, 0x225e, 0x225f, + 0x003d, 0x2261, 0x2261, 0x2263, 0x2264, 0x2265, 0x2266, 0x2267, + 0x2268, 0x2269, 0x226a, 0x226b, 0x226c, 0x224d, 0x003c, 0x003e, + 0x2264, 0x2265, 0x2272, 0x2273, 0x2272, 0x2273, 0x2276, 0x2277, + 0x2276, 0x2277, 0x227a, 0x227b, 0x227c, 0x227d, 0x227e, 0x227f, + }; + +static const unsigned short gNormalizeTable2280[] = { + /* U+2280 */ + 0x227a, 0x227b, 0x2282, 0x2283, 0x2282, 0x2283, 0x2286, 0x2287, + 0x2286, 0x2287, 0x228a, 0x228b, 0x228c, 0x228d, 0x228e, 0x228f, + 0x2290, 0x2291, 0x2292, 0x2293, 0x2294, 0x2295, 0x2296, 0x2297, + 0x2298, 0x2299, 0x229a, 0x229b, 0x229c, 0x229d, 0x229e, 0x229f, + 0x22a0, 0x22a1, 0x22a2, 0x22a3, 0x22a4, 0x22a5, 0x22a6, 0x22a7, + 0x22a8, 0x22a9, 0x22aa, 0x22ab, 0x22a2, 0x22a8, 0x22a9, 0x22ab, + 0x22b0, 0x22b1, 0x22b2, 0x22b3, 0x22b4, 0x22b5, 0x22b6, 0x22b7, + 0x22b8, 0x22b9, 0x22ba, 0x22bb, 0x22bc, 0x22bd, 0x22be, 0x22bf, + }; + +static const unsigned short gNormalizeTable22c0[] = { + /* U+22c0 */ + 0x22c0, 0x22c1, 0x22c2, 0x22c3, 0x22c4, 0x22c5, 0x22c6, 0x22c7, + 0x22c8, 0x22c9, 0x22ca, 0x22cb, 0x22cc, 0x22cd, 0x22ce, 0x22cf, + 0x22d0, 0x22d1, 0x22d2, 0x22d3, 0x22d4, 0x22d5, 0x22d6, 0x22d7, + 0x22d8, 0x22d9, 0x22da, 0x22db, 0x22dc, 0x22dd, 0x22de, 0x22df, + 0x227c, 0x227d, 0x2291, 0x2292, 0x22e4, 0x22e5, 0x22e6, 0x22e7, + 0x22e8, 0x22e9, 0x22b2, 0x22b3, 0x22b4, 0x22b5, 0x22ee, 0x22ef, + 0x22f0, 0x22f1, 0x22f2, 0x22f3, 0x22f4, 0x22f5, 0x22f6, 0x22f7, + 0x22f8, 0x22f9, 0x22fa, 0x22fb, 0x22fc, 0x22fd, 0x22fe, 0x22ff, + }; + +static const unsigned short gNormalizeTable2300[] = { + /* U+2300 */ + 0x2300, 0x2301, 0x2302, 0x2303, 0x2304, 0x2305, 0x2306, 0x2307, + 0x2308, 0x2309, 0x230a, 0x230b, 0x230c, 0x230d, 0x230e, 0x230f, + 0x2310, 0x2311, 0x2312, 0x2313, 0x2314, 0x2315, 0x2316, 0x2317, + 0x2318, 0x2319, 0x231a, 0x231b, 0x231c, 0x231d, 0x231e, 0x231f, + 0x2320, 0x2321, 0x2322, 0x2323, 0x2324, 0x2325, 0x2326, 0x2327, + 0x2328, 0x3008, 0x3009, 0x232b, 0x232c, 0x232d, 0x232e, 0x232f, + 0x2330, 0x2331, 0x2332, 0x2333, 0x2334, 0x2335, 0x2336, 0x2337, + 0x2338, 0x2339, 0x233a, 0x233b, 0x233c, 0x233d, 0x233e, 0x233f, + }; + +static const unsigned short gNormalizeTable2440[] = { + /* U+2440 */ + 0x2440, 0x2441, 0x2442, 0x2443, 0x2444, 0x2445, 0x2446, 0x2447, + 0x2448, 0x2449, 0x244a, 0x244b, 0x244c, 0x244d, 0x244e, 0x244f, + 0x2450, 0x2451, 0x2452, 0x2453, 0x2454, 0x2455, 0x2456, 0x2457, + 0x2458, 0x2459, 0x245a, 0x245b, 0x245c, 0x245d, 0x245e, 0x245f, + 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, + 0x0039, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, + 0x0031, 0x0031, 0x0031, 0x0032, 0x0028, 0x0028, 0x0028, 0x0028, + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, + }; + +static const unsigned short gNormalizeTable2480[] = { + /* U+2480 */ + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, + 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, + 0x0039, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, + 0x0031, 0x0031, 0x0031, 0x0032, 0x0028, 0x0028, 0x0028, 0x0028, + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0061, 0x0062, + 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, + }; + +static const unsigned short gNormalizeTable24c0[] = { + /* U+24c0 */ + 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, 0x0070, 0x0071, 0x0072, + 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, + 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, + 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, 0x0070, + 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, + 0x0079, 0x007a, 0x0030, 0x24eb, 0x24ec, 0x24ed, 0x24ee, 0x24ef, + 0x24f0, 0x24f1, 0x24f2, 0x24f3, 0x24f4, 0x24f5, 0x24f6, 0x24f7, + 0x24f8, 0x24f9, 0x24fa, 0x24fb, 0x24fc, 0x24fd, 0x24fe, 0x24ff, + }; + +static const unsigned short gNormalizeTable2a00[] = { + /* U+2a00 */ + 0x2a00, 0x2a01, 0x2a02, 0x2a03, 0x2a04, 0x2a05, 0x2a06, 0x2a07, + 0x2a08, 0x2a09, 0x2a0a, 0x2a0b, 0x222b, 0x2a0d, 0x2a0e, 0x2a0f, + 0x2a10, 0x2a11, 0x2a12, 0x2a13, 0x2a14, 0x2a15, 0x2a16, 0x2a17, + 0x2a18, 0x2a19, 0x2a1a, 0x2a1b, 0x2a1c, 0x2a1d, 0x2a1e, 0x2a1f, + 0x2a20, 0x2a21, 0x2a22, 0x2a23, 0x2a24, 0x2a25, 0x2a26, 0x2a27, + 0x2a28, 0x2a29, 0x2a2a, 0x2a2b, 0x2a2c, 0x2a2d, 0x2a2e, 0x2a2f, + 0x2a30, 0x2a31, 0x2a32, 0x2a33, 0x2a34, 0x2a35, 0x2a36, 0x2a37, + 0x2a38, 0x2a39, 0x2a3a, 0x2a3b, 0x2a3c, 0x2a3d, 0x2a3e, 0x2a3f, + }; + +static const unsigned short gNormalizeTable2a40[] = { + /* U+2a40 */ + 0x2a40, 0x2a41, 0x2a42, 0x2a43, 0x2a44, 0x2a45, 0x2a46, 0x2a47, + 0x2a48, 0x2a49, 0x2a4a, 0x2a4b, 0x2a4c, 0x2a4d, 0x2a4e, 0x2a4f, + 0x2a50, 0x2a51, 0x2a52, 0x2a53, 0x2a54, 0x2a55, 0x2a56, 0x2a57, + 0x2a58, 0x2a59, 0x2a5a, 0x2a5b, 0x2a5c, 0x2a5d, 0x2a5e, 0x2a5f, + 0x2a60, 0x2a61, 0x2a62, 0x2a63, 0x2a64, 0x2a65, 0x2a66, 0x2a67, + 0x2a68, 0x2a69, 0x2a6a, 0x2a6b, 0x2a6c, 0x2a6d, 0x2a6e, 0x2a6f, + 0x2a70, 0x2a71, 0x2a72, 0x2a73, 0x003a, 0x003d, 0x003d, 0x2a77, + 0x2a78, 0x2a79, 0x2a7a, 0x2a7b, 0x2a7c, 0x2a7d, 0x2a7e, 0x2a7f, + }; + +static const unsigned short gNormalizeTable2ac0[] = { + /* U+2ac0 */ + 0x2ac0, 0x2ac1, 0x2ac2, 0x2ac3, 0x2ac4, 0x2ac5, 0x2ac6, 0x2ac7, + 0x2ac8, 0x2ac9, 0x2aca, 0x2acb, 0x2acc, 0x2acd, 0x2ace, 0x2acf, + 0x2ad0, 0x2ad1, 0x2ad2, 0x2ad3, 0x2ad4, 0x2ad5, 0x2ad6, 0x2ad7, + 0x2ad8, 0x2ad9, 0x2ada, 0x2adb, 0x2add, 0x2add, 0x2ade, 0x2adf, + 0x2ae0, 0x2ae1, 0x2ae2, 0x2ae3, 0x2ae4, 0x2ae5, 0x2ae6, 0x2ae7, + 0x2ae8, 0x2ae9, 0x2aea, 0x2aeb, 0x2aec, 0x2aed, 0x2aee, 0x2aef, + 0x2af0, 0x2af1, 0x2af2, 0x2af3, 0x2af4, 0x2af5, 0x2af6, 0x2af7, + 0x2af8, 0x2af9, 0x2afa, 0x2afb, 0x2afc, 0x2afd, 0x2afe, 0x2aff, + }; + +static const unsigned short gNormalizeTable2c00[] = { + /* U+2c00 */ + 0x2c30, 0x2c31, 0x2c32, 0x2c33, 0x2c34, 0x2c35, 0x2c36, 0x2c37, + 0x2c38, 0x2c39, 0x2c3a, 0x2c3b, 0x2c3c, 0x2c3d, 0x2c3e, 0x2c3f, + 0x2c40, 0x2c41, 0x2c42, 0x2c43, 0x2c44, 0x2c45, 0x2c46, 0x2c47, + 0x2c48, 0x2c49, 0x2c4a, 0x2c4b, 0x2c4c, 0x2c4d, 0x2c4e, 0x2c4f, + 0x2c50, 0x2c51, 0x2c52, 0x2c53, 0x2c54, 0x2c55, 0x2c56, 0x2c57, + 0x2c58, 0x2c59, 0x2c5a, 0x2c5b, 0x2c5c, 0x2c5d, 0x2c5e, 0x2c2f, + 0x2c30, 0x2c31, 0x2c32, 0x2c33, 0x2c34, 0x2c35, 0x2c36, 0x2c37, + 0x2c38, 0x2c39, 0x2c3a, 0x2c3b, 0x2c3c, 0x2c3d, 0x2c3e, 0x2c3f, + }; + +static const unsigned short gNormalizeTable2c40[] = { + /* U+2c40 */ + 0x2c40, 0x2c41, 0x2c42, 0x2c43, 0x2c44, 0x2c45, 0x2c46, 0x2c47, + 0x2c48, 0x2c49, 0x2c4a, 0x2c4b, 0x2c4c, 0x2c4d, 0x2c4e, 0x2c4f, + 0x2c50, 0x2c51, 0x2c52, 0x2c53, 0x2c54, 0x2c55, 0x2c56, 0x2c57, + 0x2c58, 0x2c59, 0x2c5a, 0x2c5b, 0x2c5c, 0x2c5d, 0x2c5e, 0x2c5f, + 0x2c61, 0x2c61, 0x026b, 0x1d7d, 0x027d, 0x2c65, 0x2c66, 0x2c68, + 0x2c68, 0x2c6a, 0x2c6a, 0x2c6c, 0x2c6c, 0x0251, 0x0271, 0x0250, + 0x0252, 0x2c71, 0x2c73, 0x2c73, 0x2c74, 0x2c76, 0x2c76, 0x2c77, + 0x2c78, 0x2c79, 0x2c7a, 0x2c7b, 0x006a, 0x0076, 0x023f, 0x0240, + }; + +static const unsigned short gNormalizeTable2c80[] = { + /* U+2c80 */ + 0x2c81, 0x2c81, 0x2c83, 0x2c83, 0x2c85, 0x2c85, 0x2c87, 0x2c87, + 0x2c89, 0x2c89, 0x2c8b, 0x2c8b, 0x2c8d, 0x2c8d, 0x2c8f, 0x2c8f, + 0x2c91, 0x2c91, 0x2c93, 0x2c93, 0x2c95, 0x2c95, 0x2c97, 0x2c97, + 0x2c99, 0x2c99, 0x2c9b, 0x2c9b, 0x2c9d, 0x2c9d, 0x2c9f, 0x2c9f, + 0x2ca1, 0x2ca1, 0x2ca3, 0x2ca3, 0x2ca5, 0x2ca5, 0x2ca7, 0x2ca7, + 0x2ca9, 0x2ca9, 0x2cab, 0x2cab, 0x2cad, 0x2cad, 0x2caf, 0x2caf, + 0x2cb1, 0x2cb1, 0x2cb3, 0x2cb3, 0x2cb5, 0x2cb5, 0x2cb7, 0x2cb7, + 0x2cb9, 0x2cb9, 0x2cbb, 0x2cbb, 0x2cbd, 0x2cbd, 0x2cbf, 0x2cbf, + }; + +static const unsigned short gNormalizeTable2cc0[] = { + /* U+2cc0 */ + 0x2cc1, 0x2cc1, 0x2cc3, 0x2cc3, 0x2cc5, 0x2cc5, 0x2cc7, 0x2cc7, + 0x2cc9, 0x2cc9, 0x2ccb, 0x2ccb, 0x2ccd, 0x2ccd, 0x2ccf, 0x2ccf, + 0x2cd1, 0x2cd1, 0x2cd3, 0x2cd3, 0x2cd5, 0x2cd5, 0x2cd7, 0x2cd7, + 0x2cd9, 0x2cd9, 0x2cdb, 0x2cdb, 0x2cdd, 0x2cdd, 0x2cdf, 0x2cdf, + 0x2ce1, 0x2ce1, 0x2ce3, 0x2ce3, 0x2ce4, 0x2ce5, 0x2ce6, 0x2ce7, + 0x2ce8, 0x2ce9, 0x2cea, 0x2cec, 0x2cec, 0x2cee, 0x2cee, 0x2cef, + 0x2cf0, 0x2cf1, 0x2cf2, 0x2cf3, 0x2cf4, 0x2cf5, 0x2cf6, 0x2cf7, + 0x2cf8, 0x2cf9, 0x2cfa, 0x2cfb, 0x2cfc, 0x2cfd, 0x2cfe, 0x2cff, + }; + +static const unsigned short gNormalizeTable2d40[] = { + /* U+2d40 */ + 0x2d40, 0x2d41, 0x2d42, 0x2d43, 0x2d44, 0x2d45, 0x2d46, 0x2d47, + 0x2d48, 0x2d49, 0x2d4a, 0x2d4b, 0x2d4c, 0x2d4d, 0x2d4e, 0x2d4f, + 0x2d50, 0x2d51, 0x2d52, 0x2d53, 0x2d54, 0x2d55, 0x2d56, 0x2d57, + 0x2d58, 0x2d59, 0x2d5a, 0x2d5b, 0x2d5c, 0x2d5d, 0x2d5e, 0x2d5f, + 0x2d60, 0x2d61, 0x2d62, 0x2d63, 0x2d64, 0x2d65, 0x2d66, 0x2d67, + 0x2d68, 0x2d69, 0x2d6a, 0x2d6b, 0x2d6c, 0x2d6d, 0x2d6e, 0x2d61, + 0x2d70, 0x2d71, 0x2d72, 0x2d73, 0x2d74, 0x2d75, 0x2d76, 0x2d77, + 0x2d78, 0x2d79, 0x2d7a, 0x2d7b, 0x2d7c, 0x2d7d, 0x2d7e, 0x2d7f, + }; + +static const unsigned short gNormalizeTable2e80[] = { + /* U+2e80 */ + 0x2e80, 0x2e81, 0x2e82, 0x2e83, 0x2e84, 0x2e85, 0x2e86, 0x2e87, + 0x2e88, 0x2e89, 0x2e8a, 0x2e8b, 0x2e8c, 0x2e8d, 0x2e8e, 0x2e8f, + 0x2e90, 0x2e91, 0x2e92, 0x2e93, 0x2e94, 0x2e95, 0x2e96, 0x2e97, + 0x2e98, 0x2e99, 0x2e9a, 0x2e9b, 0x2e9c, 0x2e9d, 0x2e9e, 0x6bcd, + 0x2ea0, 0x2ea1, 0x2ea2, 0x2ea3, 0x2ea4, 0x2ea5, 0x2ea6, 0x2ea7, + 0x2ea8, 0x2ea9, 0x2eaa, 0x2eab, 0x2eac, 0x2ead, 0x2eae, 0x2eaf, + 0x2eb0, 0x2eb1, 0x2eb2, 0x2eb3, 0x2eb4, 0x2eb5, 0x2eb6, 0x2eb7, + 0x2eb8, 0x2eb9, 0x2eba, 0x2ebb, 0x2ebc, 0x2ebd, 0x2ebe, 0x2ebf, + }; + +static const unsigned short gNormalizeTable2ec0[] = { + /* U+2ec0 */ + 0x2ec0, 0x2ec1, 0x2ec2, 0x2ec3, 0x2ec4, 0x2ec5, 0x2ec6, 0x2ec7, + 0x2ec8, 0x2ec9, 0x2eca, 0x2ecb, 0x2ecc, 0x2ecd, 0x2ece, 0x2ecf, + 0x2ed0, 0x2ed1, 0x2ed2, 0x2ed3, 0x2ed4, 0x2ed5, 0x2ed6, 0x2ed7, + 0x2ed8, 0x2ed9, 0x2eda, 0x2edb, 0x2edc, 0x2edd, 0x2ede, 0x2edf, + 0x2ee0, 0x2ee1, 0x2ee2, 0x2ee3, 0x2ee4, 0x2ee5, 0x2ee6, 0x2ee7, + 0x2ee8, 0x2ee9, 0x2eea, 0x2eeb, 0x2eec, 0x2eed, 0x2eee, 0x2eef, + 0x2ef0, 0x2ef1, 0x2ef2, 0x9f9f, 0x2ef4, 0x2ef5, 0x2ef6, 0x2ef7, + 0x2ef8, 0x2ef9, 0x2efa, 0x2efb, 0x2efc, 0x2efd, 0x2efe, 0x2eff, + }; + +static const unsigned short gNormalizeTable2f00[] = { + /* U+2f00 */ + 0x4e00, 0x4e28, 0x4e36, 0x4e3f, 0x4e59, 0x4e85, 0x4e8c, 0x4ea0, + 0x4eba, 0x513f, 0x5165, 0x516b, 0x5182, 0x5196, 0x51ab, 0x51e0, + 0x51f5, 0x5200, 0x529b, 0x52f9, 0x5315, 0x531a, 0x5338, 0x5341, + 0x535c, 0x5369, 0x5382, 0x53b6, 0x53c8, 0x53e3, 0x56d7, 0x571f, + 0x58eb, 0x5902, 0x590a, 0x5915, 0x5927, 0x5973, 0x5b50, 0x5b80, + 0x5bf8, 0x5c0f, 0x5c22, 0x5c38, 0x5c6e, 0x5c71, 0x5ddb, 0x5de5, + 0x5df1, 0x5dfe, 0x5e72, 0x5e7a, 0x5e7f, 0x5ef4, 0x5efe, 0x5f0b, + 0x5f13, 0x5f50, 0x5f61, 0x5f73, 0x5fc3, 0x6208, 0x6236, 0x624b, + }; + +static const unsigned short gNormalizeTable2f40[] = { + /* U+2f40 */ + 0x652f, 0x6534, 0x6587, 0x6597, 0x65a4, 0x65b9, 0x65e0, 0x65e5, + 0x66f0, 0x6708, 0x6728, 0x6b20, 0x6b62, 0x6b79, 0x6bb3, 0x6bcb, + 0x6bd4, 0x6bdb, 0x6c0f, 0x6c14, 0x6c34, 0x706b, 0x722a, 0x7236, + 0x723b, 0x723f, 0x7247, 0x7259, 0x725b, 0x72ac, 0x7384, 0x7389, + 0x74dc, 0x74e6, 0x7518, 0x751f, 0x7528, 0x7530, 0x758b, 0x7592, + 0x7676, 0x767d, 0x76ae, 0x76bf, 0x76ee, 0x77db, 0x77e2, 0x77f3, + 0x793a, 0x79b8, 0x79be, 0x7a74, 0x7acb, 0x7af9, 0x7c73, 0x7cf8, + 0x7f36, 0x7f51, 0x7f8a, 0x7fbd, 0x8001, 0x800c, 0x8012, 0x8033, + }; + +static const unsigned short gNormalizeTable2f80[] = { + /* U+2f80 */ + 0x807f, 0x8089, 0x81e3, 0x81ea, 0x81f3, 0x81fc, 0x820c, 0x821b, + 0x821f, 0x826e, 0x8272, 0x8278, 0x864d, 0x866b, 0x8840, 0x884c, + 0x8863, 0x897e, 0x898b, 0x89d2, 0x8a00, 0x8c37, 0x8c46, 0x8c55, + 0x8c78, 0x8c9d, 0x8d64, 0x8d70, 0x8db3, 0x8eab, 0x8eca, 0x8f9b, + 0x8fb0, 0x8fb5, 0x9091, 0x9149, 0x91c6, 0x91cc, 0x91d1, 0x9577, + 0x9580, 0x961c, 0x96b6, 0x96b9, 0x96e8, 0x9751, 0x975e, 0x9762, + 0x9769, 0x97cb, 0x97ed, 0x97f3, 0x9801, 0x98a8, 0x98db, 0x98df, + 0x9996, 0x9999, 0x99ac, 0x9aa8, 0x9ad8, 0x9adf, 0x9b25, 0x9b2f, + }; + +static const unsigned short gNormalizeTable2fc0[] = { + /* U+2fc0 */ + 0x9b32, 0x9b3c, 0x9b5a, 0x9ce5, 0x9e75, 0x9e7f, 0x9ea5, 0x9ebb, + 0x9ec3, 0x9ecd, 0x9ed1, 0x9ef9, 0x9efd, 0x9f0e, 0x9f13, 0x9f20, + 0x9f3b, 0x9f4a, 0x9f52, 0x9f8d, 0x9f9c, 0x9fa0, 0x2fd6, 0x2fd7, + 0x2fd8, 0x2fd9, 0x2fda, 0x2fdb, 0x2fdc, 0x2fdd, 0x2fde, 0x2fdf, + 0x2fe0, 0x2fe1, 0x2fe2, 0x2fe3, 0x2fe4, 0x2fe5, 0x2fe6, 0x2fe7, + 0x2fe8, 0x2fe9, 0x2fea, 0x2feb, 0x2fec, 0x2fed, 0x2fee, 0x2fef, + 0x2ff0, 0x2ff1, 0x2ff2, 0x2ff3, 0x2ff4, 0x2ff5, 0x2ff6, 0x2ff7, + 0x2ff8, 0x2ff9, 0x2ffa, 0x2ffb, 0x2ffc, 0x2ffd, 0x2ffe, 0x2fff, + }; + +static const unsigned short gNormalizeTable3000[] = { + /* U+3000 */ + 0x0020, 0x3001, 0x3002, 0x3003, 0x3004, 0x3005, 0x3006, 0x3007, + 0x3008, 0x3009, 0x300a, 0x300b, 0x300c, 0x300d, 0x300e, 0x300f, + 0x3010, 0x3011, 0x3012, 0x3013, 0x3014, 0x3015, 0x3016, 0x3017, + 0x3018, 0x3019, 0x301a, 0x301b, 0x301c, 0x301d, 0x301e, 0x301f, + 0x3020, 0x3021, 0x3022, 0x3023, 0x3024, 0x3025, 0x3026, 0x3027, + 0x3028, 0x3029, 0x302a, 0x302b, 0x302c, 0x302d, 0x302e, 0x302f, + 0x3030, 0x3031, 0x3032, 0x3033, 0x3034, 0x3035, 0x3012, 0x3037, + 0x5341, 0x5344, 0x5345, 0x303b, 0x303c, 0x303d, 0x303e, 0x303f, + }; + +static const unsigned short gNormalizeTable3040[] = { + /* U+3040 */ + 0x3040, 0x3041, 0x3042, 0x3043, 0x3044, 0x3045, 0x3046, 0x3047, + 0x3048, 0x3049, 0x304a, 0x304b, 0x304b, 0x304d, 0x304d, 0x304f, + 0x304f, 0x3051, 0x3051, 0x3053, 0x3053, 0x3055, 0x3055, 0x3057, + 0x3057, 0x3059, 0x3059, 0x305b, 0x305b, 0x305d, 0x305d, 0x305f, + 0x305f, 0x3061, 0x3061, 0x3063, 0x3064, 0x3064, 0x3066, 0x3066, + 0x3068, 0x3068, 0x306a, 0x306b, 0x306c, 0x306d, 0x306e, 0x306f, + 0x306f, 0x306f, 0x3072, 0x3072, 0x3072, 0x3075, 0x3075, 0x3075, + 0x3078, 0x3078, 0x3078, 0x307b, 0x307b, 0x307b, 0x307e, 0x307f, + }; + +static const unsigned short gNormalizeTable3080[] = { + /* U+3080 */ + 0x3080, 0x3081, 0x3082, 0x3083, 0x3084, 0x3085, 0x3086, 0x3087, + 0x3088, 0x3089, 0x308a, 0x308b, 0x308c, 0x308d, 0x308e, 0x308f, + 0x3090, 0x3091, 0x3092, 0x3093, 0x3046, 0x3095, 0x3096, 0x3097, + 0x3098, 0x3099, 0x309a, 0x0020, 0x0020, 0x309d, 0x309d, 0x3088, + 0x30a0, 0x30a1, 0x30a2, 0x30a3, 0x30a4, 0x30a5, 0x30a6, 0x30a7, + 0x30a8, 0x30a9, 0x30aa, 0x30ab, 0x30ab, 0x30ad, 0x30ad, 0x30af, + 0x30af, 0x30b1, 0x30b1, 0x30b3, 0x30b3, 0x30b5, 0x30b5, 0x30b7, + 0x30b7, 0x30b9, 0x30b9, 0x30bb, 0x30bb, 0x30bd, 0x30bd, 0x30bf, + }; + +static const unsigned short gNormalizeTable30c0[] = { + /* U+30c0 */ + 0x30bf, 0x30c1, 0x30c1, 0x30c3, 0x30c4, 0x30c4, 0x30c6, 0x30c6, + 0x30c8, 0x30c8, 0x30ca, 0x30cb, 0x30cc, 0x30cd, 0x30ce, 0x30cf, + 0x30cf, 0x30cf, 0x30d2, 0x30d2, 0x30d2, 0x30d5, 0x30d5, 0x30d5, + 0x30d8, 0x30d8, 0x30d8, 0x30db, 0x30db, 0x30db, 0x30de, 0x30df, + 0x30e0, 0x30e1, 0x30e2, 0x30e3, 0x30e4, 0x30e5, 0x30e6, 0x30e7, + 0x30e8, 0x30e9, 0x30ea, 0x30eb, 0x30ec, 0x30ed, 0x30ee, 0x30ef, + 0x30f0, 0x30f1, 0x30f2, 0x30f3, 0x30a6, 0x30f5, 0x30f6, 0x30ef, + 0x30f0, 0x30f1, 0x30f2, 0x30fb, 0x30fc, 0x30fd, 0x30fd, 0x30b3, + }; + +static const unsigned short gNormalizeTable3100[] = { + /* U+3100 */ + 0x3100, 0x3101, 0x3102, 0x3103, 0x3104, 0x3105, 0x3106, 0x3107, + 0x3108, 0x3109, 0x310a, 0x310b, 0x310c, 0x310d, 0x310e, 0x310f, + 0x3110, 0x3111, 0x3112, 0x3113, 0x3114, 0x3115, 0x3116, 0x3117, + 0x3118, 0x3119, 0x311a, 0x311b, 0x311c, 0x311d, 0x311e, 0x311f, + 0x3120, 0x3121, 0x3122, 0x3123, 0x3124, 0x3125, 0x3126, 0x3127, + 0x3128, 0x3129, 0x312a, 0x312b, 0x312c, 0x312d, 0x312e, 0x312f, + 0x3130, 0x1100, 0x1101, 0x11aa, 0x1102, 0x11ac, 0x11ad, 0x1103, + 0x1104, 0x1105, 0x11b0, 0x11b1, 0x11b2, 0x11b3, 0x11b4, 0x11b5, + }; + +static const unsigned short gNormalizeTable3140[] = { + /* U+3140 */ + 0x111a, 0x1106, 0x1107, 0x1108, 0x1121, 0x1109, 0x110a, 0x110b, + 0x110c, 0x110d, 0x110e, 0x110f, 0x1110, 0x1111, 0x1112, 0x1161, + 0x1162, 0x1163, 0x1164, 0x1165, 0x1166, 0x1167, 0x1168, 0x1169, + 0x116a, 0x116b, 0x116c, 0x116d, 0x116e, 0x116f, 0x1170, 0x1171, + 0x1172, 0x1173, 0x1174, 0x1175, 0x0020, 0x1114, 0x1115, 0x11c7, + 0x11c8, 0x11cc, 0x11ce, 0x11d3, 0x11d7, 0x11d9, 0x111c, 0x11dd, + 0x11df, 0x111d, 0x111e, 0x1120, 0x1122, 0x1123, 0x1127, 0x1129, + 0x112b, 0x112c, 0x112d, 0x112e, 0x112f, 0x1132, 0x1136, 0x1140, + }; + +static const unsigned short gNormalizeTable3180[] = { + /* U+3180 */ + 0x1147, 0x114c, 0x11f1, 0x11f2, 0x1157, 0x1158, 0x1159, 0x1184, + 0x1185, 0x1188, 0x1191, 0x1192, 0x1194, 0x119e, 0x11a1, 0x318f, + 0x3190, 0x3191, 0x4e00, 0x4e8c, 0x4e09, 0x56db, 0x4e0a, 0x4e2d, + 0x4e0b, 0x7532, 0x4e59, 0x4e19, 0x4e01, 0x5929, 0x5730, 0x4eba, + 0x31a0, 0x31a1, 0x31a2, 0x31a3, 0x31a4, 0x31a5, 0x31a6, 0x31a7, + 0x31a8, 0x31a9, 0x31aa, 0x31ab, 0x31ac, 0x31ad, 0x31ae, 0x31af, + 0x31b0, 0x31b1, 0x31b2, 0x31b3, 0x31b4, 0x31b5, 0x31b6, 0x31b7, + 0x31b8, 0x31b9, 0x31ba, 0x31bb, 0x31bc, 0x31bd, 0x31be, 0x31bf, + }; + +static const unsigned short gNormalizeTable3200[] = { + /* U+3200 */ + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x321f, + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, + }; + +static const unsigned short gNormalizeTable3240[] = { + /* U+3240 */ + 0x0028, 0x0028, 0x0028, 0x0028, 0x554f, 0x5e7c, 0x6587, 0x7b8f, + 0x3248, 0x3249, 0x324a, 0x324b, 0x324c, 0x324d, 0x324e, 0x324f, + 0x0070, 0x0032, 0x0032, 0x0032, 0x0032, 0x0032, 0x0032, 0x0032, + 0x0032, 0x0032, 0x0033, 0x0033, 0x0033, 0x0033, 0x0033, 0x0033, + 0x1100, 0x1102, 0x1103, 0x1105, 0x1106, 0x1107, 0x1109, 0x110b, + 0x110c, 0x110e, 0x110f, 0x1110, 0x1111, 0x1112, 0x1100, 0x1102, + 0x1103, 0x1105, 0x1106, 0x1107, 0x1109, 0x110b, 0x110c, 0x110e, + 0x110f, 0x1110, 0x1111, 0x1112, 0x110e, 0x110c, 0x110b, 0x327f, + }; + +static const unsigned short gNormalizeTable3280[] = { + /* U+3280 */ + 0x4e00, 0x4e8c, 0x4e09, 0x56db, 0x4e94, 0x516d, 0x4e03, 0x516b, + 0x4e5d, 0x5341, 0x6708, 0x706b, 0x6c34, 0x6728, 0x91d1, 0x571f, + 0x65e5, 0x682a, 0x6709, 0x793e, 0x540d, 0x7279, 0x8ca1, 0x795d, + 0x52b4, 0x79d8, 0x7537, 0x5973, 0x9069, 0x512a, 0x5370, 0x6ce8, + 0x9805, 0x4f11, 0x5199, 0x6b63, 0x4e0a, 0x4e2d, 0x4e0b, 0x5de6, + 0x53f3, 0x533b, 0x5b97, 0x5b66, 0x76e3, 0x4f01, 0x8cc7, 0x5354, + 0x591c, 0x0033, 0x0033, 0x0033, 0x0033, 0x0034, 0x0034, 0x0034, + 0x0034, 0x0034, 0x0034, 0x0034, 0x0034, 0x0034, 0x0034, 0x0035, + }; + +static const unsigned short gNormalizeTable32c0[] = { + /* U+32c0 */ + 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, + 0x0039, 0x0031, 0x0031, 0x0031, 0x0068, 0x0065, 0x0065, 0x006c, + 0x30a2, 0x30a4, 0x30a6, 0x30a8, 0x30aa, 0x30ab, 0x30ad, 0x30af, + 0x30b1, 0x30b3, 0x30b5, 0x30b7, 0x30b9, 0x30bb, 0x30bd, 0x30bf, + 0x30c1, 0x30c4, 0x30c6, 0x30c8, 0x30ca, 0x30cb, 0x30cc, 0x30cd, + 0x30ce, 0x30cf, 0x30d2, 0x30d5, 0x30d8, 0x30db, 0x30de, 0x30df, + 0x30e0, 0x30e1, 0x30e2, 0x30e4, 0x30e6, 0x30e8, 0x30e9, 0x30ea, + 0x30eb, 0x30ec, 0x30ed, 0x30ef, 0x30f0, 0x30f1, 0x30f2, 0x32ff, + }; + +static const unsigned short gNormalizeTable3300[] = { + /* U+3300 */ + 0x30a2, 0x30a2, 0x30a2, 0x30a2, 0x30a4, 0x30a4, 0x30a6, 0x30a8, + 0x30a8, 0x30aa, 0x30aa, 0x30ab, 0x30ab, 0x30ab, 0x30ab, 0x30ab, + 0x30ad, 0x30ad, 0x30ad, 0x30ad, 0x30ad, 0x30ad, 0x30ad, 0x30ad, + 0x30af, 0x30af, 0x30af, 0x30af, 0x30b1, 0x30b3, 0x30b3, 0x30b5, + 0x30b5, 0x30b7, 0x30bb, 0x30bb, 0x30bf, 0x30c6, 0x30c8, 0x30c8, + 0x30ca, 0x30ce, 0x30cf, 0x30cf, 0x30cf, 0x30cf, 0x30d2, 0x30d2, + 0x30d2, 0x30d2, 0x30d5, 0x30d5, 0x30d5, 0x30d5, 0x30d8, 0x30d8, + 0x30d8, 0x30d8, 0x30d8, 0x30d8, 0x30d8, 0x30db, 0x30db, 0x30db, + }; + +static const unsigned short gNormalizeTable3340[] = { + /* U+3340 */ + 0x30db, 0x30db, 0x30db, 0x30de, 0x30de, 0x30de, 0x30de, 0x30de, + 0x30df, 0x30df, 0x30df, 0x30e1, 0x30e1, 0x30e1, 0x30e4, 0x30e4, + 0x30e6, 0x30ea, 0x30ea, 0x30eb, 0x30eb, 0x30ec, 0x30ec, 0x30ef, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, + 0x0031, 0x0031, 0x0031, 0x0031, 0x0032, 0x0032, 0x0032, 0x0032, + 0x0032, 0x0068, 0x0064, 0x0061, 0x0062, 0x006f, 0x0070, 0x0064, + 0x0064, 0x0064, 0x0069, 0x5e73, 0x662d, 0x5927, 0x660e, 0x682a, + }; + +static const unsigned short gNormalizeTable3380[] = { + /* U+3380 */ + 0x0070, 0x006e, 0x03bc, 0x006d, 0x006b, 0x006b, 0x006d, 0x0067, + 0x0063, 0x006b, 0x0070, 0x006e, 0x03bc, 0x03bc, 0x006d, 0x006b, + 0x0068, 0x006b, 0x006d, 0x0067, 0x0074, 0x03bc, 0x006d, 0x0064, + 0x006b, 0x0066, 0x006e, 0x03bc, 0x006d, 0x0063, 0x006b, 0x006d, + 0x0063, 0x006d, 0x006b, 0x006d, 0x0063, 0x006d, 0x006b, 0x006d, + 0x006d, 0x0070, 0x006b, 0x006d, 0x0067, 0x0072, 0x0072, 0x0072, + 0x0070, 0x006e, 0x03bc, 0x006d, 0x0070, 0x006e, 0x03bc, 0x006d, + 0x006b, 0x006d, 0x0070, 0x006e, 0x03bc, 0x006d, 0x006b, 0x006d, + }; + +static const unsigned short gNormalizeTable33c0[] = { + /* U+33c0 */ + 0x006b, 0x006d, 0x0061, 0x0062, 0x0063, 0x0063, 0x0063, 0x0063, + 0x0064, 0x0067, 0x0068, 0x0068, 0x0069, 0x006b, 0x006b, 0x006b, + 0x006c, 0x006c, 0x006c, 0x006c, 0x006d, 0x006d, 0x006d, 0x0070, + 0x0070, 0x0070, 0x0070, 0x0073, 0x0073, 0x0077, 0x0076, 0x0061, + 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, + 0x0039, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, + 0x0031, 0x0031, 0x0031, 0x0032, 0x0032, 0x0032, 0x0032, 0x0032, + 0x0032, 0x0032, 0x0032, 0x0032, 0x0032, 0x0033, 0x0033, 0x0067, + }; + +static const unsigned short gNormalizeTablea640[] = { + /* U+a640 */ + 0xa641, 0xa641, 0xa643, 0xa643, 0xa645, 0xa645, 0xa647, 0xa647, + 0xa649, 0xa649, 0xa64b, 0xa64b, 0xa64d, 0xa64d, 0xa64f, 0xa64f, + 0xa651, 0xa651, 0xa653, 0xa653, 0xa655, 0xa655, 0xa657, 0xa657, + 0xa659, 0xa659, 0xa65b, 0xa65b, 0xa65d, 0xa65d, 0xa65f, 0xa65f, + 0xa660, 0xa661, 0xa663, 0xa663, 0xa665, 0xa665, 0xa667, 0xa667, + 0xa669, 0xa669, 0xa66b, 0xa66b, 0xa66d, 0xa66d, 0xa66e, 0xa66f, + 0xa670, 0xa671, 0xa672, 0xa673, 0xa674, 0xa675, 0xa676, 0xa677, + 0xa678, 0xa679, 0xa67a, 0xa67b, 0xa67c, 0xa67d, 0xa67e, 0xa67f, + }; + +static const unsigned short gNormalizeTablea680[] = { + /* U+a680 */ + 0xa681, 0xa681, 0xa683, 0xa683, 0xa685, 0xa685, 0xa687, 0xa687, + 0xa689, 0xa689, 0xa68b, 0xa68b, 0xa68d, 0xa68d, 0xa68f, 0xa68f, + 0xa691, 0xa691, 0xa693, 0xa693, 0xa695, 0xa695, 0xa697, 0xa697, + 0xa698, 0xa699, 0xa69a, 0xa69b, 0xa69c, 0xa69d, 0xa69e, 0xa69f, + 0xa6a0, 0xa6a1, 0xa6a2, 0xa6a3, 0xa6a4, 0xa6a5, 0xa6a6, 0xa6a7, + 0xa6a8, 0xa6a9, 0xa6aa, 0xa6ab, 0xa6ac, 0xa6ad, 0xa6ae, 0xa6af, + 0xa6b0, 0xa6b1, 0xa6b2, 0xa6b3, 0xa6b4, 0xa6b5, 0xa6b6, 0xa6b7, + 0xa6b8, 0xa6b9, 0xa6ba, 0xa6bb, 0xa6bc, 0xa6bd, 0xa6be, 0xa6bf, + }; + +static const unsigned short gNormalizeTablea700[] = { + /* U+a700 */ + 0xa700, 0xa701, 0xa702, 0xa703, 0xa704, 0xa705, 0xa706, 0xa707, + 0xa708, 0xa709, 0xa70a, 0xa70b, 0xa70c, 0xa70d, 0xa70e, 0xa70f, + 0xa710, 0xa711, 0xa712, 0xa713, 0xa714, 0xa715, 0xa716, 0xa717, + 0xa718, 0xa719, 0xa71a, 0xa71b, 0xa71c, 0xa71d, 0xa71e, 0xa71f, + 0xa720, 0xa721, 0xa723, 0xa723, 0xa725, 0xa725, 0xa727, 0xa727, + 0xa729, 0xa729, 0xa72b, 0xa72b, 0xa72d, 0xa72d, 0xa72f, 0xa72f, + 0xa730, 0xa731, 0xa733, 0xa733, 0xa735, 0xa735, 0xa737, 0xa737, + 0xa739, 0xa739, 0xa73b, 0xa73b, 0xa73d, 0xa73d, 0xa73f, 0xa73f, + }; + +static const unsigned short gNormalizeTablea740[] = { + /* U+a740 */ + 0xa741, 0xa741, 0xa743, 0xa743, 0xa745, 0xa745, 0xa747, 0xa747, + 0xa749, 0xa749, 0xa74b, 0xa74b, 0xa74d, 0xa74d, 0xa74f, 0xa74f, + 0xa751, 0xa751, 0xa753, 0xa753, 0xa755, 0xa755, 0xa757, 0xa757, + 0xa759, 0xa759, 0xa75b, 0xa75b, 0xa75d, 0xa75d, 0xa75f, 0xa75f, + 0xa761, 0xa761, 0xa763, 0xa763, 0xa765, 0xa765, 0xa767, 0xa767, + 0xa769, 0xa769, 0xa76b, 0xa76b, 0xa76d, 0xa76d, 0xa76f, 0xa76f, + 0xa76f, 0xa771, 0xa772, 0xa773, 0xa774, 0xa775, 0xa776, 0xa777, + 0xa778, 0xa77a, 0xa77a, 0xa77c, 0xa77c, 0x1d79, 0xa77f, 0xa77f, + }; + +static const unsigned short gNormalizeTablea780[] = { + /* U+a780 */ + 0xa781, 0xa781, 0xa783, 0xa783, 0xa785, 0xa785, 0xa787, 0xa787, + 0xa788, 0xa789, 0xa78a, 0xa78c, 0xa78c, 0xa78d, 0xa78e, 0xa78f, + 0xa790, 0xa791, 0xa792, 0xa793, 0xa794, 0xa795, 0xa796, 0xa797, + 0xa798, 0xa799, 0xa79a, 0xa79b, 0xa79c, 0xa79d, 0xa79e, 0xa79f, + 0xa7a0, 0xa7a1, 0xa7a2, 0xa7a3, 0xa7a4, 0xa7a5, 0xa7a6, 0xa7a7, + 0xa7a8, 0xa7a9, 0xa7aa, 0xa7ab, 0xa7ac, 0xa7ad, 0xa7ae, 0xa7af, + 0xa7b0, 0xa7b1, 0xa7b2, 0xa7b3, 0xa7b4, 0xa7b5, 0xa7b6, 0xa7b7, + 0xa7b8, 0xa7b9, 0xa7ba, 0xa7bb, 0xa7bc, 0xa7bd, 0xa7be, 0xa7bf, + }; + +static const unsigned short gNormalizeTablef900[] = { + /* U+f900 */ + 0x8c48, 0x66f4, 0x8eca, 0x8cc8, 0x6ed1, 0x4e32, 0x53e5, 0x9f9c, + 0x9f9c, 0x5951, 0x91d1, 0x5587, 0x5948, 0x61f6, 0x7669, 0x7f85, + 0x863f, 0x87ba, 0x88f8, 0x908f, 0x6a02, 0x6d1b, 0x70d9, 0x73de, + 0x843d, 0x916a, 0x99f1, 0x4e82, 0x5375, 0x6b04, 0x721b, 0x862d, + 0x9e1e, 0x5d50, 0x6feb, 0x85cd, 0x8964, 0x62c9, 0x81d8, 0x881f, + 0x5eca, 0x6717, 0x6d6a, 0x72fc, 0x90ce, 0x4f86, 0x51b7, 0x52de, + 0x64c4, 0x6ad3, 0x7210, 0x76e7, 0x8001, 0x8606, 0x865c, 0x8def, + 0x9732, 0x9b6f, 0x9dfa, 0x788c, 0x797f, 0x7da0, 0x83c9, 0x9304, + }; + +static const unsigned short gNormalizeTablef940[] = { + /* U+f940 */ + 0x9e7f, 0x8ad6, 0x58df, 0x5f04, 0x7c60, 0x807e, 0x7262, 0x78ca, + 0x8cc2, 0x96f7, 0x58d8, 0x5c62, 0x6a13, 0x6dda, 0x6f0f, 0x7d2f, + 0x7e37, 0x964b, 0x52d2, 0x808b, 0x51dc, 0x51cc, 0x7a1c, 0x7dbe, + 0x83f1, 0x9675, 0x8b80, 0x62cf, 0x6a02, 0x8afe, 0x4e39, 0x5be7, + 0x6012, 0x7387, 0x7570, 0x5317, 0x78fb, 0x4fbf, 0x5fa9, 0x4e0d, + 0x6ccc, 0x6578, 0x7d22, 0x53c3, 0x585e, 0x7701, 0x8449, 0x8aaa, + 0x6bba, 0x8fb0, 0x6c88, 0x62fe, 0x82e5, 0x63a0, 0x7565, 0x4eae, + 0x5169, 0x51c9, 0x6881, 0x7ce7, 0x826f, 0x8ad2, 0x91cf, 0x52f5, + }; + +static const unsigned short gNormalizeTablef980[] = { + /* U+f980 */ + 0x5442, 0x5973, 0x5eec, 0x65c5, 0x6ffe, 0x792a, 0x95ad, 0x9a6a, + 0x9e97, 0x9ece, 0x529b, 0x66c6, 0x6b77, 0x8f62, 0x5e74, 0x6190, + 0x6200, 0x649a, 0x6f23, 0x7149, 0x7489, 0x79ca, 0x7df4, 0x806f, + 0x8f26, 0x84ee, 0x9023, 0x934a, 0x5217, 0x52a3, 0x54bd, 0x70c8, + 0x88c2, 0x8aaa, 0x5ec9, 0x5ff5, 0x637b, 0x6bae, 0x7c3e, 0x7375, + 0x4ee4, 0x56f9, 0x5be7, 0x5dba, 0x601c, 0x73b2, 0x7469, 0x7f9a, + 0x8046, 0x9234, 0x96f6, 0x9748, 0x9818, 0x4f8b, 0x79ae, 0x91b4, + 0x96b8, 0x60e1, 0x4e86, 0x50da, 0x5bee, 0x5c3f, 0x6599, 0x6a02, + }; + +static const unsigned short gNormalizeTablef9c0[] = { + /* U+f9c0 */ + 0x71ce, 0x7642, 0x84fc, 0x907c, 0x9f8d, 0x6688, 0x962e, 0x5289, + 0x677b, 0x67f3, 0x6d41, 0x6e9c, 0x7409, 0x7559, 0x786b, 0x7d10, + 0x985e, 0x516d, 0x622e, 0x9678, 0x502b, 0x5d19, 0x6dea, 0x8f2a, + 0x5f8b, 0x6144, 0x6817, 0x7387, 0x9686, 0x5229, 0x540f, 0x5c65, + 0x6613, 0x674e, 0x68a8, 0x6ce5, 0x7406, 0x75e2, 0x7f79, 0x88cf, + 0x88e1, 0x91cc, 0x96e2, 0x533f, 0x6eba, 0x541d, 0x71d0, 0x7498, + 0x85fa, 0x96a3, 0x9c57, 0x9e9f, 0x6797, 0x6dcb, 0x81e8, 0x7acb, + 0x7b20, 0x7c92, 0x72c0, 0x7099, 0x8b58, 0x4ec0, 0x8336, 0x523a, + }; + +static const unsigned short gNormalizeTablefa00[] = { + /* U+fa00 */ + 0x5207, 0x5ea6, 0x62d3, 0x7cd6, 0x5b85, 0x6d1e, 0x66b4, 0x8f3b, + 0x884c, 0x964d, 0x898b, 0x5ed3, 0x5140, 0x55c0, 0xfa0e, 0xfa0f, + 0x585a, 0xfa11, 0x6674, 0xfa13, 0xfa14, 0x51de, 0x732a, 0x76ca, + 0x793c, 0x795e, 0x7965, 0x798f, 0x9756, 0x7cbe, 0x7fbd, 0xfa1f, + 0x8612, 0xfa21, 0x8af8, 0xfa23, 0xfa24, 0x9038, 0x90fd, 0xfa27, + 0xfa28, 0xfa29, 0x98ef, 0x98fc, 0x9928, 0x9db4, 0xfa2e, 0xfa2f, + 0x4fae, 0x50e7, 0x514d, 0x52c9, 0x52e4, 0x5351, 0x559d, 0x5606, + 0x5668, 0x5840, 0x58a8, 0x5c64, 0x5c6e, 0x6094, 0x6168, 0x618e, + }; + +static const unsigned short gNormalizeTablefa40[] = { + /* U+fa40 */ + 0x61f2, 0x654f, 0x65e2, 0x6691, 0x6885, 0x6d77, 0x6e1a, 0x6f22, + 0x716e, 0x722b, 0x7422, 0x7891, 0x793e, 0x7949, 0x7948, 0x7950, + 0x7956, 0x795d, 0x798d, 0x798e, 0x7a40, 0x7a81, 0x7bc0, 0x7df4, + 0x7e09, 0x7e41, 0x7f72, 0x8005, 0x81ed, 0x8279, 0x8279, 0x8457, + 0x8910, 0x8996, 0x8b01, 0x8b39, 0x8cd3, 0x8d08, 0x8fb6, 0x9038, + 0x96e3, 0x97ff, 0x983b, 0x6075, 0xfa6c, 0x8218, 0xfa6e, 0xfa6f, + 0x4e26, 0x51b5, 0x5168, 0x4f80, 0x5145, 0x5180, 0x52c7, 0x52fa, + 0x559d, 0x5555, 0x5599, 0x55e2, 0x585a, 0x58b3, 0x5944, 0x5954, + }; + +static const unsigned short gNormalizeTablefa80[] = { + /* U+fa80 */ + 0x5a62, 0x5b28, 0x5ed2, 0x5ed9, 0x5f69, 0x5fad, 0x60d8, 0x614e, + 0x6108, 0x618e, 0x6160, 0x61f2, 0x6234, 0x63c4, 0x641c, 0x6452, + 0x6556, 0x6674, 0x6717, 0x671b, 0x6756, 0x6b79, 0x6bba, 0x6d41, + 0x6edb, 0x6ecb, 0x6f22, 0x701e, 0x716e, 0x77a7, 0x7235, 0x72af, + 0x732a, 0x7471, 0x7506, 0x753b, 0x761d, 0x761f, 0x76ca, 0x76db, + 0x76f4, 0x774a, 0x7740, 0x78cc, 0x7ab1, 0x7bc0, 0x7c7b, 0x7d5b, + 0x7df4, 0x7f3e, 0x8005, 0x8352, 0x83ef, 0x8779, 0x8941, 0x8986, + 0x8996, 0x8abf, 0x8af8, 0x8acb, 0x8b01, 0x8afe, 0x8aed, 0x8b39, + }; + +static const unsigned short gNormalizeTablefac0[] = { + /* U+fac0 */ + 0x8b8a, 0x8d08, 0x8f38, 0x9072, 0x9199, 0x9276, 0x967c, 0x96e3, + 0x9756, 0x97db, 0x97ff, 0x980b, 0x983b, 0x9b12, 0x9f9c, 0xfacf, + 0xfad0, 0xfad1, 0x3b9d, 0x4018, 0x4039, 0xfad5, 0xfad6, 0xfad7, + 0x9f43, 0x9f8e, 0xfada, 0xfadb, 0xfadc, 0xfadd, 0xfade, 0xfadf, + 0xfae0, 0xfae1, 0xfae2, 0xfae3, 0xfae4, 0xfae5, 0xfae6, 0xfae7, + 0xfae8, 0xfae9, 0xfaea, 0xfaeb, 0xfaec, 0xfaed, 0xfaee, 0xfaef, + 0xfaf0, 0xfaf1, 0xfaf2, 0xfaf3, 0xfaf4, 0xfaf5, 0xfaf6, 0xfaf7, + 0xfaf8, 0xfaf9, 0xfafa, 0xfafb, 0xfafc, 0xfafd, 0xfafe, 0xfaff, + }; + +static const unsigned short gNormalizeTablefb00[] = { + /* U+fb00 */ + 0x0066, 0x0066, 0x0066, 0x0066, 0x0066, 0x0073, 0x0073, 0xfb07, + 0xfb08, 0xfb09, 0xfb0a, 0xfb0b, 0xfb0c, 0xfb0d, 0xfb0e, 0xfb0f, + 0xfb10, 0xfb11, 0xfb12, 0x0574, 0x0574, 0x0574, 0x057e, 0x0574, + 0xfb18, 0xfb19, 0xfb1a, 0xfb1b, 0xfb1c, 0x05d9, 0xfb1e, 0x05f2, + 0x05e2, 0x05d0, 0x05d3, 0x05d4, 0x05db, 0x05dc, 0x05dd, 0x05e8, + 0x05ea, 0x002b, 0x05e9, 0x05e9, 0x05e9, 0x05e9, 0x05d0, 0x05d0, + 0x05d0, 0x05d1, 0x05d2, 0x05d3, 0x05d4, 0x05d5, 0x05d6, 0xfb37, + 0x05d8, 0x05d9, 0x05da, 0x05db, 0x05dc, 0xfb3d, 0x05de, 0xfb3f, + }; + +static const unsigned short gNormalizeTablefb40[] = { + /* U+fb40 */ + 0x05e0, 0x05e1, 0xfb42, 0x05e3, 0x05e4, 0xfb45, 0x05e6, 0x05e7, + 0x05e8, 0x05e9, 0x05ea, 0x05d5, 0x05d1, 0x05db, 0x05e4, 0x05d0, + 0x0671, 0x0671, 0x067b, 0x067b, 0x067b, 0x067b, 0x067e, 0x067e, + 0x067e, 0x067e, 0x0680, 0x0680, 0x0680, 0x0680, 0x067a, 0x067a, + 0x067a, 0x067a, 0x067f, 0x067f, 0x067f, 0x067f, 0x0679, 0x0679, + 0x0679, 0x0679, 0x06a4, 0x06a4, 0x06a4, 0x06a4, 0x06a6, 0x06a6, + 0x06a6, 0x06a6, 0x0684, 0x0684, 0x0684, 0x0684, 0x0683, 0x0683, + 0x0683, 0x0683, 0x0686, 0x0686, 0x0686, 0x0686, 0x0687, 0x0687, + }; + +static const unsigned short gNormalizeTablefb80[] = { + /* U+fb80 */ + 0x0687, 0x0687, 0x068d, 0x068d, 0x068c, 0x068c, 0x068e, 0x068e, + 0x0688, 0x0688, 0x0698, 0x0698, 0x0691, 0x0691, 0x06a9, 0x06a9, + 0x06a9, 0x06a9, 0x06af, 0x06af, 0x06af, 0x06af, 0x06b3, 0x06b3, + 0x06b3, 0x06b3, 0x06b1, 0x06b1, 0x06b1, 0x06b1, 0x06ba, 0x06ba, + 0x06bb, 0x06bb, 0x06bb, 0x06bb, 0x06d5, 0x06d5, 0x06c1, 0x06c1, + 0x06c1, 0x06c1, 0x06be, 0x06be, 0x06be, 0x06be, 0x06d2, 0x06d2, + 0x06d2, 0x06d2, 0xfbb2, 0xfbb3, 0xfbb4, 0xfbb5, 0xfbb6, 0xfbb7, + 0xfbb8, 0xfbb9, 0xfbba, 0xfbbb, 0xfbbc, 0xfbbd, 0xfbbe, 0xfbbf, + }; + +static const unsigned short gNormalizeTablefbc0[] = { + /* U+fbc0 */ + 0xfbc0, 0xfbc1, 0xfbc2, 0xfbc3, 0xfbc4, 0xfbc5, 0xfbc6, 0xfbc7, + 0xfbc8, 0xfbc9, 0xfbca, 0xfbcb, 0xfbcc, 0xfbcd, 0xfbce, 0xfbcf, + 0xfbd0, 0xfbd1, 0xfbd2, 0x06ad, 0x06ad, 0x06ad, 0x06ad, 0x06c7, + 0x06c7, 0x06c6, 0x06c6, 0x06c8, 0x06c8, 0x06c7, 0x06cb, 0x06cb, + 0x06c5, 0x06c5, 0x06c9, 0x06c9, 0x06d0, 0x06d0, 0x06d0, 0x06d0, + 0x0649, 0x0649, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, + 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, + 0x064a, 0x064a, 0x064a, 0x064a, 0x06cc, 0x06cc, 0x06cc, 0x06cc, + }; + +static const unsigned short gNormalizeTablefc00[] = { + /* U+fc00 */ + 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, 0x0628, 0x0628, 0x0628, + 0x0628, 0x0628, 0x0628, 0x062a, 0x062a, 0x062a, 0x062a, 0x062a, + 0x062a, 0x062b, 0x062b, 0x062b, 0x062b, 0x062c, 0x062c, 0x062d, + 0x062d, 0x062e, 0x062e, 0x062e, 0x0633, 0x0633, 0x0633, 0x0633, + 0x0635, 0x0635, 0x0636, 0x0636, 0x0636, 0x0636, 0x0637, 0x0637, + 0x0638, 0x0639, 0x0639, 0x063a, 0x063a, 0x0641, 0x0641, 0x0641, + 0x0641, 0x0641, 0x0641, 0x0642, 0x0642, 0x0642, 0x0642, 0x0643, + 0x0643, 0x0643, 0x0643, 0x0643, 0x0643, 0x0643, 0x0643, 0x0644, + }; + +static const unsigned short gNormalizeTablefc40[] = { + /* U+fc40 */ + 0x0644, 0x0644, 0x0644, 0x0644, 0x0644, 0x0645, 0x0645, 0x0645, + 0x0645, 0x0645, 0x0645, 0x0646, 0x0646, 0x0646, 0x0646, 0x0646, + 0x0646, 0x0647, 0x0647, 0x0647, 0x0647, 0x064a, 0x064a, 0x064a, + 0x064a, 0x064a, 0x064a, 0x0630, 0x0631, 0x0649, 0x0020, 0x0020, + 0x0020, 0x0020, 0x0020, 0x0020, 0x064a, 0x064a, 0x064a, 0x064a, + 0x064a, 0x064a, 0x0628, 0x0628, 0x0628, 0x0628, 0x0628, 0x0628, + 0x062a, 0x062a, 0x062a, 0x062a, 0x062a, 0x062a, 0x062b, 0x062b, + 0x062b, 0x062b, 0x062b, 0x062b, 0x0641, 0x0641, 0x0642, 0x0642, + }; + +static const unsigned short gNormalizeTablefc80[] = { + /* U+fc80 */ + 0x0643, 0x0643, 0x0643, 0x0643, 0x0643, 0x0644, 0x0644, 0x0644, + 0x0645, 0x0645, 0x0646, 0x0646, 0x0646, 0x0646, 0x0646, 0x0646, + 0x0649, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, + 0x064a, 0x064a, 0x064a, 0x064a, 0x0628, 0x0628, 0x0628, 0x0628, + 0x0628, 0x062a, 0x062a, 0x062a, 0x062a, 0x062a, 0x062b, 0x062c, + 0x062c, 0x062d, 0x062d, 0x062e, 0x062e, 0x0633, 0x0633, 0x0633, + 0x0633, 0x0635, 0x0635, 0x0635, 0x0636, 0x0636, 0x0636, 0x0636, + 0x0637, 0x0638, 0x0639, 0x0639, 0x063a, 0x063a, 0x0641, 0x0641, + }; + +static const unsigned short gNormalizeTablefcc0[] = { + /* U+fcc0 */ + 0x0641, 0x0641, 0x0642, 0x0642, 0x0643, 0x0643, 0x0643, 0x0643, + 0x0643, 0x0644, 0x0644, 0x0644, 0x0644, 0x0644, 0x0645, 0x0645, + 0x0645, 0x0645, 0x0646, 0x0646, 0x0646, 0x0646, 0x0646, 0x0647, + 0x0647, 0x0647, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, + 0x064a, 0x0628, 0x0628, 0x062a, 0x062a, 0x062b, 0x062b, 0x0633, + 0x0633, 0x0634, 0x0634, 0x0643, 0x0643, 0x0644, 0x0646, 0x0646, + 0x064a, 0x064a, 0x0640, 0x0640, 0x0640, 0x0637, 0x0637, 0x0639, + 0x0639, 0x063a, 0x063a, 0x0633, 0x0633, 0x0634, 0x0634, 0x062d, + }; + +static const unsigned short gNormalizeTablefd00[] = { + /* U+fd00 */ + 0x062d, 0x062c, 0x062c, 0x062e, 0x062e, 0x0635, 0x0635, 0x0636, + 0x0636, 0x0634, 0x0634, 0x0634, 0x0634, 0x0634, 0x0633, 0x0635, + 0x0636, 0x0637, 0x0637, 0x0639, 0x0639, 0x063a, 0x063a, 0x0633, + 0x0633, 0x0634, 0x0634, 0x062d, 0x062d, 0x062c, 0x062c, 0x062e, + 0x062e, 0x0635, 0x0635, 0x0636, 0x0636, 0x0634, 0x0634, 0x0634, + 0x0634, 0x0634, 0x0633, 0x0635, 0x0636, 0x0634, 0x0634, 0x0634, + 0x0634, 0x0633, 0x0634, 0x0637, 0x0633, 0x0633, 0x0633, 0x0634, + 0x0634, 0x0634, 0x0637, 0x0638, 0x0627, 0x0627, 0xfd3e, 0xfd3f, + }; + +static const unsigned short gNormalizeTablefd40[] = { + /* U+fd40 */ + 0xfd40, 0xfd41, 0xfd42, 0xfd43, 0xfd44, 0xfd45, 0xfd46, 0xfd47, + 0xfd48, 0xfd49, 0xfd4a, 0xfd4b, 0xfd4c, 0xfd4d, 0xfd4e, 0xfd4f, + 0x062a, 0x062a, 0x062a, 0x062a, 0x062a, 0x062a, 0x062a, 0x062a, + 0x062c, 0x062c, 0x062d, 0x062d, 0x0633, 0x0633, 0x0633, 0x0633, + 0x0633, 0x0633, 0x0633, 0x0633, 0x0635, 0x0635, 0x0635, 0x0634, + 0x0634, 0x0634, 0x0634, 0x0634, 0x0634, 0x0634, 0x0636, 0x0636, + 0x0636, 0x0637, 0x0637, 0x0637, 0x0637, 0x0639, 0x0639, 0x0639, + 0x0639, 0x063a, 0x063a, 0x063a, 0x0641, 0x0641, 0x0642, 0x0642, + }; + +static const unsigned short gNormalizeTablefd80[] = { + /* U+fd80 */ + 0x0644, 0x0644, 0x0644, 0x0644, 0x0644, 0x0644, 0x0644, 0x0644, + 0x0644, 0x0645, 0x0645, 0x0645, 0x0645, 0x0645, 0x0645, 0x0645, + 0xfd90, 0xfd91, 0x0645, 0x0647, 0x0647, 0x0646, 0x0646, 0x0646, + 0x0646, 0x0646, 0x0646, 0x0646, 0x064a, 0x064a, 0x0628, 0x062a, + 0x062a, 0x062a, 0x062a, 0x062a, 0x062a, 0x062c, 0x062c, 0x062c, + 0x0633, 0x0635, 0x0634, 0x0636, 0x0644, 0x0644, 0x064a, 0x064a, + 0x064a, 0x0645, 0x0642, 0x0646, 0x0642, 0x0644, 0x0639, 0x0643, + 0x0646, 0x0645, 0x0644, 0x0643, 0x0644, 0x0646, 0x062c, 0x062d, + }; + +static const unsigned short gNormalizeTablefdc0[] = { + /* U+fdc0 */ + 0x0645, 0x0641, 0x0628, 0x0643, 0x0639, 0x0635, 0x0633, 0x0646, + 0xfdc8, 0xfdc9, 0xfdca, 0xfdcb, 0xfdcc, 0xfdcd, 0xfdce, 0xfdcf, + 0xfdd0, 0xfdd1, 0xfdd2, 0xfdd3, 0xfdd4, 0xfdd5, 0xfdd6, 0xfdd7, + 0xfdd8, 0xfdd9, 0xfdda, 0xfddb, 0xfddc, 0xfddd, 0xfdde, 0xfddf, + 0xfde0, 0xfde1, 0xfde2, 0xfde3, 0xfde4, 0xfde5, 0xfde6, 0xfde7, + 0xfde8, 0xfde9, 0xfdea, 0xfdeb, 0xfdec, 0xfded, 0xfdee, 0xfdef, + 0x0635, 0x0642, 0x0627, 0x0627, 0x0645, 0x0635, 0x0631, 0x0639, + 0x0648, 0x0635, 0x0635, 0x062c, 0x0631, 0xfdfd, 0xfdfe, 0xfdff, + }; + +static const unsigned short gNormalizeTablefe00[] = { + /* U+fe00 */ + 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, + 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, + 0x002c, 0x3001, 0x3002, 0x003a, 0x003b, 0x0021, 0x003f, 0x3016, + 0x3017, 0x002e, 0xfe1a, 0xfe1b, 0xfe1c, 0xfe1d, 0xfe1e, 0xfe1f, + 0xfe20, 0xfe21, 0xfe22, 0xfe23, 0xfe24, 0xfe25, 0xfe26, 0xfe27, + 0xfe28, 0xfe29, 0xfe2a, 0xfe2b, 0xfe2c, 0xfe2d, 0xfe2e, 0xfe2f, + 0x002e, 0x2014, 0x2013, 0x005f, 0x005f, 0x0028, 0x0029, 0x007b, + 0x007d, 0x3014, 0x3015, 0x3010, 0x3011, 0x300a, 0x300b, 0x3008, + }; + +static const unsigned short gNormalizeTablefe40[] = { + /* U+fe40 */ + 0x3009, 0x300c, 0x300d, 0x300e, 0x300f, 0xfe45, 0xfe46, 0x005b, + 0x005d, 0x0020, 0x0020, 0x0020, 0x0020, 0x005f, 0x005f, 0x005f, + 0x002c, 0x3001, 0x002e, 0xfe53, 0x003b, 0x003a, 0x003f, 0x0021, + 0x2014, 0x0028, 0x0029, 0x007b, 0x007d, 0x3014, 0x3015, 0x0023, + 0x0026, 0x002a, 0x002b, 0x002d, 0x003c, 0x003e, 0x003d, 0xfe67, + 0x005c, 0x0024, 0x0025, 0x0040, 0xfe6c, 0xfe6d, 0xfe6e, 0xfe6f, + 0x0020, 0x0640, 0x0020, 0xfe73, 0x0020, 0xfe75, 0x0020, 0x0640, + 0x0020, 0x0640, 0x0020, 0x0640, 0x0020, 0x0640, 0x0020, 0x0640, + }; + +static const unsigned short gNormalizeTablefe80[] = { + /* U+fe80 */ + 0x0621, 0x0627, 0x0627, 0x0627, 0x0627, 0x0648, 0x0648, 0x0627, + 0x0627, 0x064a, 0x064a, 0x064a, 0x064a, 0x0627, 0x0627, 0x0628, + 0x0628, 0x0628, 0x0628, 0x0629, 0x0629, 0x062a, 0x062a, 0x062a, + 0x062a, 0x062b, 0x062b, 0x062b, 0x062b, 0x062c, 0x062c, 0x062c, + 0x062c, 0x062d, 0x062d, 0x062d, 0x062d, 0x062e, 0x062e, 0x062e, + 0x062e, 0x062f, 0x062f, 0x0630, 0x0630, 0x0631, 0x0631, 0x0632, + 0x0632, 0x0633, 0x0633, 0x0633, 0x0633, 0x0634, 0x0634, 0x0634, + 0x0634, 0x0635, 0x0635, 0x0635, 0x0635, 0x0636, 0x0636, 0x0636, + }; + +static const unsigned short gNormalizeTablefec0[] = { + /* U+fec0 */ + 0x0636, 0x0637, 0x0637, 0x0637, 0x0637, 0x0638, 0x0638, 0x0638, + 0x0638, 0x0639, 0x0639, 0x0639, 0x0639, 0x063a, 0x063a, 0x063a, + 0x063a, 0x0641, 0x0641, 0x0641, 0x0641, 0x0642, 0x0642, 0x0642, + 0x0642, 0x0643, 0x0643, 0x0643, 0x0643, 0x0644, 0x0644, 0x0644, + 0x0644, 0x0645, 0x0645, 0x0645, 0x0645, 0x0646, 0x0646, 0x0646, + 0x0646, 0x0647, 0x0647, 0x0647, 0x0647, 0x0648, 0x0648, 0x0649, + 0x0649, 0x064a, 0x064a, 0x064a, 0x064a, 0x0644, 0x0644, 0x0644, + 0x0644, 0x0644, 0x0644, 0x0644, 0x0644, 0xfefd, 0xfefe, 0x0020, + }; + +static const unsigned short gNormalizeTableff00[] = { + /* U+ff00 */ + 0xff00, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + }; + +static const unsigned short gNormalizeTableff40[] = { + /* U+ff40 */ + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x2985, + 0x2986, 0x3002, 0x300c, 0x300d, 0x3001, 0x30fb, 0x30f2, 0x30a1, + 0x30a3, 0x30a5, 0x30a7, 0x30a9, 0x30e3, 0x30e5, 0x30e7, 0x30c3, + 0x30fc, 0x30a2, 0x30a4, 0x30a6, 0x30a8, 0x30aa, 0x30ab, 0x30ad, + 0x30af, 0x30b1, 0x30b3, 0x30b5, 0x30b7, 0x30b9, 0x30bb, 0x30bd, + }; + +static const unsigned short gNormalizeTableff80[] = { + /* U+ff80 */ + 0x30bf, 0x30c1, 0x30c4, 0x30c6, 0x30c8, 0x30ca, 0x30cb, 0x30cc, + 0x30cd, 0x30ce, 0x30cf, 0x30d2, 0x30d5, 0x30d8, 0x30db, 0x30de, + 0x30df, 0x30e0, 0x30e1, 0x30e2, 0x30e4, 0x30e6, 0x30e8, 0x30e9, + 0x30ea, 0x30eb, 0x30ec, 0x30ed, 0x30ef, 0x30f3, 0x3099, 0x309a, + 0x0020, 0x1100, 0x1101, 0x11aa, 0x1102, 0x11ac, 0x11ad, 0x1103, + 0x1104, 0x1105, 0x11b0, 0x11b1, 0x11b2, 0x11b3, 0x11b4, 0x11b5, + 0x111a, 0x1106, 0x1107, 0x1108, 0x1121, 0x1109, 0x110a, 0x110b, + 0x110c, 0x110d, 0x110e, 0x110f, 0x1110, 0x1111, 0x1112, 0xffbf, + }; + +static const unsigned short gNormalizeTableffc0[] = { + /* U+ffc0 */ + 0xffc0, 0xffc1, 0x1161, 0x1162, 0x1163, 0x1164, 0x1165, 0x1166, + 0xffc8, 0xffc9, 0x1167, 0x1168, 0x1169, 0x116a, 0x116b, 0x116c, + 0xffd0, 0xffd1, 0x116d, 0x116e, 0x116f, 0x1170, 0x1171, 0x1172, + 0xffd8, 0xffd9, 0x1173, 0x1174, 0x1175, 0xffdd, 0xffde, 0xffdf, + 0x00a2, 0x00a3, 0x00ac, 0x0020, 0x00a6, 0x00a5, 0x20a9, 0xffe7, + 0x2502, 0x2190, 0x2191, 0x2192, 0x2193, 0x25a0, 0x25cb, 0xffef, + 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, + 0x0020, 0xfff9, 0xfffa, 0xfffb, 0xfffc, 0xfffd, 0xfffe, 0xffff, + }; + +static const unsigned short* gNormalizeTable[] = { + 0, gNormalizeTable0040, gNormalizeTable0080, gNormalizeTable00c0, + gNormalizeTable0100, gNormalizeTable0140, gNormalizeTable0180, gNormalizeTable01c0, + gNormalizeTable0200, gNormalizeTable0240, gNormalizeTable0280, gNormalizeTable02c0, + 0, gNormalizeTable0340, gNormalizeTable0380, gNormalizeTable03c0, + gNormalizeTable0400, gNormalizeTable0440, gNormalizeTable0480, gNormalizeTable04c0, + gNormalizeTable0500, gNormalizeTable0540, gNormalizeTable0580, 0, + gNormalizeTable0600, gNormalizeTable0640, 0, gNormalizeTable06c0, + 0, 0, 0, 0, + 0, 0, 0, 0, + gNormalizeTable0900, gNormalizeTable0940, 0, gNormalizeTable09c0, + gNormalizeTable0a00, gNormalizeTable0a40, 0, 0, + 0, gNormalizeTable0b40, gNormalizeTable0b80, gNormalizeTable0bc0, + 0, gNormalizeTable0c40, 0, gNormalizeTable0cc0, + 0, gNormalizeTable0d40, 0, gNormalizeTable0dc0, + gNormalizeTable0e00, 0, gNormalizeTable0e80, gNormalizeTable0ec0, + gNormalizeTable0f00, gNormalizeTable0f40, gNormalizeTable0f80, 0, + gNormalizeTable1000, 0, gNormalizeTable1080, gNormalizeTable10c0, + 0, gNormalizeTable1140, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, gNormalizeTable1780, 0, + gNormalizeTable1800, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + gNormalizeTable1b00, gNormalizeTable1b40, 0, 0, + 0, 0, 0, 0, + gNormalizeTable1d00, gNormalizeTable1d40, gNormalizeTable1d80, 0, + gNormalizeTable1e00, gNormalizeTable1e40, gNormalizeTable1e80, gNormalizeTable1ec0, + gNormalizeTable1f00, gNormalizeTable1f40, gNormalizeTable1f80, gNormalizeTable1fc0, + gNormalizeTable2000, gNormalizeTable2040, gNormalizeTable2080, 0, + gNormalizeTable2100, gNormalizeTable2140, gNormalizeTable2180, gNormalizeTable21c0, + gNormalizeTable2200, gNormalizeTable2240, gNormalizeTable2280, gNormalizeTable22c0, + gNormalizeTable2300, 0, 0, 0, + 0, gNormalizeTable2440, gNormalizeTable2480, gNormalizeTable24c0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + gNormalizeTable2a00, gNormalizeTable2a40, 0, gNormalizeTable2ac0, + 0, 0, 0, 0, + gNormalizeTable2c00, gNormalizeTable2c40, gNormalizeTable2c80, gNormalizeTable2cc0, + 0, gNormalizeTable2d40, 0, 0, + 0, 0, gNormalizeTable2e80, gNormalizeTable2ec0, + gNormalizeTable2f00, gNormalizeTable2f40, gNormalizeTable2f80, gNormalizeTable2fc0, + gNormalizeTable3000, gNormalizeTable3040, gNormalizeTable3080, gNormalizeTable30c0, + gNormalizeTable3100, gNormalizeTable3140, gNormalizeTable3180, 0, + gNormalizeTable3200, gNormalizeTable3240, gNormalizeTable3280, gNormalizeTable32c0, + gNormalizeTable3300, gNormalizeTable3340, gNormalizeTable3380, gNormalizeTable33c0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, gNormalizeTablea640, gNormalizeTablea680, 0, + gNormalizeTablea700, gNormalizeTablea740, gNormalizeTablea780, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + gNormalizeTablef900, gNormalizeTablef940, gNormalizeTablef980, gNormalizeTablef9c0, + gNormalizeTablefa00, gNormalizeTablefa40, gNormalizeTablefa80, gNormalizeTablefac0, + gNormalizeTablefb00, gNormalizeTablefb40, gNormalizeTablefb80, gNormalizeTablefbc0, + gNormalizeTablefc00, gNormalizeTablefc40, gNormalizeTablefc80, gNormalizeTablefcc0, + gNormalizeTablefd00, gNormalizeTablefd40, gNormalizeTablefd80, gNormalizeTablefdc0, + gNormalizeTablefe00, gNormalizeTablefe40, gNormalizeTablefe80, gNormalizeTablefec0, + gNormalizeTableff00, gNormalizeTableff40, gNormalizeTableff80, gNormalizeTableffc0, +}; + +unsigned int normalize_character(const unsigned int c) +{ + if (c >= 0x10000 || !gNormalizeTable[c >> 6]) + return c; + return gNormalizeTable[c >> 6][c & 0x3f]; +} + diff --git a/mailnews/extensions/fts3/src/README.mozilla b/mailnews/extensions/fts3/src/README.mozilla new file mode 100644 index 000000000..0bfe7deb3 --- /dev/null +++ b/mailnews/extensions/fts3/src/README.mozilla @@ -0,0 +1,3 @@ +fts3_porter.c code is from SQLite3. + +This customized tokenizer "mozporter" by Mozilla supports CJK indexing using bi-gram. So you have to use bi-gram search string if you wanto to search CJK character. diff --git a/mailnews/extensions/fts3/src/fts3_porter.c b/mailnews/extensions/fts3/src/fts3_porter.c new file mode 100644 index 000000000..2276244c1 --- /dev/null +++ b/mailnews/extensions/fts3/src/fts3_porter.c @@ -0,0 +1,1150 @@ +/* +** 2006 September 30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Implementation of the full-text-search tokenizer that implements +** a Porter stemmer. +** +*/ + +/* + * This file is based on the SQLite FTS3 Porter Stemmer implementation. + * + * This is an attempt to provide some level of full-text search to users of + * Thunderbird who use languages that are not space/punctuation delimited. + * This is accomplished by performing bi-gram indexing of characters fall + * into the unicode space occupied by character sets used in such languages. + * + * Bi-gram indexing means that given the string "12345" we would index the + * pairs "12", "23", "34", and "45" (with position information). We do this + * because we are not sure where the word/semantic boundaries are in that + * string. Then, when a user searches for "234" the FTS3 engine tokenizes the + * search query into "23" and "34". Using special phrase-logic FTS3 requires + * the matches to have the tokens "23" and "34" adjacent to each other and in + * that order. In theory if the user searched for "2345" we we could just + * search for "23 NEAR/2 34". Unfortunately, NEAR does not imply ordering, + * so even though that would be more efficient, we would lose correctness + * and cannot do it. + * + * The efficiency and usability of bi-gram search assumes that the character + * space is large enough and actually observed bi-grams sufficiently + * distributed throughout the potential space so that the search bi-grams + * generated when the user issues a query find a 'reasonable' number of + * documents for each bi-gram match. + * + * Mozilla contributors: + * Makoto Kato <m_kato@ga2.so-net.ne.jp> + * Andrew Sutherland <asutherland@asutherland.org> + */ + +/* +** The code in this file is only compiled if: +** +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or +** +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +*/ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + + +#include <assert.h> +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <ctype.h> + +#include "fts3_tokenizer.h" + +/* need some defined to compile without sqlite3 code */ + +#define sqlite3_malloc malloc +#define sqlite3_free free +#define sqlite3_realloc realloc + +static const unsigned char sqlite3Utf8Trans1[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, +}; + +typedef unsigned char u8; + +/** + * SQLite helper macro from sqlite3.c (really utf.c) to encode a unicode + * character into utf8. + * + * @param zOut A pointer to the current write position that is updated by + * the routine. At entry it should point to one-past the last valid + * encoded byte. The same holds true at exit. + * @param c The character to encode; this should be an unsigned int. + */ +#define WRITE_UTF8(zOut, c) { \ + if( c<0x0080 ){ \ + *zOut++ = (u8)(c&0xff); \ + } \ + else if( c<0x0800 ){ \ + *zOut++ = 0xC0 + (u8)((c>>6) & 0x1F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + } \ + else if( c<0x10000 ){ \ + *zOut++ = 0xE0 + (u8)((c>>12) & 0x0F); \ + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + }else{ \ + *zOut++ = 0xf0 + (u8)((c>>18) & 0x07); \ + *zOut++ = 0x80 + (u8)((c>>12) & 0x3F); \ + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + } \ +} + +/** + * Fudge factor to avoid buffer overwrites when WRITE_UTF8 is involved. + * + * Our normalization table includes entries that may result in a larger + * utf-8 encoding. Namely, 023a maps to 2c65. This is a growth from 2 bytes + * as utf-8 encoded to 3 bytes. This is currently the only transition possible + * because 1-byte encodings are known to stay 1-byte and our normalization + * table is 16-bit and so can't generate a 4-byte encoded output. + * + * For simplicity, we just multiple by 2 which covers the current case and + * potential growth for 2-byte to 4-byte growth. We can afford to do this + * because we're not talking about a lot of memory here as a rule. + */ +#define MAX_UTF8_GROWTH_FACTOR 2 + +/** + * Helper from sqlite3.c to read a single UTF8 character. + * + * The clever bit with multi-byte reading is that you keep going until you find + * a byte whose top bits are not '10'. A single-byte UTF8 character will have + * '00' or '01', and a multi-byte UTF8 character must start with '11'. + * + * In the event of illegal UTF-8 this macro may read an arbitrary number of + * characters but will never read past zTerm. The resulting character value + * of illegal UTF-8 can be anything, although efforts are made to return the + * illegal character (0xfffd) for UTF-16 surrogates. + * + * @param zIn A pointer to the current position that is updated by the routine, + * pointing at the start of the next character when the routine returns. + * @param zTerm A pointer one past the end of the buffer. + * @param c The 'unsigned int' to hold the resulting character value. Do not + * use a short or a char. + */ +#define READ_UTF8(zIn, zTerm, c) { \ + c = *(zIn++); \ + if( c>=0xc0 ){ \ + c = sqlite3Utf8Trans1[c-0xc0]; \ + while( zIn!=zTerm && (*zIn & 0xc0)==0x80 ){ \ + c = (c<<6) + (0x3f & *(zIn++)); \ + } \ + if( c<0x80 \ + || (c&0xFFFFF800)==0xD800 \ + || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \ + } \ +} + +/* end of compatible block to complie codes */ + +/* +** Class derived from sqlite3_tokenizer +*/ +typedef struct porter_tokenizer { + sqlite3_tokenizer base; /* Base class */ +} porter_tokenizer; + +/* +** Class derived from sqlit3_tokenizer_cursor +*/ +typedef struct porter_tokenizer_cursor { + sqlite3_tokenizer_cursor base; + const char *zInput; /* input we are tokenizing */ + int nInput; /* size of the input */ + int iOffset; /* current position in zInput */ + int iToken; /* index of next token to be returned */ + unsigned char *zToken; /* storage for current token */ + int nAllocated; /* space allocated to zToken buffer */ + /** + * Store the offset of the second character in the bi-gram pair that we just + * emitted so that we can consider it being the first character in a bi-gram + * pair. + * The value 0 indicates that there is no previous such character. This is + * an acceptable sentinel value because the 0th offset can never be the + * offset of the second in a bi-gram pair. + * + * For example, let us say we are tokenizing a string of 4 CJK characters + * represented by the byte-string "11223344" where each repeated digit + * indicates 2-bytes of storage used to encode the character in UTF-8. + * (It actually takes 3, btw.) Then on the passes to emit each token, + * the iOffset and iPrevGigramOffset values at entry will be: + * + * 1122: iOffset = 0, iPrevBigramOffset = 0 + * 2233: iOffset = 4, iPrevBigramOffset = 2 + * 3344: iOffset = 6, iPrevBigramOffset = 4 + * (nothing will be emitted): iOffset = 8, iPrevBigramOffset = 6 + */ + int iPrevBigramOffset; /* previous result was bi-gram */ +} porter_tokenizer_cursor; + + +/* Forward declaration */ +static const sqlite3_tokenizer_module porterTokenizerModule; + +/* from normalize.c */ +extern unsigned int normalize_character(const unsigned int c); + +/* +** Create a new tokenizer instance. +*/ +static int porterCreate( + int argc, const char * const *argv, + sqlite3_tokenizer **ppTokenizer +){ + porter_tokenizer *t; + t = (porter_tokenizer *) sqlite3_malloc(sizeof(*t)); + if( t==NULL ) return SQLITE_NOMEM; + memset(t, 0, sizeof(*t)); + *ppTokenizer = &t->base; + return SQLITE_OK; +} + +/* +** Destroy a tokenizer +*/ +static int porterDestroy(sqlite3_tokenizer *pTokenizer){ + sqlite3_free(pTokenizer); + return SQLITE_OK; +} + +/* +** Prepare to begin tokenizing a particular string. The input +** string to be tokenized is zInput[0..nInput-1]. A cursor +** used to incrementally tokenize this string is returned in +** *ppCursor. +*/ +static int porterOpen( + sqlite3_tokenizer *pTokenizer, /* The tokenizer */ + const char *zInput, int nInput, /* String to be tokenized */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ +){ + porter_tokenizer_cursor *c; + + c = (porter_tokenizer_cursor *) sqlite3_malloc(sizeof(*c)); + if( c==NULL ) return SQLITE_NOMEM; + + c->zInput = zInput; + if( zInput==0 ){ + c->nInput = 0; + }else if( nInput<0 ){ + c->nInput = (int)strlen(zInput); + }else{ + c->nInput = nInput; + } + c->iOffset = 0; /* start tokenizing at the beginning */ + c->iToken = 0; + c->zToken = NULL; /* no space allocated, yet. */ + c->nAllocated = 0; + c->iPrevBigramOffset = 0; + + *ppCursor = &c->base; + return SQLITE_OK; +} + +/* +** Close a tokenization cursor previously opened by a call to +** porterOpen() above. +*/ +static int porterClose(sqlite3_tokenizer_cursor *pCursor){ + porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor; + sqlite3_free(c->zToken); + sqlite3_free(c); + return SQLITE_OK; +} +/* +** Vowel or consonant +*/ +static const char cType[] = { + 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, + 1, 1, 1, 2, 1 +}; + +/* +** isConsonant() and isVowel() determine if their first character in +** the string they point to is a consonant or a vowel, according +** to Porter ruls. +** +** A consonate is any letter other than 'a', 'e', 'i', 'o', or 'u'. +** 'Y' is a consonant unless it follows another consonant, +** in which case it is a vowel. +** +** In these routine, the letters are in reverse order. So the 'y' rule +** is that 'y' is a consonant unless it is followed by another +** consonent. +*/ +static int isVowel(const char*); +static int isConsonant(const char *z){ + int j; + char x = *z; + if( x==0 ) return 0; + assert( x>='a' && x<='z' ); + j = cType[x-'a']; + if( j<2 ) return j; + return z[1]==0 || isVowel(z + 1); +} +static int isVowel(const char *z){ + int j; + char x = *z; + if( x==0 ) return 0; + assert( x>='a' && x<='z' ); + j = cType[x-'a']; + if( j<2 ) return 1-j; + return isConsonant(z + 1); +} + +/* +** Let any sequence of one or more vowels be represented by V and let +** C be sequence of one or more consonants. Then every word can be +** represented as: +** +** [C] (VC){m} [V] +** +** In prose: A word is an optional consonant followed by zero or +** vowel-consonant pairs followed by an optional vowel. "m" is the +** number of vowel consonant pairs. This routine computes the value +** of m for the first i bytes of a word. +** +** Return true if the m-value for z is 1 or more. In other words, +** return true if z contains at least one vowel that is followed +** by a consonant. +** +** In this routine z[] is in reverse order. So we are really looking +** for an instance of of a consonant followed by a vowel. +*/ +static int m_gt_0(const char *z){ + while( isVowel(z) ){ z++; } + if( *z==0 ) return 0; + while( isConsonant(z) ){ z++; } + return *z!=0; +} + +/* Like mgt0 above except we are looking for a value of m which is +** exactly 1 +*/ +static int m_eq_1(const char *z){ + while( isVowel(z) ){ z++; } + if( *z==0 ) return 0; + while( isConsonant(z) ){ z++; } + if( *z==0 ) return 0; + while( isVowel(z) ){ z++; } + if( *z==0 ) return 1; + while( isConsonant(z) ){ z++; } + return *z==0; +} + +/* Like mgt0 above except we are looking for a value of m>1 instead +** or m>0 +*/ +static int m_gt_1(const char *z){ + while( isVowel(z) ){ z++; } + if( *z==0 ) return 0; + while( isConsonant(z) ){ z++; } + if( *z==0 ) return 0; + while( isVowel(z) ){ z++; } + if( *z==0 ) return 0; + while( isConsonant(z) ){ z++; } + return *z!=0; +} + +/* +** Return TRUE if there is a vowel anywhere within z[0..n-1] +*/ +static int hasVowel(const char *z){ + while( isConsonant(z) ){ z++; } + return *z!=0; +} + +/* +** Return TRUE if the word ends in a double consonant. +** +** The text is reversed here. So we are really looking at +** the first two characters of z[]. +*/ +static int doubleConsonant(const char *z){ + return isConsonant(z) && z[0]==z[1] && isConsonant(z+1); +} + +/* +** Return TRUE if the word ends with three letters which +** are consonant-vowel-consonent and where the final consonant +** is not 'w', 'x', or 'y'. +** +** The word is reversed here. So we are really checking the +** first three letters and the first one cannot be in [wxy]. +*/ +static int star_oh(const char *z){ + return + z[0]!=0 && isConsonant(z) && + z[0]!='w' && z[0]!='x' && z[0]!='y' && + z[1]!=0 && isVowel(z+1) && + z[2]!=0 && isConsonant(z+2); +} + +/* +** If the word ends with zFrom and xCond() is true for the stem +** of the word that preceeds the zFrom ending, then change the +** ending to zTo. +** +** The input word *pz and zFrom are both in reverse order. zTo +** is in normal order. +** +** Return TRUE if zFrom matches. Return FALSE if zFrom does not +** match. Not that TRUE is returned even if xCond() fails and +** no substitution occurs. +*/ +static int stem( + char **pz, /* The word being stemmed (Reversed) */ + const char *zFrom, /* If the ending matches this... (Reversed) */ + const char *zTo, /* ... change the ending to this (not reversed) */ + int (*xCond)(const char*) /* Condition that must be true */ +){ + char *z = *pz; + while( *zFrom && *zFrom==*z ){ z++; zFrom++; } + if( *zFrom!=0 ) return 0; + if( xCond && !xCond(z) ) return 1; + while( *zTo ){ + *(--z) = *(zTo++); + } + *pz = z; + return 1; +} + +/** + * Voiced sound mark is only on Japanese. It is like accent. It combines with + * previous character. Example, "サ" (Katakana) with "ã‚›" (voiced sound mark) is + * "ザ". Although full-width character mapping has combined character like "ザ", + * there is no combined character on half-width Katanaka character mapping. + */ +static int isVoicedSoundMark(const unsigned int c) +{ + if (c == 0xff9e || c == 0xff9f || c == 0x3099 || c == 0x309a) + return 1; + return 0; +} + +/** + * How many unicode characters to take from the front and back of a term in + * |copy_stemmer|. + */ +#define COPY_STEMMER_COPY_HALF_LEN 10 + +/** + * Normalizing but non-stemming term copying. + * + * The original function would take 10 bytes from the front and 10 bytes from + * the back if there were no digits in the string and it was more than 20 + * bytes long. If there were digits involved that would decrease to 3 bytes + * from the front and 3 from the back. This would potentially corrupt utf-8 + * encoded characters, which is fine from the perspective of the FTS3 logic. + * + * In our revised form we now operate on a unicode character basis rather than + * a byte basis. Additionally we use the same length limit even if there are + * digits involved because it's not clear digit token-space reduction is saving + * us from anything and could be hurting. Specifically, if no one is ever + * going to search on things with digits, then we should just remove them. + * Right now, the space reduction is going to increase false positives when + * people do search on them and increase the number of collisions sufficiently + * to make it really expensive. The caveat is there will be some increase in + * index size which could be meaningful if people are receiving lots of emails + * full of distinct numbers. + * + * In order to do the copy-from-the-front and copy-from-the-back trick, once + * we reach N characters in, we set zFrontEnd to the current value of zOut + * (which represents the termination of the first part of the result string) + * and set zBackStart to the value of zOutStart. We then advanced zBackStart + * along a character at a time as we write more characters. Once we have + * traversed the entire string, if zBackStart > zFrontEnd, then we know + * the string should be shrunk using the characters in the two ranges. + * + * (It would be faster to scan from the back with specialized logic but that + * particular logic seems easy to screw up and we don't have unit tests in here + * to the extent required.) + * + * @param zIn Input string to normalize and potentially shrink. + * @param nBytesIn The number of bytes in zIn, distinct from the number of + * unicode characters encoded in zIn. + * @param zOut The string to write our output into. This must have at least + * nBytesIn * MAX_UTF8_GROWTH_FACTOR in order to compensate for + * normalization that results in a larger utf-8 encoding. + * @param pnBytesOut Integer to write the number of bytes in zOut into. + */ +static void copy_stemmer(const unsigned char *zIn, const int nBytesIn, + unsigned char *zOut, int *pnBytesOut){ + const unsigned char *zInTerm = zIn + nBytesIn; + unsigned char *zOutStart = zOut; + unsigned int c; + unsigned int charCount = 0; + unsigned char *zFrontEnd = NULL, *zBackStart = NULL; + unsigned int trashC; + + /* copy normalized character */ + while (zIn < zInTerm) { + READ_UTF8(zIn, zInTerm, c); + c = normalize_character(c); + + /* ignore voiced/semi-voiced sound mark */ + if (!isVoicedSoundMark(c)) { + /* advance one non-voiced sound mark character. */ + if (zBackStart) + READ_UTF8(zBackStart, zOut, trashC); + + WRITE_UTF8(zOut, c); + charCount++; + if (charCount == COPY_STEMMER_COPY_HALF_LEN) { + zFrontEnd = zOut; + zBackStart = zOutStart; + } + } + } + + /* if we need to shrink the string, transplant the back bytes */ + if (zBackStart > zFrontEnd) { /* this handles when both are null too */ + size_t backBytes = zOut - zBackStart; + memmove(zFrontEnd, zBackStart, backBytes); + zOut = zFrontEnd + backBytes; + } + *zOut = 0; + *pnBytesOut = zOut - zOutStart; +} + + +/* +** Stem the input word zIn[0..nIn-1]. Store the output in zOut. +** zOut is at least big enough to hold nIn bytes. Write the actual +** size of the output word (exclusive of the '\0' terminator) into *pnOut. +** +** Any upper-case characters in the US-ASCII character set ([A-Z]) +** are converted to lower case. Upper-case UTF characters are +** unchanged. +** +** Words that are longer than about 20 bytes are stemmed by retaining +** a few bytes from the beginning and the end of the word. If the +** word contains digits, 3 bytes are taken from the beginning and +** 3 bytes from the end. For long words without digits, 10 bytes +** are taken from each end. US-ASCII case folding still applies. +** +** If the input word contains not digits but does characters not +** in [a-zA-Z] then no stemming is attempted and this routine just +** copies the input into the input into the output with US-ASCII +** case folding. +** +** Stemming never increases the length of the word. So there is +** no chance of overflowing the zOut buffer. +*/ +static void porter_stemmer( + const unsigned char *zIn, + unsigned int nIn, + unsigned char *zOut, + int *pnOut +){ + unsigned int i, j, c; + char zReverse[28]; + char *z, *z2; + const unsigned char *zTerm = zIn + nIn; + const unsigned char *zTmp = zIn; + + if( nIn<3 || nIn>=sizeof(zReverse)-7 ){ + /* The word is too big or too small for the porter stemmer. + ** Fallback to the copy stemmer */ + copy_stemmer(zIn, nIn, zOut, pnOut); + return; + } + for (j = sizeof(zReverse) - 6; zTmp < zTerm; j--) { + READ_UTF8(zTmp, zTerm, c); + c = normalize_character(c); + if( c>='a' && c<='z' ){ + zReverse[j] = c; + }else{ + /* The use of a character not in [a-zA-Z] means that we fallback + ** to the copy stemmer */ + copy_stemmer(zIn, nIn, zOut, pnOut); + return; + } + } + memset(&zReverse[sizeof(zReverse)-5], 0, 5); + z = &zReverse[j+1]; + + + /* Step 1a */ + if( z[0]=='s' ){ + if( + !stem(&z, "sess", "ss", 0) && + !stem(&z, "sei", "i", 0) && + !stem(&z, "ss", "ss", 0) + ){ + z++; + } + } + + /* Step 1b */ + z2 = z; + if( stem(&z, "dee", "ee", m_gt_0) ){ + /* Do nothing. The work was all in the test */ + }else if( + (stem(&z, "gni", "", hasVowel) || stem(&z, "de", "", hasVowel)) + && z!=z2 + ){ + if( stem(&z, "ta", "ate", 0) || + stem(&z, "lb", "ble", 0) || + stem(&z, "zi", "ize", 0) ){ + /* Do nothing. The work was all in the test */ + }else if( doubleConsonant(z) && (*z!='l' && *z!='s' && *z!='z') ){ + z++; + }else if( m_eq_1(z) && star_oh(z) ){ + *(--z) = 'e'; + } + } + + /* Step 1c */ + if( z[0]=='y' && hasVowel(z+1) ){ + z[0] = 'i'; + } + + /* Step 2 */ + switch( z[1] ){ + case 'a': + (void) (stem(&z, "lanoita", "ate", m_gt_0) || + stem(&z, "lanoit", "tion", m_gt_0)); + break; + case 'c': + (void) (stem(&z, "icne", "ence", m_gt_0) || + stem(&z, "icna", "ance", m_gt_0)); + break; + case 'e': + (void) (stem(&z, "rezi", "ize", m_gt_0)); + break; + case 'g': + (void) (stem(&z, "igol", "log", m_gt_0)); + break; + case 'l': + (void) (stem(&z, "ilb", "ble", m_gt_0) || + stem(&z, "illa", "al", m_gt_0) || + stem(&z, "iltne", "ent", m_gt_0) || + stem(&z, "ile", "e", m_gt_0) || + stem(&z, "ilsuo", "ous", m_gt_0)); + break; + case 'o': + (void) (stem(&z, "noitazi", "ize", m_gt_0) || + stem(&z, "noita", "ate", m_gt_0) || + stem(&z, "rota", "ate", m_gt_0)); + break; + case 's': + (void) (stem(&z, "msila", "al", m_gt_0) || + stem(&z, "ssenevi", "ive", m_gt_0) || + stem(&z, "ssenluf", "ful", m_gt_0) || + stem(&z, "ssensuo", "ous", m_gt_0)); + break; + case 't': + (void) (stem(&z, "itila", "al", m_gt_0) || + stem(&z, "itivi", "ive", m_gt_0) || + stem(&z, "itilib", "ble", m_gt_0)); + break; + } + + /* Step 3 */ + switch( z[0] ){ + case 'e': + (void) (stem(&z, "etaci", "ic", m_gt_0) || + stem(&z, "evita", "", m_gt_0) || + stem(&z, "ezila", "al", m_gt_0)); + break; + case 'i': + (void) (stem(&z, "itici", "ic", m_gt_0)); + break; + case 'l': + (void) (stem(&z, "laci", "ic", m_gt_0) || + stem(&z, "luf", "", m_gt_0)); + break; + case 's': + (void) (stem(&z, "ssen", "", m_gt_0)); + break; + } + + /* Step 4 */ + switch( z[1] ){ + case 'a': + if( z[0]=='l' && m_gt_1(z+2) ){ + z += 2; + } + break; + case 'c': + if( z[0]=='e' && z[2]=='n' && (z[3]=='a' || z[3]=='e') && m_gt_1(z+4) ){ + z += 4; + } + break; + case 'e': + if( z[0]=='r' && m_gt_1(z+2) ){ + z += 2; + } + break; + case 'i': + if( z[0]=='c' && m_gt_1(z+2) ){ + z += 2; + } + break; + case 'l': + if( z[0]=='e' && z[2]=='b' && (z[3]=='a' || z[3]=='i') && m_gt_1(z+4) ){ + z += 4; + } + break; + case 'n': + if( z[0]=='t' ){ + if( z[2]=='a' ){ + if( m_gt_1(z+3) ){ + z += 3; + } + }else if( z[2]=='e' ){ + (void) (stem(&z, "tneme", "", m_gt_1) || + stem(&z, "tnem", "", m_gt_1) || + stem(&z, "tne", "", m_gt_1)); + } + } + break; + case 'o': + if( z[0]=='u' ){ + if( m_gt_1(z+2) ){ + z += 2; + } + }else if( z[3]=='s' || z[3]=='t' ){ + (void) (stem(&z, "noi", "", m_gt_1)); + } + break; + case 's': + if( z[0]=='m' && z[2]=='i' && m_gt_1(z+3) ){ + z += 3; + } + break; + case 't': + (void) (stem(&z, "eta", "", m_gt_1) || + stem(&z, "iti", "", m_gt_1)); + break; + case 'u': + if( z[0]=='s' && z[2]=='o' && m_gt_1(z+3) ){ + z += 3; + } + break; + case 'v': + case 'z': + if( z[0]=='e' && z[2]=='i' && m_gt_1(z+3) ){ + z += 3; + } + break; + } + + /* Step 5a */ + if( z[0]=='e' ){ + if( m_gt_1(z+1) ){ + z++; + }else if( m_eq_1(z+1) && !star_oh(z+1) ){ + z++; + } + } + + /* Step 5b */ + if( m_gt_1(z) && z[0]=='l' && z[1]=='l' ){ + z++; + } + + /* z[] is now the stemmed word in reverse order. Flip it back + ** around into forward order and return. + */ + *pnOut = i = strlen(z); + zOut[i] = 0; + while( *z ){ + zOut[--i] = *(z++); + } +} + +/** + * Indicate whether characters in the 0x30 - 0x7f region can be part of a token. + * Letters and numbers can; punctuation (and 'del') can't. + */ +static const char porterIdChar[] = { +/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ +}; + +/** + * Test whether a character is a (non-ascii) space character or not. isDelim + * uses the existing porter stemmer logic for anything in the ASCII (< 0x80) + * space which covers 0x20. + * + * 0x2000-0x206F is the general punctuation table. 0x2000 - 0x200b are spaces. + * The spaces 0x2000 - 0x200a are all defined as roughly equivalent to a + * standard 0x20 space. 0x200b is a "zero width space" (ZWSP) and not like an + * 0x20 space. 0x202f is a narrow no-break space and roughly equivalent to an + * 0x20 space. 0x205f is a "medium mathematical space" and defined as roughly + * equivalent to an 0x20 space. + */ +#define IS_UNI_SPACE(x) (((x)>=0x2000&&(x)<=0x200a) || (x)==0x202f || (x)==0x205f) +/** + * What we are checking for: + * - 0x3001: Ideographic comma (-> 0x2c ',') + * - 0x3002: Ideographic full stop (-> 0x2e '.') + * - 0xff0c: fullwidth comma (~ wide 0x2c ',') + * - 0xff0e: fullwidth full stop (~ wide 0x2e '.') + * - 0xff61: halfwidth ideographic full stop (~ narrow 0x3002) + * - 0xff64: halfwidth ideographic comma (~ narrow 0x3001) + * + * It is possible we should be treating other things as delimiters! + */ +#define IS_JA_DELIM(x) (((x)==0x3001)||((x)==0xFF64)||((x)==0xFF0E)||((x)==0x3002)||((x)==0xFF61)||((x)==0xFF0C)) + +/** + * The previous character was a delimeter (which includes the start of the + * string). + */ +#define BIGRAM_RESET 0 +/** + * The previous character was a CJK character and we have only seen one of them. + * If we had seen more than one in a row it would be the BIGRAM_USE state. + */ +#define BIGRAM_UNKNOWN 1 +/** + * We have seen two or more CJK characters in a row. + */ +#define BIGRAM_USE 2 +/** + * The previous character was ASCII or something in the unicode general scripts + * area that we do not believe is a delimeter. We call it 'alpha' as in + * alphabetic/alphanumeric and something that should be tokenized based on + * delimiters rather than on a bi-gram basis. + */ +#define BIGRAM_ALPHA 3 + +static int isDelim( + const unsigned char *zCur, /* IN: current pointer of token */ + const unsigned char *zTerm, /* IN: one character beyond end of token */ + int *len, /* OUT: analyzed bytes in this token */ + int *state /* IN/OUT: analyze state */ +){ + const unsigned char *zIn = zCur; + unsigned int c; + int delim; + + /* get the unicode character to analyze */ + READ_UTF8(zIn, zTerm, c); + c = normalize_character(c); + *len = zIn - zCur; + + /* ASCII character range has rule */ + if( c < 0x80 ){ + // This is original porter stemmer isDelim logic. + // 0x0 - 0x1f are all control characters, 0x20 is space, 0x21-0x2f are + // punctuation. + delim = (c < 0x30 || !porterIdChar[c - 0x30]); + // cases: "&a", "&." + if (*state == BIGRAM_USE || *state == BIGRAM_UNKNOWN ){ + /* previous maybe CJK and current is ascii */ + *state = BIGRAM_ALPHA; /*ascii*/ + delim = 1; /* must break */ + } else if (delim == 1) { + // cases: "a.", ".." + /* this is delimiter character */ + *state = BIGRAM_RESET; /*reset*/ + } else { + // cases: "aa", ".a" + *state = BIGRAM_ALPHA; /*ascii*/ + } + return delim; + } + + // (at this point we must be a non-ASCII character) + + /* voiced/semi-voiced sound mark is ignore */ + if (isVoicedSoundMark(c) && *state != BIGRAM_ALPHA) { + /* ignore this because it is combined with previous char */ + return 0; + } + + /* this isn't CJK range, so return as no delim */ + // Anything less than 0x2000 (except to U+0E00-U+0EFF and U+1780-U+17FF) + // is the general scripts area and should not be bi-gram indexed. + // 0xa000 - 0a4cf is the Yi area. It is apparently a phonetic language whose + // usage does not appear to have simple delimeter rules, so we're leaving it + // as bigram processed. This is a guess, if you know better, let us know. + // (We previously bailed on this range too.) + // Addition, U+0E00-U+0E7F is Thai, U+0E80-U+0EFF is Laos, + // and U+1780-U+17FF is Khmer. It is no easy way to break each word. + // So these should use bi-gram too. + // cases: "aa", ".a", "&a" + if (c < 0xe00 || + (c >= 0xf00 && c < 0x1780) || + (c >= 0x1800 && c < 0x2000)) { + *state = BIGRAM_ALPHA; /* not really ASCII but same idea; tokenize it */ + return 0; + } + + // (at this point we must be a bi-grammable char or delimiter) + + /* this is space character or delim character */ + // cases: "a.", "..", "&." + if( IS_UNI_SPACE(c) || IS_JA_DELIM(c) ){ + *state = BIGRAM_RESET; /* reset */ + return 1; /* it actually is a delimiter; report as such */ + } + + // (at this point we must be a bi-grammable char) + + // cases: "a&" + if( *state==BIGRAM_ALPHA ){ + /* Previous is ascii and current maybe CJK */ + *state = BIGRAM_UNKNOWN; /* mark as unknown */ + return 1; /* break to emit the ASCII token*/ + } + + /* We have no rule for CJK!. use bi-gram */ + // cases: "&&" + if( *state==BIGRAM_UNKNOWN || *state==BIGRAM_USE ){ + /* previous state is unknown. mark as bi-gram */ + *state = BIGRAM_USE; + return 1; /* break to emit the digram */ + } + + // cases: ".&" (*state == BIGRAM_RESET) + *state = BIGRAM_UNKNOWN; /* mark as unknown */ + return 0; /* no need to break; nothing to emit */ +} + +/** + * Generate a new token. There are basically three types of token we can + * generate: + * - A porter stemmed token. This is a word entirely comprised of ASCII + * characters. We run the porter stemmer algorithm against the word. + * Because we have no way to know what is and is not an English word + * (the only language for which the porter stemmer was designed), this + * could theoretically map multiple words that are not variations of the + * same word down to the same root, resulting in potentially unexpected + * result inclusions in the search results. We accept this result because + * there's not a lot we can do about it and false positives are much + * better than false negatives. + * - A copied token; case/accent-folded but not stemmed. We call the porter + * stemmer for all non-CJK cases and it diverts to the copy stemmer if it + * sees any non-ASCII characters (after folding) or if the string is too + * long. The copy stemmer will shrink the string if it is deemed too long. + * - A bi-gram token; two CJK-ish characters. For query reasons we generate a + * series of overlapping bi-grams. (We can't require the user to start their + * search based on the arbitrary context of the indexed documents.) + * + * It may be useful to think of this function as operating at the points between + * characters. While we are considering the 'current' character (the one after + * the 'point'), we are also interested in the 'previous' character (the one + * preceding the point). + * At any 'point', there are a number of possible situations which I will + * illustrate with pairs of characters. 'a' means alphanumeric ASCII or a + * non-ASCII character that is not bi-grammable or a delimeter, '.' + * means a delimiter (space or punctuation), '&' means a bi-grammable + * character. + * - aa: We are in the midst of a token. State remains BIGRAM_ALPHA. + * - a.: We will generate a porter stemmed or copied token. State was + * BIGRAM_ALPHA, gets set to BIGRAM_RESET. + * - a&: We will generate a porter stemmed or copied token; we will set our + * state to BIGRAM_UNKNOWN to indicate we have seen one bigram character + * but that it is not yet time to emit a bigram. + * - .a: We are starting a token. State was BIGRAM_RESET, gets set to + * BIGRAM_ALPHA. + * - ..: We skip/eat the delimeters. State stays BIGRAM_RESET. + * - .&: State set to BIGRAM_UNKNOWN to indicate we have seen one bigram char. + * - &a: If the state was BIGRAM_USE, we generate a bi-gram token. If the state + * was BIGRAM_UNKNOWN we had only seen one CJK character and so don't do + * anything. State is set to BIGRAM_ALPHA. + * - &.: Same as the "&a" case, but state is set to BIGRAM_RESET. + * - &&: We will generate a bi-gram token. State was either BIGRAM_UNKNOWN or + * BIGRAM_USE, gets set to BIGRAM_USE. + */ +static int porterNext( + sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by porterOpen */ + const char **pzToken, /* OUT: *pzToken is the token text */ + int *pnBytes, /* OUT: Number of bytes in token */ + int *piStartOffset, /* OUT: Starting offset of token */ + int *piEndOffset, /* OUT: Ending offset of token */ + int *piPosition /* OUT: Position integer of token */ +){ + porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor; + const unsigned char *z = (unsigned char *) c->zInput; + int len = 0; + int state; + + while( c->iOffset < c->nInput ){ + int iStartOffset, numChars; + + /* + * This loop basically has two modes of operation: + * - general processing (iPrevBigramOffset == 0 here) + * - CJK processing (iPrevBigramOffset != 0 here) + * + * In an general processing pass we skip over all the delimiters, leaving us + * at a character that promises to produce a token. This could be a CJK + * token (state == BIGRAM_USE) or an ALPHA token (state == BIGRAM_ALPHA). + * If it was a CJK token, we transition into CJK state for the next loop. + * If it was an alpha token, our current offset is pointing at a delimiter + * (which could be a CJK character), so it is good that our next pass + * through the function and loop will skip over any delimiters. If the + * delimiter we hit was a CJK character, the next time through we will + * not treat it as a delimiter though; the entry state for that scan is + * BIGRAM_RESET so the transition is not treated as a delimiter! + * + * The CJK pass always starts with the second character in a bi-gram emitted + * as a token in the previous step. No delimiter skipping is required + * because we know that first character might produce a token for us. It + * only 'might' produce a token because the previous pass performed no + * lookahead and cannot be sure it is followed by another CJK character. + * This is why + */ + + // If we have a previous bigram offset + if (c->iPrevBigramOffset == 0) { + /* Scan past delimiter characters */ + state = BIGRAM_RESET; /* reset */ + while (c->iOffset < c->nInput && + isDelim(z + c->iOffset, z + c->nInput, &len, &state)) { + c->iOffset += len; + } + + } else { + /* for bigram indexing, use previous offset */ + c->iOffset = c->iPrevBigramOffset; + } + + /* Count non-delimiter characters. */ + iStartOffset = c->iOffset; + numChars = 0; + + // Start from a reset state. This means the first character we see + // (which will not be a delimiter) determines which of ALPHA or CJK modes + // we are operating in. (It won't be a delimiter because in a 'general' + // pass as defined above, we will have eaten all the delimiters, and in + // a CJK pass we are guaranteed that the first character is CJK.) + state = BIGRAM_RESET; /* state is reset */ + // Advance until it is time to emit a token. + // For ALPHA characters, this means advancing until we encounter a delimiter + // or a CJK character. iOffset will be pointing at the delimiter or CJK + // character, aka one beyond the last ALPHA character. + // For CJK characters this means advancing until we encounter an ALPHA + // character, a delimiter, or we have seen two consecutive CJK + // characters. iOffset points at the ALPHA/delimiter in the first 2 cases + // and the second of two CJK characters in the last case. + // Because of the way this loop is structured, iOffset is only updated + // when we don't terminate. However, if we terminate, len still contains + // the number of bytes in the character found at iOffset. (This is useful + // in the CJK case.) + while (c->iOffset < c->nInput && + !isDelim(z + c->iOffset, z + c->nInput, &len, &state)) { + c->iOffset += len; + numChars++; + } + + if (state == BIGRAM_USE) { + /* Split word by bigram */ + // Right now iOffset is pointing at the second character in a pair. + // Save this offset so next-time through we start with that as the + // first character. + c->iPrevBigramOffset = c->iOffset; + // And now advance so that iOffset is pointing at the character after + // the second character in the bi-gram pair. Also count the char. + c->iOffset += len; + numChars++; + } else { + /* Reset bigram offset */ + c->iPrevBigramOffset = 0; + } + + /* We emit a token if: + * - there are two ideograms together, + * - there are three chars or more, + * - we think this is a query and wildcard magic is desired. + * We think is a wildcard query when we have a single character, it starts + * at the start of the buffer, it's CJK, our current offset is one shy of + * nInput and the character at iOffset is '*'. Because the state gets + * clobbered by the incidence of '*' our requirement for CJK is that the + * implied character length is at least 3 given that it takes at least 3 + * bytes to encode to 0x2000. + */ + // It is possible we have no token to emit here if iPrevBigramOffset was not + // 0 on entry and there was no second CJK character. iPrevBigramOffset + // will now be 0 if that is the case (and c->iOffset == iStartOffset). + if (// allow two-character words only if in bigram + (numChars == 2 && state == BIGRAM_USE) || + // otherwise, drop two-letter words (considered stop-words) + (numChars >=3) || + // wildcard case: + (numChars == 1 && iStartOffset == 0 && + (c->iOffset >= 3) && + (c->iOffset == c->nInput - 1) && + (z[c->iOffset] == '*'))) { + /* figure out the number of bytes to copy/stem */ + int n = c->iOffset - iStartOffset; + /* make sure there is enough buffer space */ + if (n * MAX_UTF8_GROWTH_FACTOR > c->nAllocated) { + c->nAllocated = n * MAX_UTF8_GROWTH_FACTOR + 20; + c->zToken = sqlite3_realloc(c->zToken, c->nAllocated); + if (c->zToken == NULL) + return SQLITE_NOMEM; + } + + if (state == BIGRAM_USE) { + /* This is by bigram. So it is unnecessary to convert word */ + copy_stemmer(&z[iStartOffset], n, c->zToken, pnBytes); + } else { + porter_stemmer(&z[iStartOffset], n, c->zToken, pnBytes); + } + *pzToken = (const char*)c->zToken; + *piStartOffset = iStartOffset; + *piEndOffset = c->iOffset; + *piPosition = c->iToken++; + return SQLITE_OK; + } + } + return SQLITE_DONE; +} + +/* +** The set of routines that implement the porter-stemmer tokenizer +*/ +static const sqlite3_tokenizer_module porterTokenizerModule = { + 0, + porterCreate, + porterDestroy, + porterOpen, + porterClose, + porterNext, +}; + +/* +** Allocate a new porter tokenizer. Return a pointer to the new +** tokenizer in *ppModule +*/ +void sqlite3Fts3PorterTokenizerModule( + sqlite3_tokenizer_module const**ppModule +){ + *ppModule = &porterTokenizerModule; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ diff --git a/mailnews/extensions/fts3/src/fts3_tokenizer.h b/mailnews/extensions/fts3/src/fts3_tokenizer.h new file mode 100644 index 000000000..906303db4 --- /dev/null +++ b/mailnews/extensions/fts3/src/fts3_tokenizer.h @@ -0,0 +1,148 @@ +/* +** 2006 July 10 +** +** The author disclaims copyright to this source code. +** +************************************************************************* +** Defines the interface to tokenizers used by fulltext-search. There +** are three basic components: +** +** sqlite3_tokenizer_module is a singleton defining the tokenizer +** interface functions. This is essentially the class structure for +** tokenizers. +** +** sqlite3_tokenizer is used to define a particular tokenizer, perhaps +** including customization information defined at creation time. +** +** sqlite3_tokenizer_cursor is generated by a tokenizer to generate +** tokens from a particular input. +*/ +#ifndef _FTS3_TOKENIZER_H_ +#define _FTS3_TOKENIZER_H_ + +/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time. +** If tokenizers are to be allowed to call sqlite3_*() functions, then +** we will need a way to register the API consistently. +*/ +#include "sqlite3.h" + +/* +** Structures used by the tokenizer interface. When a new tokenizer +** implementation is registered, the caller provides a pointer to +** an sqlite3_tokenizer_module containing pointers to the callback +** functions that make up an implementation. +** +** When an fts3 table is created, it passes any arguments passed to +** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the +** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer +** implementation. The xCreate() function in turn returns an +** sqlite3_tokenizer structure representing the specific tokenizer to +** be used for the fts3 table (customized by the tokenizer clause arguments). +** +** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen() +** method is called. It returns an sqlite3_tokenizer_cursor object +** that may be used to tokenize a specific input buffer based on +** the tokenization rules supplied by a specific sqlite3_tokenizer +** object. +*/ +typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module; +typedef struct sqlite3_tokenizer sqlite3_tokenizer; +typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor; + +struct sqlite3_tokenizer_module { + + /* + ** Structure version. Should always be set to 0. + */ + int iVersion; + + /* + ** Create a new tokenizer. The values in the argv[] array are the + ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL + ** TABLE statement that created the fts3 table. For example, if + ** the following SQL is executed: + ** + ** CREATE .. USING fts3( ... , tokenizer <tokenizer-name> arg1 arg2) + ** + ** then argc is set to 2, and the argv[] array contains pointers + ** to the strings "arg1" and "arg2". + ** + ** This method should return either SQLITE_OK (0), or an SQLite error + ** code. If SQLITE_OK is returned, then *ppTokenizer should be set + ** to point at the newly created tokenizer structure. The generic + ** sqlite3_tokenizer.pModule variable should not be initialised by + ** this callback. The caller will do so. + */ + int (*xCreate)( + int argc, /* Size of argv array */ + const char *const*argv, /* Tokenizer argument strings */ + sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ + ); + + /* + ** Destroy an existing tokenizer. The fts3 module calls this method + ** exactly once for each successful call to xCreate(). + */ + int (*xDestroy)(sqlite3_tokenizer *pTokenizer); + + /* + ** Create a tokenizer cursor to tokenize an input buffer. The caller + ** is responsible for ensuring that the input buffer remains valid + ** until the cursor is closed (using the xClose() method). + */ + int (*xOpen)( + sqlite3_tokenizer *pTokenizer, /* Tokenizer object */ + const char *pInput, int nBytes, /* Input buffer */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Created tokenizer cursor */ + ); + + /* + ** Destroy an existing tokenizer cursor. The fts3 module calls this + ** method exactly once for each successful call to xOpen(). + */ + int (*xClose)(sqlite3_tokenizer_cursor *pCursor); + + /* + ** Retrieve the next token from the tokenizer cursor pCursor. This + ** method should either return SQLITE_OK and set the values of the + ** "OUT" variables identified below, or SQLITE_DONE to indicate that + ** the end of the buffer has been reached, or an SQLite error code. + ** + ** *ppToken should be set to point at a buffer containing the + ** normalized version of the token (i.e. after any case-folding and/or + ** stemming has been performed). *pnBytes should be set to the length + ** of this buffer in bytes. The input text that generated the token is + ** identified by the byte offsets returned in *piStartOffset and + ** *piEndOffset. *piStartOffset should be set to the index of the first + ** byte of the token in the input buffer. *piEndOffset should be set + ** to the index of the first byte just past the end of the token in + ** the input buffer. + ** + ** The buffer *ppToken is set to point at is managed by the tokenizer + ** implementation. It is only required to be valid until the next call + ** to xNext() or xClose(). + */ + /* TODO(shess) current implementation requires pInput to be + ** nul-terminated. This should either be fixed, or pInput/nBytes + ** should be converted to zInput. + */ + int (*xNext)( + sqlite3_tokenizer_cursor *pCursor, /* Tokenizer cursor */ + const char **ppToken, int *pnBytes, /* OUT: Normalized text for token */ + int *piStartOffset, /* OUT: Byte offset of token in input buffer */ + int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */ + int *piPosition /* OUT: Number of tokens returned before this one */ + ); +}; + +struct sqlite3_tokenizer { + const sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */ + /* Tokenizer implementations will typically add additional fields */ +}; + +struct sqlite3_tokenizer_cursor { + sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */ + /* Tokenizer implementations will typically add additional fields */ +}; + +#endif /* _FTS3_TOKENIZER_H_ */ diff --git a/mailnews/extensions/fts3/src/moz.build b/mailnews/extensions/fts3/src/moz.build new file mode 100644 index 000000000..a2b3c60d5 --- /dev/null +++ b/mailnews/extensions/fts3/src/moz.build @@ -0,0 +1,18 @@ +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +SOURCES += [ + 'fts3_porter.c', + 'Normalize.c', +] + +SOURCES += [ + 'nsFts3Tokenizer.cpp', + 'nsGlodaRankerFunction.cpp', +] + +FINAL_LIBRARY = 'mail' + +CXXFLAGS += CONFIG['SQLITE_CFLAGS'] diff --git a/mailnews/extensions/fts3/src/nsFts3Tokenizer.cpp b/mailnews/extensions/fts3/src/nsFts3Tokenizer.cpp new file mode 100644 index 000000000..12bd70ead --- /dev/null +++ b/mailnews/extensions/fts3/src/nsFts3Tokenizer.cpp @@ -0,0 +1,72 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsFts3Tokenizer.h" + +#include "nsGlodaRankerFunction.h" + +#include "nsIFts3Tokenizer.h" +#include "mozIStorageConnection.h" +#include "mozIStorageStatement.h" +#include "nsStringGlue.h" + +extern "C" void sqlite3Fts3PorterTokenizerModule( + sqlite3_tokenizer_module const**ppModule); + +extern "C" void glodaRankFunc(sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal); + +NS_IMPL_ISUPPORTS(nsFts3Tokenizer,nsIFts3Tokenizer) + +nsFts3Tokenizer::nsFts3Tokenizer() +{ +} + +nsFts3Tokenizer::~nsFts3Tokenizer() +{ +} + +NS_IMETHODIMP +nsFts3Tokenizer::RegisterTokenizer(mozIStorageConnection *connection) +{ + nsresult rv; + nsCOMPtr<mozIStorageStatement> selectStatement; + + // -- register the tokenizer + rv = connection->CreateStatement(NS_LITERAL_CSTRING( + "SELECT fts3_tokenizer(?1, ?2)"), + getter_AddRefs(selectStatement)); + NS_ENSURE_SUCCESS(rv, rv); + + const sqlite3_tokenizer_module* module = nullptr; + sqlite3Fts3PorterTokenizerModule(&module); + if (!module) + return NS_ERROR_FAILURE; + + rv = selectStatement->BindUTF8StringParameter( + 0, NS_LITERAL_CSTRING("mozporter")); + NS_ENSURE_SUCCESS(rv, rv); + rv = selectStatement->BindBlobParameter(1, + (uint8_t*)&module, + sizeof(module)); + NS_ENSURE_SUCCESS(rv, rv); + + bool hasMore; + rv = selectStatement->ExecuteStep(&hasMore); + NS_ENSURE_SUCCESS(rv, rv); + + // -- register the ranking function + nsCOMPtr<mozIStorageFunction> func = new nsGlodaRankerFunction(); + NS_ENSURE_TRUE(func, NS_ERROR_OUT_OF_MEMORY); + rv = connection->CreateFunction( + NS_LITERAL_CSTRING("glodaRank"), + -1, // variable argument support + func + ); + NS_ENSURE_SUCCESS(rv, rv); + + return rv; +} diff --git a/mailnews/extensions/fts3/src/nsFts3Tokenizer.h b/mailnews/extensions/fts3/src/nsFts3Tokenizer.h new file mode 100644 index 000000000..eb1e83bd5 --- /dev/null +++ b/mailnews/extensions/fts3/src/nsFts3Tokenizer.h @@ -0,0 +1,26 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef nsFts3Tokenizer_h__ +#define nsFts3Tokenizer_h__ + +#include "nsCOMPtr.h" +#include "nsIFts3Tokenizer.h" +#include "fts3_tokenizer.h" + +extern const sqlite3_tokenizer_module* getWindowsTokenizer(); + +class nsFts3Tokenizer final : public nsIFts3Tokenizer { +public: + NS_DECL_ISUPPORTS + NS_DECL_NSIFTS3TOKENIZER + + nsFts3Tokenizer(); + +private: + ~nsFts3Tokenizer(); +}; + +#endif diff --git a/mailnews/extensions/fts3/src/nsFts3TokenizerCID.h b/mailnews/extensions/fts3/src/nsFts3TokenizerCID.h new file mode 100644 index 000000000..1f9c101bc --- /dev/null +++ b/mailnews/extensions/fts3/src/nsFts3TokenizerCID.h @@ -0,0 +1,16 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef nsFts3TokenizerCID_h__ +#define nsFts3TokenizerCID_h__ + +#define NS_FTS3TOKENIZER_CONTRACTID \ + "@mozilla.org/messenger/fts3tokenizer;1" +#define NS_FTS3TOKENIZER_CID \ +{ /* a67d724d-0015-4e2e-8cad-b84775330924 */ \ + 0xa67d724d, 0x0015, 0x4e2e, \ + { 0x8c, 0xad, 0xb8, 0x47, 0x75, 0x33, 0x09, 0x24 }} + +#endif /* nsFts3TokenizerCID_h__ */ diff --git a/mailnews/extensions/fts3/src/nsGlodaRankerFunction.cpp b/mailnews/extensions/fts3/src/nsGlodaRankerFunction.cpp new file mode 100644 index 000000000..fb576685d --- /dev/null +++ b/mailnews/extensions/fts3/src/nsGlodaRankerFunction.cpp @@ -0,0 +1,145 @@ +/* ***** BEGIN LICENSE BLOCK ***** + * Version: MPL 1.1/GPL 2.0/LGPL 2.1 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is Thunderbird Global Database. + * + * The Initial Developer of the Original Code is the Mozilla Foundation. + * Portions created by the Initial Developer are Copyright (C) 2010 + * the Initial Developer. All Rights Reserved. + * + * Contributor(s): + * Andrew Sutherland <asutherland@asutherland.org> + * + * Alternatively, the contents of this file may be used under the terms of + * either the GNU General Public License Version 2 or later (the "GPL"), or + * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), + * in which case the provisions of the GPL or the LGPL are applicable instead + * of those above. If you wish to allow use of your version of this file only + * under the terms of either the GPL or the LGPL, and not to allow others to + * use your version of this file under the terms of the MPL, indicate your + * decision by deleting the provisions above and replace them with the notice + * and other provisions required by the GPL or the LGPL. If you do not delete + * the provisions above, a recipient may use your version of this file under + * the terms of any one of the MPL, the GPL or the LGPL. + * + * ***** END LICENSE BLOCK ***** */ + +#include "nsGlodaRankerFunction.h" +#include "mozIStorageValueArray.h" + +#include "sqlite3.h" + +#include "nsCOMPtr.h" +#include "nsVariant.h" +#include "nsComponentManagerUtils.h" + +#ifndef SQLITE_VERSION_NUMBER +#error "We need SQLITE_VERSION_NUMBER defined!" +#endif + +NS_IMPL_ISUPPORTS(nsGlodaRankerFunction, mozIStorageFunction) + +nsGlodaRankerFunction::nsGlodaRankerFunction() +{ +} + +nsGlodaRankerFunction::~nsGlodaRankerFunction() +{ +} + +static uint32_t COLUMN_SATURATION[] = {10, 1, 1, 1, 1}; + +/** + * Our ranking function basically just multiplies the weight of the column + * against the number of (saturating) matches. + * + * The original code is a SQLite example ranking function, although somewhat + * rather modified at this point. All SQLite code is public domain, so we are + * subsuming it to MPL1.1/LGPL2/GPL2. + */ +NS_IMETHODIMP +nsGlodaRankerFunction::OnFunctionCall(mozIStorageValueArray *aArguments, + nsIVariant **_result) +{ + // all argument names are maintained from the original SQLite code. + uint32_t nVal; + nsresult rv = aArguments->GetNumEntries(&nVal); + NS_ENSURE_SUCCESS(rv, rv); + + /* Check that the number of arguments passed to this function is correct. + * If not, return an error. Set aArgsData to point to the array + * of unsigned integer values returned by FTS3 function. Set nPhrase + * to contain the number of reportable phrases in the users full-text + * query, and nCol to the number of columns in the table. + */ + if (nVal < 1) + return NS_ERROR_INVALID_ARG; + + uint32_t lenArgsData; + uint32_t *aArgsData = (uint32_t *)aArguments->AsSharedBlob(0, &lenArgsData); + + uint32_t nPhrase = aArgsData[0]; + uint32_t nCol = aArgsData[1]; + if (nVal != (1 + nCol)) + return NS_ERROR_INVALID_ARG; + + double score = 0.0; + + // SQLite 3.6.22 has a different matchinfo layout than SQLite 3.6.23+ +#if SQLITE_VERSION_NUMBER <= 3006022 + /* Iterate through each phrase in the users query. */ + for (uint32_t iPhrase = 0; iPhrase < nPhrase; iPhrase++) { + // in SQ + for (uint32_t iCol = 0; iCol < nCol; iCol++) { + uint32_t nHitCount = aArgsData[2 + (iPhrase+1)*nCol + iCol]; + double weight = aArguments->AsDouble(iCol+1); + if (nHitCount > 0) { + score += (nHitCount > COLUMN_SATURATION[iCol]) ? + (COLUMN_SATURATION[iCol] * weight) : + (nHitCount * weight); + } + } + } +#else + /* Iterate through each phrase in the users query. */ + for (uint32_t iPhrase = 0; iPhrase < nPhrase; iPhrase++) { + /* Now iterate through each column in the users query. For each column, + ** increment the relevancy score by: + ** + ** (<hit count> / <global hit count>) * <column weight> + ** + ** aPhraseinfo[] points to the start of the data for phrase iPhrase. So + ** the hit count and global hit counts for each column are found in + ** aPhraseinfo[iCol*3] and aPhraseinfo[iCol*3+1], respectively. + */ + uint32_t *aPhraseinfo = &aArgsData[2 + iPhrase*nCol*3]; + for (uint32_t iCol = 0; iCol < nCol; iCol++) { + uint32_t nHitCount = aPhraseinfo[3 * iCol]; + double weight = aArguments->AsDouble(iCol+1); + if (nHitCount > 0) { + score += (nHitCount > COLUMN_SATURATION[iCol]) ? + (COLUMN_SATURATION[iCol] * weight) : + (nHitCount * weight); + } + } + } +#endif + + nsCOMPtr<nsIWritableVariant> result = new nsVariant(); + + rv = result->SetAsDouble(score); + NS_ENSURE_SUCCESS(rv, rv); + + NS_ADDREF(*_result = result); + return NS_OK; +} diff --git a/mailnews/extensions/fts3/src/nsGlodaRankerFunction.h b/mailnews/extensions/fts3/src/nsGlodaRankerFunction.h new file mode 100644 index 000000000..5c5c920d0 --- /dev/null +++ b/mailnews/extensions/fts3/src/nsGlodaRankerFunction.h @@ -0,0 +1,25 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef _nsGlodaRankerFunction_h_ +#define _nsGlodaRankerFunction_h_ + +#include "mozIStorageFunction.h" + +/** + * Basically a port of the example FTS3 ranking function to mozStorage's + * view of the universe. This might get fancier at some point. + */ +class nsGlodaRankerFunction final : public mozIStorageFunction +{ +public: + NS_DECL_ISUPPORTS + NS_DECL_MOZISTORAGEFUNCTION + + nsGlodaRankerFunction(); +private: + ~nsGlodaRankerFunction(); +}; + +#endif // _nsGlodaRankerFunction_h_ diff --git a/mailnews/extensions/mailviews/content/mailViews.dat b/mailnews/extensions/mailviews/content/mailViews.dat new file mode 100644 index 000000000..8565d0061 --- /dev/null +++ b/mailnews/extensions/mailviews/content/mailViews.dat @@ -0,0 +1,22 @@ +version="8" +logging="no" +name="People I Know" +enabled="yes" +type="1" +condition="AND (from,is in ab,moz-abmdbdirectory://abook.mab)" +name="Recent Mail" +enabled="yes" +type="1" +condition="AND (age in days,is less than,1)" +name="Last 5 Days" +enabled="yes" +type="1" +condition="AND (age in days,is less than,5)" +name="Not Junk" +enabled="yes" +type="1" +condition="AND (junk status,isn't,2)" +name="Has Attachments" +enabled="yes" +type="1" +condition="AND (has attachment status,is,true)" diff --git a/mailnews/extensions/mailviews/content/moz.build b/mailnews/extensions/mailviews/content/moz.build new file mode 100644 index 000000000..f5564e55b --- /dev/null +++ b/mailnews/extensions/mailviews/content/moz.build @@ -0,0 +1,8 @@ +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +FINAL_TARGET_FILES.defaults.messenger += [ + 'mailViews.dat', +] diff --git a/mailnews/extensions/mailviews/public/moz.build b/mailnews/extensions/mailviews/public/moz.build new file mode 100644 index 000000000..4d8dd8287 --- /dev/null +++ b/mailnews/extensions/mailviews/public/moz.build @@ -0,0 +1,12 @@ +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +XPIDL_SOURCES += [ + 'nsIMsgMailView.idl', + 'nsIMsgMailViewList.idl', +] + +XPIDL_MODULE = 'mailview' + diff --git a/mailnews/extensions/mailviews/public/nsIMsgMailView.idl b/mailnews/extensions/mailviews/public/nsIMsgMailView.idl new file mode 100644 index 000000000..54f22e76f --- /dev/null +++ b/mailnews/extensions/mailviews/public/nsIMsgMailView.idl @@ -0,0 +1,35 @@ +/* -*- Mode: IDL; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + + +#include "nsISupports.idl" +// Disable deprecation warnings generated by nsISupportsArray and associated +// classes. +%{C++ +#if defined(__GNUC__) +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#elif defined(_MSC_VER) +#pragma warning (disable : 4996) +#endif +%} +interface nsISupportsArray; + +interface nsIMsgSearchTerm; + +[scriptable, uuid(28AC84DF-CBE5-430d-A5C0-4FA63B5424DF)] +interface nsIMsgMailView : nsISupports { + attribute wstring mailViewName; + readonly attribute wstring prettyName; // localized pretty name + + // the array of search terms + attribute nsISupportsArray searchTerms; + + // these two helper methods are required to allow searchTermsOverlay.js to + // manipulate a mail view without knowing it is dealing with a mail view. nsIMsgFilter + // and nsIMsgSearchSession have the same two methods....we should probably make an interface around them. + void appendTerm(in nsIMsgSearchTerm term); + nsIMsgSearchTerm createTerm(); + +}; diff --git a/mailnews/extensions/mailviews/public/nsIMsgMailViewList.idl b/mailnews/extensions/mailviews/public/nsIMsgMailViewList.idl new file mode 100644 index 000000000..e0c846dae --- /dev/null +++ b/mailnews/extensions/mailviews/public/nsIMsgMailViewList.idl @@ -0,0 +1,28 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" +#include "nsIMsgMailView.idl" + +/////////////////////////////////////////////////////////////////////////////// +// A mail view list is a list of mail views a particular implementor provides +/////////////////////////////////////////////////////////////////////////////// + +typedef long nsMsgMailViewListFileAttribValue; + +[scriptable, uuid(6DD798D7-9528-49e6-9447-3AAF14D2D36F)] +interface nsIMsgMailViewList : nsISupports { + + readonly attribute unsigned long mailViewCount; + + nsIMsgMailView getMailViewAt(in unsigned long mailViewIndex); + + void addMailView(in nsIMsgMailView mailView); + void removeMailView(in nsIMsgMailView mailView); + + nsIMsgMailView createMailView(); + + void save(); +}; diff --git a/mailnews/extensions/mailviews/src/moz.build b/mailnews/extensions/mailviews/src/moz.build new file mode 100644 index 000000000..ee128fb8a --- /dev/null +++ b/mailnews/extensions/mailviews/src/moz.build @@ -0,0 +1,11 @@ +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +SOURCES += [ + 'nsMsgMailViewList.cpp', +] + +FINAL_LIBRARY = 'mail' + diff --git a/mailnews/extensions/mailviews/src/nsMsgMailViewList.cpp b/mailnews/extensions/mailviews/src/nsMsgMailViewList.cpp new file mode 100644 index 000000000..8e5e04a3f --- /dev/null +++ b/mailnews/extensions/mailviews/src/nsMsgMailViewList.cpp @@ -0,0 +1,312 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsMsgMailViewList.h" +// Disable deprecation warnings generated by nsISupportsArray and associated +// classes. +#if defined(__GNUC__) +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#elif defined(_MSC_VER) +#pragma warning (disable : 4996) +#endif +#include "nsISupportsArray.h" +#include "nsIFileChannel.h" +#include "nsIMsgFilterService.h" +#include "nsIMsgMailSession.h" +#include "nsIMsgSearchTerm.h" +#include "nsMsgBaseCID.h" +#include "nsAppDirectoryServiceDefs.h" +#include "nsDirectoryServiceUtils.h" +#include "nsIFile.h" +#include "nsComponentManagerUtils.h" +#include "mozilla/Services.h" +#include "nsIMsgFilter.h" + +#define kDefaultViewPeopleIKnow "People I Know" +#define kDefaultViewRecent "Recent Mail" +#define kDefaultViewFiveDays "Last 5 Days" +#define kDefaultViewNotJunk "Not Junk" +#define kDefaultViewHasAttachments "Has Attachments" + +nsMsgMailView::nsMsgMailView() +{ + mViewSearchTerms = do_CreateInstance(NS_SUPPORTSARRAY_CONTRACTID); +} + +NS_IMPL_ADDREF(nsMsgMailView) +NS_IMPL_RELEASE(nsMsgMailView) +NS_IMPL_QUERY_INTERFACE(nsMsgMailView, nsIMsgMailView) + +nsMsgMailView::~nsMsgMailView() +{ + if (mViewSearchTerms) + mViewSearchTerms->Clear(); +} + +NS_IMETHODIMP nsMsgMailView::GetMailViewName(char16_t ** aMailViewName) +{ + NS_ENSURE_ARG_POINTER(aMailViewName); + + *aMailViewName = ToNewUnicode(mName); + return NS_OK; +} + +NS_IMETHODIMP nsMsgMailView::SetMailViewName(const char16_t * aMailViewName) +{ + mName = aMailViewName; + return NS_OK; +} + +NS_IMETHODIMP nsMsgMailView::GetPrettyName(char16_t ** aMailViewName) +{ + NS_ENSURE_ARG_POINTER(aMailViewName); + + nsresult rv = NS_OK; + if (!mBundle) + { + nsCOMPtr<nsIStringBundleService> bundleService = + mozilla::services::GetStringBundleService(); + NS_ENSURE_TRUE(bundleService, NS_ERROR_UNEXPECTED); + bundleService->CreateBundle("chrome://messenger/locale/mailviews.properties", + getter_AddRefs(mBundle)); + } + + NS_ENSURE_TRUE(mBundle, NS_ERROR_FAILURE); + + // see if mName has an associated pretty name inside our string bundle and if so, use that as the pretty name + // otherwise just return mName + if (mName.EqualsLiteral(kDefaultViewPeopleIKnow)) + rv = mBundle->GetStringFromName(u"mailViewPeopleIKnow", aMailViewName); + else if (mName.EqualsLiteral(kDefaultViewRecent)) + rv = mBundle->GetStringFromName(u"mailViewRecentMail", aMailViewName); + else if (mName.EqualsLiteral(kDefaultViewFiveDays)) + rv = mBundle->GetStringFromName(u"mailViewLastFiveDays", aMailViewName); + else if (mName.EqualsLiteral(kDefaultViewNotJunk)) + rv = mBundle->GetStringFromName(u"mailViewNotJunk", aMailViewName); + else if (mName.EqualsLiteral(kDefaultViewHasAttachments)) + rv = mBundle->GetStringFromName(u"mailViewHasAttachments", aMailViewName); + else + *aMailViewName = ToNewUnicode(mName); + + return rv; +} + +NS_IMETHODIMP nsMsgMailView::GetSearchTerms(nsISupportsArray ** aSearchTerms) +{ + NS_ENSURE_ARG_POINTER(aSearchTerms); + NS_IF_ADDREF(*aSearchTerms = mViewSearchTerms); + return NS_OK; +} + +NS_IMETHODIMP nsMsgMailView::SetSearchTerms(nsISupportsArray * aSearchTerms) +{ + mViewSearchTerms = aSearchTerms; + return NS_OK; +} + +NS_IMETHODIMP nsMsgMailView::AppendTerm(nsIMsgSearchTerm * aTerm) +{ + NS_ENSURE_TRUE(aTerm, NS_ERROR_NULL_POINTER); + + return mViewSearchTerms->AppendElement(static_cast<nsISupports*>(aTerm)); +} + +NS_IMETHODIMP nsMsgMailView::CreateTerm(nsIMsgSearchTerm **aResult) +{ + NS_ENSURE_ARG_POINTER(aResult); + nsCOMPtr<nsIMsgSearchTerm> searchTerm = do_CreateInstance("@mozilla.org/messenger/searchTerm;1"); + NS_IF_ADDREF(*aResult = searchTerm); + return NS_OK; +} + +///////////////////////////////////////////////////////////////////////////// +// nsMsgMailViewList implementation +///////////////////////////////////////////////////////////////////////////// +nsMsgMailViewList::nsMsgMailViewList() +{ + LoadMailViews(); +} + +NS_IMPL_ADDREF(nsMsgMailViewList) +NS_IMPL_RELEASE(nsMsgMailViewList) +NS_IMPL_QUERY_INTERFACE(nsMsgMailViewList, nsIMsgMailViewList) + +nsMsgMailViewList::~nsMsgMailViewList() +{ + +} + +NS_IMETHODIMP nsMsgMailViewList::GetMailViewCount(uint32_t * aCount) +{ + NS_ENSURE_ARG_POINTER(aCount); + + *aCount = m_mailViews.Length(); + return NS_OK; +} + +NS_IMETHODIMP nsMsgMailViewList::GetMailViewAt(uint32_t aMailViewIndex, nsIMsgMailView ** aMailView) +{ + NS_ENSURE_ARG_POINTER(aMailView); + + uint32_t mailViewCount = m_mailViews.Length(); + + NS_ENSURE_ARG(mailViewCount > aMailViewIndex); + + NS_IF_ADDREF(*aMailView = m_mailViews[aMailViewIndex]); + return NS_OK; +} + +NS_IMETHODIMP nsMsgMailViewList::AddMailView(nsIMsgMailView * aMailView) +{ + NS_ENSURE_ARG_POINTER(aMailView); + + m_mailViews.AppendElement(aMailView); + return NS_OK; +} + +NS_IMETHODIMP nsMsgMailViewList::RemoveMailView(nsIMsgMailView * aMailView) +{ + NS_ENSURE_ARG_POINTER(aMailView); + + m_mailViews.RemoveElement(aMailView); + return NS_OK; +} + +NS_IMETHODIMP nsMsgMailViewList::CreateMailView(nsIMsgMailView ** aMailView) +{ + NS_ENSURE_ARG_POINTER(aMailView); + + nsMsgMailView * mailView = new nsMsgMailView; + NS_ENSURE_TRUE(mailView, NS_ERROR_OUT_OF_MEMORY); + + NS_IF_ADDREF(*aMailView = mailView); + return NS_OK; +} + +NS_IMETHODIMP nsMsgMailViewList::Save() +{ + // brute force...remove all the old filters in our filter list, then we'll re-add our current + // list + nsCOMPtr<nsIMsgFilter> msgFilter; + uint32_t numFilters = 0; + if (mFilterList) + mFilterList->GetFilterCount(&numFilters); + while (numFilters) + { + mFilterList->RemoveFilterAt(numFilters - 1); + numFilters--; + } + + // now convert our mail view list into a filter list and save it + ConvertMailViewListToFilterList(); + + // now save the filters to our file + return mFilterList ? mFilterList->SaveToDefaultFile() : NS_ERROR_FAILURE; +} + +nsresult nsMsgMailViewList::ConvertMailViewListToFilterList() +{ + uint32_t mailViewCount = m_mailViews.Length(); + nsCOMPtr<nsIMsgMailView> mailView; + nsCOMPtr<nsIMsgFilter> newMailFilter; + nsString mailViewName; + for (uint32_t index = 0; index < mailViewCount; index++) + { + GetMailViewAt(index, getter_AddRefs(mailView)); + if (!mailView) + continue; + mailView->GetMailViewName(getter_Copies(mailViewName)); + mFilterList->CreateFilter(mailViewName, getter_AddRefs(newMailFilter)); + if (!newMailFilter) + continue; + + nsCOMPtr<nsISupportsArray> searchTerms; + mailView->GetSearchTerms(getter_AddRefs(searchTerms)); + newMailFilter->SetSearchTerms(searchTerms); + mFilterList->InsertFilterAt(index, newMailFilter); + } + + return NS_OK; +} + +nsresult nsMsgMailViewList::LoadMailViews() +{ + nsCOMPtr<nsIFile> file; + nsresult rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = file->AppendNative(nsDependentCString("mailViews.dat")); + + // if the file doesn't exist, we should try to get it from the defaults directory and copy it over + bool exists = false; + file->Exists(&exists); + if (!exists) + { + nsCOMPtr<nsIMsgMailSession> mailSession = do_GetService(NS_MSGMAILSESSION_CONTRACTID, &rv); + NS_ENSURE_SUCCESS(rv, rv); + nsCOMPtr<nsIFile> defaultMessagesFile; + nsCOMPtr<nsIFile> profileDir; + rv = mailSession->GetDataFilesDir("messenger", getter_AddRefs(defaultMessagesFile)); + rv = defaultMessagesFile->AppendNative(nsDependentCString("mailViews.dat")); + + // get the profile directory + rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(profileDir)); + + // now copy the file over to the profile directory + defaultMessagesFile->CopyToNative(profileDir, EmptyCString()); + } + // this is kind of a hack but I think it will be an effective hack. The filter service already knows how to + // take a nsIFile and parse the contents into filters which are very similar to mail views. Intead of + // re-writing all of that dirty parsing code, let's just re-use it then convert the results into a data strcuture + // we wish to give to our consumers. + + nsCOMPtr<nsIMsgFilterService> filterService = do_GetService(NS_MSGFILTERSERVICE_CONTRACTID, &rv); + nsCOMPtr<nsIMsgFilterList> mfilterList; + + rv = filterService->OpenFilterList(file, nullptr, nullptr, getter_AddRefs(mFilterList)); + NS_ENSURE_SUCCESS(rv, rv); + + return ConvertFilterListToMailViews(); +} +/** + * Converts the filter list into our mail view objects, + * stripping out just the info we need. + */ +nsresult nsMsgMailViewList::ConvertFilterListToMailViews() +{ + nsresult rv = NS_OK; + m_mailViews.Clear(); + + // iterate over each filter in the list + uint32_t numFilters = 0; + mFilterList->GetFilterCount(&numFilters); + for (uint32_t index = 0; index < numFilters; index++) + { + nsCOMPtr<nsIMsgFilter> msgFilter; + rv = mFilterList->GetFilterAt(index, getter_AddRefs(msgFilter)); + if (NS_FAILED(rv) || !msgFilter) + continue; + + // create a new nsIMsgMailView for this item + nsCOMPtr<nsIMsgMailView> newMailView; + rv = CreateMailView(getter_AddRefs(newMailView)); + NS_ENSURE_SUCCESS(rv, rv); + + nsString filterName; + msgFilter->GetFilterName(filterName); + newMailView->SetMailViewName(filterName.get()); + + nsCOMPtr<nsISupportsArray> filterSearchTerms; + rv = msgFilter->GetSearchTerms(getter_AddRefs(filterSearchTerms)); + NS_ENSURE_SUCCESS(rv, rv); + rv = newMailView->SetSearchTerms(filterSearchTerms); + NS_ENSURE_SUCCESS(rv, rv); + + // now append this new mail view to our global list view + m_mailViews.AppendElement(newMailView); + } + + return rv; +} diff --git a/mailnews/extensions/mailviews/src/nsMsgMailViewList.h b/mailnews/extensions/mailviews/src/nsMsgMailViewList.h new file mode 100644 index 000000000..321803392 --- /dev/null +++ b/mailnews/extensions/mailviews/src/nsMsgMailViewList.h @@ -0,0 +1,61 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + + +#ifndef _nsMsgMailViewList_H_ +#define _nsMsgMailViewList_H_ + +#include "nscore.h" +#include "nsIMsgMailViewList.h" +#include "nsCOMPtr.h" +#include "nsCOMArray.h" +// Disable deprecation warnings generated by nsISupportsArray and associated +// classes. +#if defined(__GNUC__) +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#elif defined(_MSC_VER) +#pragma warning (disable : 4996) +#endif +#include "nsISupportsArray.h" +#include "nsIStringBundle.h" +#include "nsStringGlue.h" +#include "nsIMsgFilterList.h" + +// a mail View is just a name and an array of search terms +class nsMsgMailView : public nsIMsgMailView +{ +public: + NS_DECL_ISUPPORTS + NS_DECL_NSIMSGMAILVIEW + + nsMsgMailView(); + +protected: + virtual ~nsMsgMailView(); + nsString mName; + nsCOMPtr<nsIStringBundle> mBundle; + nsCOMPtr<nsISupportsArray> mViewSearchTerms; +}; + + +class nsMsgMailViewList : public nsIMsgMailViewList +{ +public: + NS_DECL_ISUPPORTS + NS_DECL_NSIMSGMAILVIEWLIST + + nsMsgMailViewList(); + +protected: + virtual ~nsMsgMailViewList(); + nsresult LoadMailViews(); // reads in user defined mail views from our default file + nsresult ConvertFilterListToMailViews(); + nsresult ConvertMailViewListToFilterList(); + + nsCOMArray<nsIMsgMailView> m_mailViews; + nsCOMPtr<nsIMsgFilterList> mFilterList; // our internal filter list representation +}; + +#endif diff --git a/mailnews/extensions/mailviews/src/nsMsgMailViewsCID.h b/mailnews/extensions/mailviews/src/nsMsgMailViewsCID.h new file mode 100644 index 000000000..9487e5f4c --- /dev/null +++ b/mailnews/extensions/mailviews/src/nsMsgMailViewsCID.h @@ -0,0 +1,17 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef nsMsgMailViewsCID_h__ +#define nsMsgMailViewsCID_h__ + +#define NS_MSGMAILVIEWLIST_CONTRACTID \ + "@mozilla.org/messenger/mailviewlist;1" + +#define NS_MSGMAILVIEWLIST_CID \ +{ /* A0258267-44FD-4886-A858-8192615178EC */ \ + 0xa0258267, 0x44fd, 0x4886, \ + { 0xa8, 0x58, 0x81, 0x92, 0x61, 0x51, 0x78, 0xec }} + +#endif /* nsMsgMailViewsCID_h__*/ diff --git a/mailnews/extensions/mdn/content/am-mdn.js b/mailnews/extensions/mdn/content/am-mdn.js new file mode 100644 index 000000000..4dd9e8d8d --- /dev/null +++ b/mailnews/extensions/mdn/content/am-mdn.js @@ -0,0 +1,155 @@ +/* -*- Mode: Java; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +var useCustomPrefs; +var requestReceipt; +var leaveInInbox; +var moveToSent; +var receiptSend; +var neverReturn; +var returnSome; +var notInToCcPref; +var notInToCcLabel; +var outsideDomainPref; +var outsideDomainLabel; +var otherCasesPref; +var otherCasesLabel; +var receiptArriveLabel; +var receiptRequestLabel; +var gIdentity; +var gIncomingServer; +var gMdnPrefBranch; + +function onInit() +{ + useCustomPrefs = document.getElementById("identity.use_custom_prefs"); + requestReceipt = document.getElementById("identity.request_return_receipt_on"); + leaveInInbox = document.getElementById("leave_in_inbox"); + moveToSent = document.getElementById("move_to_sent"); + receiptSend = document.getElementById("server.mdn_report_enabled"); + neverReturn = document.getElementById("never_return"); + returnSome = document.getElementById("return_some"); + notInToCcPref = document.getElementById("server.mdn_not_in_to_cc"); + notInToCcLabel = document.getElementById("notInToCcLabel"); + outsideDomainPref = document.getElementById("server.mdn_outside_domain"); + outsideDomainLabel = document.getElementById("outsideDomainLabel"); + otherCasesPref = document.getElementById("server.mdn_other"); + otherCasesLabel = document.getElementById("otherCasesLabel"); + receiptArriveLabel = document.getElementById("receiptArriveLabel"); + receiptRequestLabel = document.getElementById("receiptRequestLabel"); + + EnableDisableCustomSettings(); + + return true; +} + +function onSave() +{ + +} + +function EnableDisableCustomSettings() { + if (useCustomPrefs && (useCustomPrefs.getAttribute("value") == "false")) { + requestReceipt.setAttribute("disabled", "true"); + leaveInInbox.setAttribute("disabled", "true"); + moveToSent.setAttribute("disabled", "true"); + neverReturn.setAttribute("disabled", "true"); + returnSome.setAttribute("disabled", "true"); + receiptArriveLabel.setAttribute("disabled", "true"); + receiptRequestLabel.setAttribute("disabled", "true"); + } + else { + requestReceipt.removeAttribute("disabled"); + leaveInInbox.removeAttribute("disabled"); + moveToSent.removeAttribute("disabled"); + neverReturn.removeAttribute("disabled"); + returnSome.removeAttribute("disabled"); + receiptArriveLabel.removeAttribute("disabled"); + receiptRequestLabel.removeAttribute("disabled"); + } + EnableDisableAllowedReceipts(); + // Lock id based prefs + onLockPreference("mail.identity", gIdentity.key); + // Lock server based prefs + onLockPreference("mail.server", gIncomingServer.key); + return true; +} + +function EnableDisableAllowedReceipts() { + if (receiptSend) { + if (!neverReturn.getAttribute("disabled") && (receiptSend.getAttribute("value") != "false")) { + notInToCcPref.removeAttribute("disabled"); + notInToCcLabel.removeAttribute("disabled"); + outsideDomainPref.removeAttribute("disabled"); + outsideDomainLabel.removeAttribute("disabled"); + otherCasesPref.removeAttribute("disabled"); + otherCasesLabel.removeAttribute("disabled"); + } + else { + notInToCcPref.setAttribute("disabled", "true"); + notInToCcLabel.setAttribute("disabled", "true"); + outsideDomainPref.setAttribute("disabled", "true"); + outsideDomainLabel.setAttribute("disabled", "true"); + otherCasesPref.setAttribute("disabled", "true"); + otherCasesLabel.setAttribute("disabled", "true"); + } + } + return true; +} + +function onPreInit(account, accountValues) +{ + gIdentity = account.defaultIdentity; + gIncomingServer = account.incomingServer; +} + +// Disables xul elements that have associated preferences locked. +function onLockPreference(initPrefString, keyString) +{ + var finalPrefString; + + var allPrefElements = [ + { prefstring:"request_return_receipt_on", id:"identity.request_return_receipt_on"}, + { prefstring:"select_custom_prefs", id:"identity.select_custom_prefs"}, + { prefstring:"select_global_prefs", id:"identity.select_global_prefs"}, + { prefstring:"incorporate_return_receipt", id:"server.incorporate_return_receipt"}, + { prefstring:"never_return", id:"never_return"}, + { prefstring:"return_some", id:"return_some"}, + { prefstring:"mdn_not_in_to_cc", id:"server.mdn_not_in_to_cc"}, + { prefstring:"mdn_outside_domain", id:"server.mdn_outside_domain"}, + { prefstring:"mdn_other", id:"server.mdn_other"}, + ]; + + finalPrefString = initPrefString + "." + keyString + "."; + gMdnPrefBranch = Services.prefs.getBranch(finalPrefString); + + disableIfLocked( allPrefElements ); +} + +function disableIfLocked( prefstrArray ) +{ + for (var i=0; i<prefstrArray.length; i++) { + var id = prefstrArray[i].id; + var element = document.getElementById(id); + if (gMdnPrefBranch.prefIsLocked(prefstrArray[i].prefstring)) { + if (id == "server.incorporate_return_receipt") + { + document.getElementById("leave_in_inbox").setAttribute("disabled", "true"); + document.getElementById("move_to_sent").setAttribute("disabled", "true"); + } + else + element.setAttribute("disabled", "true"); + } + } +} + +/** + * Opens Preferences (Options) dialog on the pane and tab where + * the global receipts settings can be found. + */ +function showGlobalReceipts() { + openPrefsFromAccountManager("paneAdvanced", "generalTab", + {subdialog: "showReturnReceipts"}, "receipts_pane"); +} diff --git a/mailnews/extensions/mdn/content/am-mdn.xul b/mailnews/extensions/mdn/content/am-mdn.xul new file mode 100644 index 000000000..c290752ab --- /dev/null +++ b/mailnews/extensions/mdn/content/am-mdn.xul @@ -0,0 +1,136 @@ +<?xml version="1.0"?> + +<!-- + + This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> + +<?xml-stylesheet href="chrome://messenger/skin/accountManage.css" type="text/css"?> + +<!DOCTYPE page SYSTEM "chrome://messenger/locale/am-mdn.dtd"> + +<page xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul" + title="&pane.title;" + onload="parent.onPanelLoaded('am-mdn.xul');"> + + <vbox flex="1" style="overflow: auto;"> + <stringbundle id="bundle_smime" src="chrome://messenger/locale/am-mdn.properties"/> + <script type="application/javascript" src="chrome://messenger/content/AccountManager.js"/> + <script type="application/javascript" src="chrome://messenger/content/amUtils.js"/> + <script type="application/javascript" src="chrome://messenger/content/am-mdn.js"/> + + <dialogheader title="&pane.title;"/> + + <groupbox> + + <caption label="&pane.title;"/> + + <hbox id="prefChoices" align="center" flex="1"> + <radiogroup id="identity.use_custom_prefs" wsm_persist="true" genericattr="true" + preftype="bool" prefstring="mail.identity.%identitykey%.use_custom_prefs" + oncommand="EnableDisableCustomSettings();" flex="1"> + <radio id="identity.select_global_prefs" + value="false" + label="&useGlobalPrefs.label;" + accesskey="&useGlobalPrefs.accesskey;"/> + <hbox flex="1"> + <spacer flex="1"/> + <button id="globalReceiptsLink" + label="&globalReceipts.label;" + accesskey="&globalReceipts.accesskey;" + oncommand="showGlobalReceipts();"/> + </hbox> + <radio id="identity.select_custom_prefs" + value="true" + label="&useCustomPrefs.label;" + accesskey="&useCustomPrefs.accesskey;"/> + </radiogroup> + </hbox> + + <vbox id="returnReceiptSettings" class="indent" align="start"> + <checkbox id="identity.request_return_receipt_on" label="&requestReceipt.label;" + accesskey="&requestReceipt.accesskey;" + wsm_persist="true" genericattr="true" iscontrolcontainer="true" + preftype="bool" prefstring="mail.identity.%identitykey%.request_return_receipt_on"/> + + <separator/> + + <vbox id="receiptArrive"> + <label id="receiptArriveLabel" control="server.incorporate_return_receipt">&receiptArrive.label;</label> + <radiogroup id="server.incorporate_return_receipt" wsm_persist="true" genericattr="true" + preftype="int" prefstring="mail.server.%serverkey%.incorporate_return_receipt" + class="indent"> + <radio id="leave_in_inbox" value="0" label="&leaveIt.label;" + accesskey="&leaveIt.accesskey;"/> + <radio id="move_to_sent" value="1" label="&moveToSent.label;" + accesskey="&moveToSent.accesskey;"/> + </radiogroup> + </vbox> + + <separator/> + + <vbox id="receiptRequest"> + <label id="receiptRequestLabel" control="server.mdn_report_enabled">&requestMDN.label;</label> + <radiogroup id="server.mdn_report_enabled" wsm_persist="true" genericattr="true" + preftype="bool" prefstring="mail.server.%serverkey%.mdn_report_enabled" + oncommand="EnableDisableAllowedReceipts();" + class="indent"> + <radio id="never_return" value="false" label="&never.label;" + accesskey="&never.accesskey;"/> + <radio id="return_some" value="true" label="&returnSome.label;" + accesskey="&returnSome.accesskey;"/> + + <hbox id="receiptSendIf" class="indent"> + <grid> + <columns><column/><column/></columns> + <rows> + <row align="center"> + <label id="notInToCcLabel" value="¬InToCc.label;" + accesskey="¬InToCc.accesskey;" control="server.mdn_not_in_to_cc"/> + <menulist id="server.mdn_not_in_to_cc" wsm_persist="true" genericattr="true" + preftype="int" prefstring="mail.server.%serverkey%.mdn_not_in_to_cc"> + <menupopup> + <menuitem value="0" label="&neverSend.label;"/> + <menuitem value="1" label="&alwaysSend.label;"/> + <menuitem value="2" label="&askMe.label;"/> + </menupopup> + </menulist> + </row> + <row align="center"> + <label id="outsideDomainLabel" value="&outsideDomain.label;" + accesskey="&outsideDomain.accesskey;" control="server.mdn_outside_domain"/> + <menulist id="server.mdn_outside_domain" wsm_persist="true" genericattr="true" + preftype="int" prefstring="mail.server.%serverkey%.mdn_outside_domain"> + <menupopup> + <menuitem value="0" label="&neverSend.label;"/> + <menuitem value="1" label="&alwaysSend.label;"/> + <menuitem value="2" label="&askMe.label;"/> + </menupopup> + </menulist> + </row> + <row align="center"> + <label id="otherCasesLabel" value="&otherCases.label;" + accesskey="&otherCases.accesskey;" control="server.mdn_other"/> + <menulist id="server.mdn_other" wsm_persist="true" genericattr="true" + preftype="int" prefstring="mail.server.%serverkey%.mdn_other"> + <menupopup> + <menuitem value="0" label="&neverSend.label;"/> + <menuitem value="1" label="&alwaysSend.label;"/> + <menuitem value="2" label="&askMe.label;"/> + </menupopup> + </menulist> + </row> + </rows> + </grid> + </hbox> + </radiogroup> + + </vbox> + + </vbox> + + </groupbox> + </vbox> + +</page> diff --git a/mailnews/extensions/mdn/content/mdn.js b/mailnews/extensions/mdn/content/mdn.js new file mode 100644 index 000000000..9ac481573 --- /dev/null +++ b/mailnews/extensions/mdn/content/mdn.js @@ -0,0 +1,23 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * default prefs for mdn + */ + +pref("mail.identity.default.use_custom_prefs", false); // false: Use global true: Use custom + +pref("mail.identity.default.request_return_receipt_on", false); + +pref("mail.server.default.incorporate_return_receipt", 0); // 0: Inbox/filter 1: Sent folder + +pref("mail.server.default.mdn_report_enabled", true); // false: Never return receipts true: Return some receipts + +pref("mail.server.default.mdn_not_in_to_cc", 2); // 0: Never 1: Always 2: Ask me 3: Denial +pref("mail.server.default.mdn_outside_domain", 2); +pref("mail.server.default.mdn_other", 2); + +pref("mail.identity.default.request_receipt_header_type", 0); // return receipt header type - 0: MDN-DNT 1: RRT 2: Both + +pref("mail.server.default.mdn_report_enabled", true); diff --git a/mailnews/extensions/mdn/jar.mn b/mailnews/extensions/mdn/jar.mn new file mode 100644 index 000000000..74fde6b60 --- /dev/null +++ b/mailnews/extensions/mdn/jar.mn @@ -0,0 +1,7 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +messenger.jar: + content/messenger/am-mdn.xul (content/am-mdn.xul) + content/messenger/am-mdn.js (content/am-mdn.js) diff --git a/mailnews/extensions/mdn/moz.build b/mailnews/extensions/mdn/moz.build new file mode 100644 index 000000000..5aaa32895 --- /dev/null +++ b/mailnews/extensions/mdn/moz.build @@ -0,0 +1,12 @@ +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +DIRS += ['src'] + +JAR_MANIFESTS += ['jar.mn'] + +JS_PREFERENCE_FILES += [ + 'content/mdn.js', +] diff --git a/mailnews/extensions/mdn/src/mdn-service.js b/mailnews/extensions/mdn/src/mdn-service.js new file mode 100644 index 000000000..32e1cbde1 --- /dev/null +++ b/mailnews/extensions/mdn/src/mdn-service.js @@ -0,0 +1,24 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +Components.utils.import("resource://gre/modules/XPCOMUtils.jsm"); + +function MDNService() {} + +MDNService.prototype = { + name: "mdn", + chromePackageName: "messenger", + showPanel: function(server) { + // don't show the panel for news, rss, im or local accounts + return (server.type != "nntp" && server.type != "rss" && + server.type != "im" && server.type != "none"); + }, + + QueryInterface: XPCOMUtils.generateQI([Components.interfaces.nsIMsgAccountManagerExtension]), + classID: Components.ID("{e007d92e-1dd1-11b2-a61e-dc962c9b8571}"), +}; + +var components = [MDNService]; +var NSGetFactory = XPCOMUtils.generateNSGetFactory(components); diff --git a/mailnews/extensions/mdn/src/mdn-service.manifest b/mailnews/extensions/mdn/src/mdn-service.manifest new file mode 100644 index 000000000..493a3c69b --- /dev/null +++ b/mailnews/extensions/mdn/src/mdn-service.manifest @@ -0,0 +1,3 @@ +component {e007d92e-1dd1-11b2-a61e-dc962c9b8571} mdn-service.js +contract @mozilla.org/accountmanager/extension;1?name=mdn {e007d92e-1dd1-11b2-a61e-dc962c9b8571} +category mailnews-accountmanager-extensions mdn-account-manager-extension @mozilla.org/accountmanager/extension;1?name=mdn diff --git a/mailnews/extensions/mdn/src/moz.build b/mailnews/extensions/mdn/src/moz.build new file mode 100644 index 000000000..1aef9b208 --- /dev/null +++ b/mailnews/extensions/mdn/src/moz.build @@ -0,0 +1,16 @@ +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +SOURCES += [ + 'nsMsgMdnGenerator.cpp', +] + +EXTRA_COMPONENTS += [ + 'mdn-service.js', + 'mdn-service.manifest', +] + +FINAL_LIBRARY = 'mail' + diff --git a/mailnews/extensions/mdn/src/nsMsgMdnCID.h b/mailnews/extensions/mdn/src/nsMsgMdnCID.h new file mode 100644 index 000000000..4dd832b7f --- /dev/null +++ b/mailnews/extensions/mdn/src/nsMsgMdnCID.h @@ -0,0 +1,22 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef nsMsgMdnCID_h__ +#define nsMsgMdnCID_h__ + +#include "nsISupports.h" +#include "nsIFactory.h" +#include "nsIComponentManager.h" + +#include "nsIMsgMdnGenerator.h" + +#define NS_MSGMDNGENERATOR_CONTRACTID \ + "@mozilla.org/messenger-mdn/generator;1" +#define NS_MSGMDNGENERATOR_CID \ +{ /* ec917b13-8f73-4d4d-9146-d7f7aafe9076 */ \ + 0xec917b13, 0x8f73, 0x4d4d, \ + { 0x91, 0x46, 0xd7, 0xf7, 0xaa, 0xfe, 0x90, 0x76 }} + +#endif /* nsMsgMdnCID_h__ */ diff --git a/mailnews/extensions/mdn/src/nsMsgMdnGenerator.cpp b/mailnews/extensions/mdn/src/nsMsgMdnGenerator.cpp new file mode 100644 index 000000000..31b55fe2f --- /dev/null +++ b/mailnews/extensions/mdn/src/nsMsgMdnGenerator.cpp @@ -0,0 +1,1139 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsMsgMdnGenerator.h" +#include "nsImapCore.h" +#include "nsIMsgImapMailFolder.h" +#include "nsIMsgAccountManager.h" +#include "nsMsgBaseCID.h" +#include "nsMimeTypes.h" +#include "prprf.h" +#include "prmem.h" +#include "prsystem.h" +#include "nsMsgI18N.h" +#include "nsMailHeaders.h" +#include "nsMsgLocalFolderHdrs.h" +#include "nsIHttpProtocolHandler.h" +#include "nsISmtpService.h" // for actually sending the message... +#include "nsMsgCompCID.h" +#include "nsComposeStrings.h" +#include "nsISmtpServer.h" +#include "nsIPrompt.h" +#include "nsIMsgCompUtils.h" +#include "nsIPrefService.h" +#include "nsIPrefBranch.h" +#include "nsIStringBundle.h" +#include "nsDirectoryServiceDefs.h" +#include "nsMsgUtils.h" +#include "nsNetUtil.h" +#include "nsIMsgDatabase.h" +#include "mozilla/Services.h" +#include "nsIArray.h" +#include "nsArrayUtils.h" +#include "mozilla/mailnews/MimeHeaderParser.h" + +using namespace mozilla::mailnews; + +#define MDN_NOT_IN_TO_CC ((int) 0x0001) +#define MDN_OUTSIDE_DOMAIN ((int) 0x0002) + +#define HEADER_RETURN_PATH "Return-Path" +#define HEADER_DISPOSITION_NOTIFICATION_TO "Disposition-Notification-To" +#define HEADER_APPARENTLY_TO "Apparently-To" +#define HEADER_ORIGINAL_RECIPIENT "Original-Recipient" +#define HEADER_REPORTING_UA "Reporting-UA" +#define HEADER_MDN_GATEWAY "MDN-Gateway" +#define HEADER_FINAL_RECIPIENT "Final-Recipient" +#define HEADER_DISPOSITION "Disposition" +#define HEADER_ORIGINAL_MESSAGE_ID "Original-Message-ID" +#define HEADER_FAILURE "Failure" +#define HEADER_ERROR "Error" +#define HEADER_WARNING "Warning" +#define HEADER_RETURN_RECEIPT_TO "Return-Receipt-To" +#define HEADER_X_ACCEPT_LANGUAGE "X-Accept-Language" + +#define PUSH_N_FREE_STRING(p) \ + do { if (p) { rv = WriteString(p); PR_smprintf_free(p); p=0; \ + if (NS_FAILED(rv)) return rv; } \ + else { return NS_ERROR_OUT_OF_MEMORY; } } while (0) + +// String bundle for mdn. Class static. +#define MDN_STRINGBUNDLE_URL "chrome://messenger/locale/msgmdn.properties" + +#if defined(DEBUG_jefft) +#define DEBUG_MDN(s) printf("%s\n", s) +#else +#define DEBUG_MDN(s) +#endif + +// machine parsible string; should not be localized +char DispositionTypes[7][16] = { + "displayed", + "dispatched", + "processed", + "deleted", + "denied", + "failed", + "" +}; + +NS_IMPL_ISUPPORTS(nsMsgMdnGenerator, nsIMsgMdnGenerator, nsIUrlListener) + +nsMsgMdnGenerator::nsMsgMdnGenerator() +{ + m_disposeType = eDisplayed; + m_outputStream = nullptr; + m_reallySendMdn = false; + m_autoSend = false; + m_autoAction = false; + m_mdnEnabled = false; + m_notInToCcOp = eNeverSendOp; + m_outsideDomainOp = eNeverSendOp; + m_otherOp = eNeverSendOp; +} + +nsMsgMdnGenerator::~nsMsgMdnGenerator() +{ +} + +nsresult nsMsgMdnGenerator::FormatStringFromName(const char16_t *aName, + const char16_t *aString, + char16_t **aResultString) +{ + DEBUG_MDN("nsMsgMdnGenerator::FormatStringFromName"); + + nsCOMPtr<nsIStringBundleService> bundleService = + mozilla::services::GetStringBundleService(); + NS_ENSURE_TRUE(bundleService, NS_ERROR_UNEXPECTED); + + nsCOMPtr <nsIStringBundle> bundle; + nsresult rv = bundleService->CreateBundle(MDN_STRINGBUNDLE_URL, + getter_AddRefs(bundle)); + NS_ENSURE_SUCCESS(rv,rv); + + const char16_t *formatStrings[1] = { aString }; + rv = bundle->FormatStringFromName(aName, + formatStrings, 1, aResultString); + NS_ENSURE_SUCCESS(rv,rv); + return rv; +} + +nsresult nsMsgMdnGenerator::GetStringFromName(const char16_t *aName, + char16_t **aResultString) +{ + DEBUG_MDN("nsMsgMdnGenerator::GetStringFromName"); + + nsCOMPtr<nsIStringBundleService> bundleService = + mozilla::services::GetStringBundleService(); + NS_ENSURE_TRUE(bundleService, NS_ERROR_UNEXPECTED); + + nsCOMPtr <nsIStringBundle> bundle; + nsresult rv = bundleService->CreateBundle(MDN_STRINGBUNDLE_URL, + getter_AddRefs(bundle)); + NS_ENSURE_SUCCESS(rv,rv); + + rv = bundle->GetStringFromName(aName, aResultString); + NS_ENSURE_SUCCESS(rv,rv); + return rv; +} + +nsresult nsMsgMdnGenerator::StoreMDNSentFlag(nsIMsgFolder *folder, + nsMsgKey key) +{ + DEBUG_MDN("nsMsgMdnGenerator::StoreMDNSentFlag"); + + nsCOMPtr<nsIMsgDatabase> msgDB; + nsresult rv = folder->GetMsgDatabase(getter_AddRefs(msgDB)); + NS_ENSURE_SUCCESS(rv, rv); + rv = msgDB->MarkMDNSent(key, true, nullptr); + + nsCOMPtr<nsIMsgImapMailFolder> imapFolder = do_QueryInterface(folder); + // Store the $MDNSent flag if the folder is an Imap Mail Folder + if (imapFolder) + return imapFolder->StoreImapFlags(kImapMsgMDNSentFlag, true, &key, 1, nullptr); + return rv; +} + +nsresult nsMsgMdnGenerator::ClearMDNNeededFlag(nsIMsgFolder *folder, + nsMsgKey key) +{ + DEBUG_MDN("nsMsgMdnGenerator::ClearMDNNeededFlag"); + + nsCOMPtr<nsIMsgDatabase> msgDB; + nsresult rv = folder->GetMsgDatabase(getter_AddRefs(msgDB)); + NS_ENSURE_SUCCESS(rv, rv); + return msgDB->MarkMDNNeeded(key, false, nullptr); +} + +bool nsMsgMdnGenerator::ProcessSendMode() +{ + DEBUG_MDN("nsMsgMdnGenerator::ProcessSendMode"); + int32_t miscState = 0; + + if (m_identity) + { + m_identity->GetEmail(m_email); + if (m_email.IsEmpty()) + return m_reallySendMdn; + + const char *accountDomain = strchr(m_email.get(), '@'); + if (!accountDomain) + return m_reallySendMdn; + + if (MailAddrMatch(m_email.get(), m_dntRrt.get())) // return address is self, don't send + return false; + + // *** fix me see Bug 132504 for more information + // *** what if the message has been filtered to different account + if (!PL_strcasestr(m_dntRrt.get(), accountDomain)) + miscState |= MDN_OUTSIDE_DOMAIN; + if (NotInToOrCc()) + miscState |= MDN_NOT_IN_TO_CC; + m_reallySendMdn = true; + // ********* + // How are we gona deal with the auto forwarding issues? Some server + // didn't bother to add addition header or modify existing header to + // thev message when forwarding. They simply copy the exact same + // message to another user's mailbox. Some change To: to + // Apparently-To: + // Unfortunately, there is nothing we can do. It's out of our control. + // ********* + // starting from lowest denominator to highest + if (!miscState) + { // under normal situation: recipent is in to and cc list, + // and the sender is from the same domain + switch (m_otherOp) + { + default: + case eNeverSendOp: + m_reallySendMdn = false; + break; + case eAutoSendOp: + m_autoSend = true; + break; + case eAskMeOp: + m_autoSend = false; + break; + case eDeniedOp: + m_autoSend = true; + m_disposeType = eDenied; + break; + } + } + else if (miscState == (MDN_OUTSIDE_DOMAIN | MDN_NOT_IN_TO_CC)) + { + if (m_outsideDomainOp != m_notInToCcOp) + { + m_autoSend = false; // ambiguous; always ask user + } + else + { + switch (m_outsideDomainOp) + { + default: + case eNeverSendOp: + m_reallySendMdn = false; + break; + case eAutoSendOp: + m_autoSend = true; + break; + case eAskMeOp: + m_autoSend = false; + break; + } + } + } + else if (miscState & MDN_OUTSIDE_DOMAIN) + { + switch (m_outsideDomainOp) + { + default: + case eNeverSendOp: + m_reallySendMdn = false; + break; + case eAutoSendOp: + m_autoSend = true; + break; + case eAskMeOp: + m_autoSend = false; + break; + } + } + else if (miscState & MDN_NOT_IN_TO_CC) + { + switch (m_notInToCcOp) + { + default: + case eNeverSendOp: + m_reallySendMdn = false; + break; + case eAutoSendOp: + m_autoSend = true; + break; + case eAskMeOp: + m_autoSend = false; + break; + } + } + } + return m_reallySendMdn; +} + +bool nsMsgMdnGenerator::MailAddrMatch(const char *addr1, const char *addr2) +{ + // Comparing two email addresses returns true if matched; local/account + // part comparison is case sensitive; domain part comparison is case + // insensitive + DEBUG_MDN("nsMsgMdnGenerator::MailAddrMatch"); + bool isMatched = true; + const char *atSign1 = nullptr, *atSign2 = nullptr; + const char *lt = nullptr, *local1 = nullptr, *local2 = nullptr; + const char *end1 = nullptr, *end2 = nullptr; + + if (!addr1 || !addr2) + return false; + + lt = strchr(addr1, '<'); + local1 = !lt ? addr1 : lt+1; + lt = strchr(addr2, '<'); + local2 = !lt ? addr2 : lt+1; + end1 = strchr(local1, '>'); + if (!end1) + end1 = addr1 + strlen(addr1); + end2 = strchr(local2, '>'); + if (!end2) + end2 = addr2 + strlen(addr2); + atSign1 = strchr(local1, '@'); + atSign2 = strchr(local2, '@'); + if (!atSign1 || !atSign2 // ill formed addr spec + || (atSign1 - local1) != (atSign2 - local2)) + isMatched = false; + else if (strncmp(local1, local2, (atSign1-local1))) // case sensitive + // compare for local part + isMatched = false; + else if ((end1 - atSign1) != (end2 - atSign2) || + PL_strncasecmp(atSign1, atSign2, (end1-atSign1))) // case + // insensitive compare for domain part + isMatched = false; + return isMatched; +} + +bool nsMsgMdnGenerator::NotInToOrCc() +{ + DEBUG_MDN("nsMsgMdnGenerator::NotInToOrCc"); + nsCString reply_to; + nsCString to; + nsCString cc; + + m_identity->GetReplyTo(reply_to); + m_headers->ExtractHeader(HEADER_TO, true, to); + m_headers->ExtractHeader(HEADER_CC, true, cc); + + // start with a simple check + if ((!to.IsEmpty() && PL_strcasestr(to.get(), m_email.get())) || + (!cc.IsEmpty() && PL_strcasestr(cc.get(), m_email.get()))) { + return false; + } + + if ((!reply_to.IsEmpty() && !to.IsEmpty() && PL_strcasestr(to.get(), reply_to.get())) || + (!reply_to.IsEmpty() && !cc.IsEmpty() && PL_strcasestr(cc.get(), reply_to.get()))) { + return false; + } + return true; +} + +bool nsMsgMdnGenerator::ValidateReturnPath() +{ + DEBUG_MDN("nsMsgMdnGenerator::ValidateReturnPath"); + // ValidateReturnPath applies to Automatic Send Mode only. If we were not + // in auto send mode we simply by passing the check + if (!m_autoSend) + return m_reallySendMdn; + + nsCString returnPath; + m_headers->ExtractHeader(HEADER_RETURN_PATH, false, returnPath); + if (returnPath.IsEmpty()) + { + m_autoSend = false; + return m_reallySendMdn; + } + m_autoSend = MailAddrMatch(returnPath.get(), m_dntRrt.get()); + return m_reallySendMdn; +} + +nsresult nsMsgMdnGenerator::CreateMdnMsg() +{ + DEBUG_MDN("nsMsgMdnGenerator::CreateMdnMsg"); + nsresult rv; + + nsCOMPtr<nsIFile> tmpFile; + rv = GetSpecialDirectoryWithFileName(NS_OS_TEMP_DIR, + "mdnmsg", + getter_AddRefs(m_file)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = m_file->CreateUnique(nsIFile::NORMAL_FILE_TYPE, 00600); + NS_ENSURE_SUCCESS(rv, rv); + rv = MsgNewBufferedFileOutputStream(getter_AddRefs(m_outputStream), + m_file, + PR_CREATE_FILE | PR_WRONLY | PR_TRUNCATE, + 0664); + NS_ASSERTION(NS_SUCCEEDED(rv),"creating mdn: failed to output stream"); + if (NS_FAILED(rv)) + return NS_OK; + + rv = CreateFirstPart(); + if (NS_SUCCEEDED(rv)) + { + rv = CreateSecondPart(); + if (NS_SUCCEEDED(rv)) + rv = CreateThirdPart(); + } + + if (m_outputStream) + { + m_outputStream->Flush(); + m_outputStream->Close(); + } + if (NS_FAILED(rv)) + m_file->Remove(false); + else + rv = SendMdnMsg(); + + return NS_OK; +} + +nsresult nsMsgMdnGenerator::CreateFirstPart() +{ + DEBUG_MDN("nsMsgMdnGenerator::CreateFirstPart"); + char *convbuf = nullptr, *tmpBuffer = nullptr; + char *parm = nullptr; + nsString firstPart1; + nsString firstPart2; + nsresult rv = NS_OK; + nsCOMPtr <nsIMsgCompUtils> compUtils; + + if (m_mimeSeparator.IsEmpty()) + { + compUtils = do_GetService(NS_MSGCOMPUTILS_CONTRACTID, &rv); + NS_ENSURE_SUCCESS(rv, rv); + rv = compUtils->MimeMakeSeparator("mdn", getter_Copies(m_mimeSeparator)); + NS_ENSURE_SUCCESS(rv, rv); + } + if (m_mimeSeparator.IsEmpty()) + return NS_ERROR_OUT_OF_MEMORY; + + tmpBuffer = (char *) PR_CALLOC(256); + + if (!tmpBuffer) + return NS_ERROR_OUT_OF_MEMORY; + + PRExplodedTime now; + PR_ExplodeTime(PR_Now(), PR_LocalTimeParameters, &now); + + int gmtoffset = (now.tm_params.tp_gmt_offset + now.tm_params.tp_dst_offset) + / 60; + /* Use PR_FormatTimeUSEnglish() to format the date in US English format, + then figure out what our local GMT offset is, and append it (since + PR_FormatTimeUSEnglish() can't do that.) Generate four digit years as + per RFC 1123 (superceding RFC 822.) + */ + PR_FormatTimeUSEnglish(tmpBuffer, 100, + "Date: %a, %d %b %Y %H:%M:%S ", + &now); + + PR_snprintf(tmpBuffer + strlen(tmpBuffer), 100, + "%c%02d%02d" CRLF, + (gmtoffset >= 0 ? '+' : '-'), + ((gmtoffset >= 0 ? gmtoffset : -gmtoffset) / 60), + ((gmtoffset >= 0 ? gmtoffset : -gmtoffset) % 60)); + + rv = WriteString(tmpBuffer); + PR_Free(tmpBuffer); + if (NS_FAILED(rv)) + return rv; + + bool conformToStandard = false; + if (compUtils) + compUtils->GetMsgMimeConformToStandard(&conformToStandard); + + nsString fullName; + m_identity->GetFullName(fullName); + + nsCString fullAddress; + // convert fullName to UTF8 before passing it to MakeMimeAddress + MakeMimeAddress(NS_ConvertUTF16toUTF8(fullName), m_email, fullAddress); + + convbuf = nsMsgI18NEncodeMimePartIIStr(fullAddress.get(), + true, m_charset.get(), 0, conformToStandard); + + parm = PR_smprintf("From: %s" CRLF, convbuf ? convbuf : m_email.get()); + + rv = FormatStringFromName(u"MsgMdnMsgSentTo", NS_ConvertASCIItoUTF16(m_email).get(), + getter_Copies(firstPart1)); + if (NS_FAILED(rv)) + return rv; + + PUSH_N_FREE_STRING (parm); + + PR_Free(convbuf); + + if (compUtils) + { + nsCString msgId; + rv = compUtils->MsgGenerateMessageId(m_identity, getter_Copies(msgId)); + tmpBuffer = PR_smprintf("Message-ID: %s" CRLF, msgId.get()); + PUSH_N_FREE_STRING(tmpBuffer); + } + + nsString receipt_string; + switch (m_disposeType) + { + case nsIMsgMdnGenerator::eDisplayed: + rv = GetStringFromName( + u"MdnDisplayedReceipt", + getter_Copies(receipt_string)); + break; + case nsIMsgMdnGenerator::eDispatched: + rv = GetStringFromName( + u"MdnDispatchedReceipt", + getter_Copies(receipt_string)); + break; + case nsIMsgMdnGenerator::eProcessed: + rv = GetStringFromName( + u"MdnProcessedReceipt", + getter_Copies(receipt_string)); + break; + case nsIMsgMdnGenerator::eDeleted: + rv = GetStringFromName( + u"MdnDeletedReceipt", + getter_Copies(receipt_string)); + break; + case nsIMsgMdnGenerator::eDenied: + rv = GetStringFromName( + u"MdnDeniedReceipt", + getter_Copies(receipt_string)); + break; + case nsIMsgMdnGenerator::eFailed: + rv = GetStringFromName( + u"MdnFailedReceipt", + getter_Copies(receipt_string)); + break; + default: + rv = NS_ERROR_INVALID_ARG; + break; + } + + if (NS_FAILED(rv)) + return rv; + + receipt_string.AppendLiteral(" - "); + + char * encodedReceiptString = nsMsgI18NEncodeMimePartIIStr(NS_ConvertUTF16toUTF8(receipt_string).get(), false, + "UTF-8", 0, conformToStandard); + + nsCString subject; + m_headers->ExtractHeader(HEADER_SUBJECT, false, subject); + convbuf = nsMsgI18NEncodeMimePartIIStr(subject.Length() ? subject.get() : "[no subject]", + false, m_charset.get(), 0, conformToStandard); + tmpBuffer = PR_smprintf("Subject: %s%s" CRLF, + encodedReceiptString, + (convbuf ? convbuf : (subject.Length() ? subject.get() : + "[no subject]"))); + + PUSH_N_FREE_STRING(tmpBuffer); + PR_Free(convbuf); + PR_Free(encodedReceiptString); + + convbuf = nsMsgI18NEncodeMimePartIIStr(m_dntRrt.get(), true, m_charset.get(), 0, conformToStandard); + tmpBuffer = PR_smprintf("To: %s" CRLF, convbuf ? convbuf : + m_dntRrt.get()); + PUSH_N_FREE_STRING(tmpBuffer); + + PR_Free(convbuf); + + // *** This is not in the spec. I am adding this so we could do + // threading + m_headers->ExtractHeader(HEADER_MESSAGE_ID, false, m_messageId); + + if (!m_messageId.IsEmpty()) + { + if (*m_messageId.get() == '<') + tmpBuffer = PR_smprintf("References: %s" CRLF, m_messageId.get()); + else + tmpBuffer = PR_smprintf("References: <%s>" CRLF, m_messageId.get()); + PUSH_N_FREE_STRING(tmpBuffer); + } + tmpBuffer = PR_smprintf("%s" CRLF, "MIME-Version: 1.0"); + PUSH_N_FREE_STRING(tmpBuffer); + + tmpBuffer = PR_smprintf("Content-Type: multipart/report; \ +report-type=disposition-notification;\r\n\tboundary=\"%s\"" CRLF CRLF, + m_mimeSeparator.get()); + PUSH_N_FREE_STRING(tmpBuffer); + + tmpBuffer = PR_smprintf("--%s" CRLF, m_mimeSeparator.get()); + PUSH_N_FREE_STRING(tmpBuffer); + + tmpBuffer = PR_smprintf("Content-Type: text/plain; charset=UTF-8" CRLF); + PUSH_N_FREE_STRING(tmpBuffer); + + tmpBuffer = PR_smprintf("Content-Transfer-Encoding: %s" CRLF CRLF, + ENCODING_8BIT); + PUSH_N_FREE_STRING(tmpBuffer); + + if (!firstPart1.IsEmpty()) + { + tmpBuffer = PR_smprintf("%s" CRLF CRLF, NS_ConvertUTF16toUTF8(firstPart1).get()); + PUSH_N_FREE_STRING(tmpBuffer); + } + + switch (m_disposeType) + { + case nsIMsgMdnGenerator::eDisplayed: + rv = GetStringFromName( + u"MsgMdnDisplayed", + getter_Copies(firstPart2)); + break; + case nsIMsgMdnGenerator::eDispatched: + rv = GetStringFromName( + u"MsgMdnDispatched", + getter_Copies(firstPart2)); + break; + case nsIMsgMdnGenerator::eProcessed: + rv = GetStringFromName( + u"MsgMdnProcessed", + getter_Copies(firstPart2)); + break; + case nsIMsgMdnGenerator::eDeleted: + rv = GetStringFromName( + u"MsgMdnDeleted", + getter_Copies(firstPart2)); + break; + case nsIMsgMdnGenerator::eDenied: + rv = GetStringFromName( + u"MsgMdnDenied", + getter_Copies(firstPart2)); + break; + case nsIMsgMdnGenerator::eFailed: + rv = GetStringFromName( + u"MsgMdnFailed", + getter_Copies(firstPart2)); + break; + default: + rv = NS_ERROR_INVALID_ARG; + break; + } + + if (NS_FAILED(rv)) + return rv; + + if (!firstPart2.IsEmpty()) + { + tmpBuffer = + PR_smprintf("%s" CRLF CRLF, + NS_ConvertUTF16toUTF8(firstPart2).get()); + PUSH_N_FREE_STRING(tmpBuffer); + } + + return rv; +} + +nsresult nsMsgMdnGenerator::CreateSecondPart() +{ + DEBUG_MDN("nsMsgMdnGenerator::CreateSecondPart"); + char *tmpBuffer = nullptr; + char *convbuf = nullptr; + nsresult rv = NS_OK; + nsCOMPtr <nsIMsgCompUtils> compUtils; + bool conformToStandard = false; + + tmpBuffer = PR_smprintf("--%s" CRLF, m_mimeSeparator.get()); + PUSH_N_FREE_STRING(tmpBuffer); + + tmpBuffer = PR_smprintf("%s" CRLF, "Content-Type: message/disposition-notification; name=\042MDNPart2.txt\042"); + PUSH_N_FREE_STRING(tmpBuffer); + + tmpBuffer = PR_smprintf("%s" CRLF, "Content-Disposition: inline"); + PUSH_N_FREE_STRING(tmpBuffer); + + tmpBuffer = PR_smprintf("Content-Transfer-Encoding: %s" CRLF CRLF, + ENCODING_7BIT); + PUSH_N_FREE_STRING(tmpBuffer); + + nsCOMPtr<nsIHttpProtocolHandler> pHTTPHandler = + do_GetService(NS_NETWORK_PROTOCOL_CONTRACTID_PREFIX "http", &rv); + if (NS_SUCCEEDED(rv) && pHTTPHandler) + { + nsAutoCString userAgentString; + pHTTPHandler->GetUserAgent(userAgentString); + + if (!userAgentString.IsEmpty()) + { + // Prepend the product name with the dns name according to RFC 3798. + char hostName[256]; + PR_GetSystemInfo(PR_SI_HOSTNAME_UNTRUNCATED, hostName, sizeof hostName); + if ((hostName[0] != '\0') && (strchr(hostName, '.') != NULL)) + { + userAgentString.Insert("; ", 0); + userAgentString.Insert(nsDependentCString(hostName), 0); + } + + tmpBuffer = PR_smprintf("Reporting-UA: %s" CRLF, + userAgentString.get()); + PUSH_N_FREE_STRING(tmpBuffer); + } + } + + nsCString originalRecipient; + m_headers->ExtractHeader(HEADER_ORIGINAL_RECIPIENT, false, + originalRecipient); + + if (!originalRecipient.IsEmpty()) + { + tmpBuffer = PR_smprintf("Original-Recipient: %s" CRLF, + originalRecipient.get()); + PUSH_N_FREE_STRING(tmpBuffer); + } + + compUtils = do_GetService(NS_MSGCOMPUTILS_CONTRACTID, &rv); + if (compUtils) + compUtils->GetMsgMimeConformToStandard(&conformToStandard); + + convbuf = nsMsgI18NEncodeMimePartIIStr( + m_email.get(), true, m_charset.get(), 0, + conformToStandard); + tmpBuffer = PR_smprintf("Final-Recipient: rfc822;%s" CRLF, convbuf ? + convbuf : m_email.get()); + PUSH_N_FREE_STRING(tmpBuffer); + + PR_Free (convbuf); + + if (*m_messageId.get() == '<') + tmpBuffer = PR_smprintf("Original-Message-ID: %s" CRLF, m_messageId.get()); + else + tmpBuffer = PR_smprintf("Original-Message-ID: <%s>" CRLF, m_messageId.get()); + PUSH_N_FREE_STRING(tmpBuffer); + + tmpBuffer = PR_smprintf("Disposition: %s/%s; %s" CRLF CRLF, + (m_autoAction ? "automatic-action" : + "manual-action"), + (m_autoSend ? "MDN-sent-automatically" : + "MDN-sent-manually"), + DispositionTypes[(int) m_disposeType]); + PUSH_N_FREE_STRING(tmpBuffer); + + return rv; +} + +nsresult nsMsgMdnGenerator::CreateThirdPart() +{ + DEBUG_MDN("nsMsgMdnGenerator::CreateThirdPart"); + char *tmpBuffer = nullptr; + nsresult rv = NS_OK; + + tmpBuffer = PR_smprintf("--%s" CRLF, m_mimeSeparator.get()); + PUSH_N_FREE_STRING(tmpBuffer); + + tmpBuffer = PR_smprintf("%s" CRLF, "Content-Type: text/rfc822-headers; name=\042MDNPart3.txt\042"); + PUSH_N_FREE_STRING(tmpBuffer); + + tmpBuffer = PR_smprintf("%s" CRLF, "Content-Transfer-Encoding: 7bit"); + PUSH_N_FREE_STRING(tmpBuffer); + + tmpBuffer = PR_smprintf("%s" CRLF CRLF, "Content-Disposition: inline"); + PUSH_N_FREE_STRING(tmpBuffer); + + rv = OutputAllHeaders(); + + if (NS_FAILED(rv)) + return rv; + + rv = WriteString(CRLF); + if (NS_FAILED(rv)) + return rv; + + tmpBuffer = PR_smprintf("--%s--" CRLF, m_mimeSeparator.get()); + PUSH_N_FREE_STRING(tmpBuffer); + + return rv; +} + + +nsresult nsMsgMdnGenerator::OutputAllHeaders() +{ + DEBUG_MDN("nsMsgMdnGenerator::OutputAllHeaders"); + nsCString all_headers; + int32_t all_headers_size = 0; + nsresult rv = NS_OK; + + rv = m_headers->GetAllHeaders(all_headers); + if (NS_FAILED(rv)) + return rv; + all_headers_size = all_headers.Length(); + char *buf = (char *) all_headers.get(), + *buf_end = (char *) all_headers.get()+all_headers_size; + char *start = buf, *end = buf; + + while (buf < buf_end) + { + switch (*buf) + { + case 0: + if (*(buf+1) == '\n') + { + // *buf = '\r'; + end = buf; + } + else if (*(buf+1) == 0) + { + // the case of message id + *buf = '>'; + } + break; + case '\r': + end = buf; + *buf = 0; + break; + case '\n': + if (buf > start && *(buf-1) == 0) + { + start = buf + 1; + end = start; + } + else + { + end = buf; + } + *buf = 0; + break; + default: + break; + } + buf++; + + if (end > start && *end == 0) + { + // strip out private X-Mozilla-Status header & X-Mozilla-Draft-Info && envelope header + if (!PL_strncasecmp(start, X_MOZILLA_STATUS, X_MOZILLA_STATUS_LEN) + || !PL_strncasecmp(start, X_MOZILLA_DRAFT_INFO, X_MOZILLA_DRAFT_INFO_LEN) + || !PL_strncasecmp(start, "From ", 5)) + { + while ( end < buf_end && + (*end == '\n' || *end == '\r' || *end == 0)) + end++; + start = end; + } + else + { + NS_ASSERTION (*end == 0, "content of end should be null"); + rv = WriteString(start); + if (NS_FAILED(rv)) + return rv; + rv = WriteString(CRLF); + while ( end < buf_end && + (*end == '\n' || *end == '\r' || *end == 0)) + end++; + start = end; + } + buf = start; + } + } + return NS_OK; +} + +nsresult nsMsgMdnGenerator::SendMdnMsg() +{ + DEBUG_MDN("nsMsgMdnGenerator::SendMdnMsg"); + nsresult rv; + nsCOMPtr<nsISmtpService> smtpService = do_GetService(NS_SMTPSERVICE_CONTRACTID, &rv); + NS_ENSURE_SUCCESS(rv,rv); + + nsCOMPtr<nsIRequest> aRequest; + smtpService->SendMailMessage(m_file, m_dntRrt.get(), m_identity, + nullptr, this, nullptr, nullptr, false, nullptr, + getter_AddRefs(aRequest)); + + return NS_OK; +} + +nsresult nsMsgMdnGenerator::WriteString( const char *str ) +{ + NS_ENSURE_ARG (str); + uint32_t len = strlen(str); + uint32_t wLen = 0; + + return m_outputStream->Write(str, len, &wLen); +} + +nsresult nsMsgMdnGenerator::InitAndProcess(bool *needToAskUser) +{ + DEBUG_MDN("nsMsgMdnGenerator::InitAndProcess"); + nsresult rv = m_folder->GetServer(getter_AddRefs(m_server)); + nsCOMPtr<nsIMsgAccountManager> accountManager = + do_GetService(NS_MSGACCOUNTMANAGER_CONTRACTID, &rv); + if (accountManager && m_server) + { + if (!m_identity) + { + // check if this is a message delivered to the global inbox, + // in which case we find the originating account's identity. + nsCString accountKey; + m_headers->ExtractHeader(HEADER_X_MOZILLA_ACCOUNT_KEY, false, + accountKey); + nsCOMPtr <nsIMsgAccount> account; + if (!accountKey.IsEmpty()) + accountManager->GetAccount(accountKey, getter_AddRefs(account)); + if (account) + account->GetIncomingServer(getter_AddRefs(m_server)); + + if (m_server) + { + // Find the correct identity based on the "To:" and "Cc:" header + nsCString mailTo; + nsCString mailCC; + m_headers->ExtractHeader(HEADER_TO, true, mailTo); + m_headers->ExtractHeader(HEADER_CC, true, mailCC); + nsCOMPtr<nsIArray> servIdentities; + accountManager->GetIdentitiesForServer(m_server, getter_AddRefs(servIdentities)); + if (servIdentities) + { + nsCOMPtr<nsIMsgIdentity> ident; + nsCString identEmail; + uint32_t count = 0; + servIdentities->GetLength(&count); + // First check in the "To:" header + for (uint32_t i = 0; i < count; i++) + { + ident = do_QueryElementAt(servIdentities, i, &rv); + if (NS_FAILED(rv)) + continue; + ident->GetEmail(identEmail); + if (!mailTo.IsEmpty() && !identEmail.IsEmpty() && + mailTo.Find(identEmail, CaseInsensitiveCompare) != kNotFound) + { + m_identity = ident; + break; + } + } + // If no match, check the "Cc:" header + if (!m_identity) + { + for (uint32_t i = 0; i < count; i++) + { + rv = servIdentities->QueryElementAt(i, NS_GET_IID(nsIMsgIdentity),getter_AddRefs(ident)); + if (NS_FAILED(rv)) + continue; + ident->GetEmail(identEmail); + if (!mailCC.IsEmpty() && !identEmail.IsEmpty() && + mailCC.Find(identEmail, CaseInsensitiveCompare) != kNotFound) + { + m_identity = ident; + break; + } + } + } + } + + // If no match again, use the first identity + if (!m_identity) + rv = accountManager->GetFirstIdentityForServer(m_server, getter_AddRefs(m_identity)); + } + } + NS_ENSURE_SUCCESS(rv,rv); + + if (m_identity) + { + bool useCustomPrefs = false; + m_identity->GetBoolAttribute("use_custom_prefs", &useCustomPrefs); + if (useCustomPrefs) + { + bool bVal = false; + m_server->GetBoolValue("mdn_report_enabled", &bVal); + m_mdnEnabled = bVal; + m_server->GetIntValue("mdn_not_in_to_cc", &m_notInToCcOp); + m_server->GetIntValue("mdn_outside_domain", + &m_outsideDomainOp); + m_server->GetIntValue("mdn_other", &m_otherOp); + } + else + { + bool bVal = false; + + nsCOMPtr<nsIPrefBranch> prefBranch(do_GetService(NS_PREFSERVICE_CONTRACTID, &rv)); + if (NS_FAILED(rv)) + return rv; + + if(prefBranch) + { + prefBranch->GetBoolPref("mail.mdn.report.enabled", + &bVal); + m_mdnEnabled = bVal; + prefBranch->GetIntPref("mail.mdn.report.not_in_to_cc", + &m_notInToCcOp); + prefBranch->GetIntPref("mail.mdn.report.outside_domain", + &m_outsideDomainOp); + prefBranch->GetIntPref("mail.mdn.report.other", + &m_otherOp); + } + } + } + } + + rv = m_folder->GetCharset(m_charset); + if (m_mdnEnabled) + { + m_headers->ExtractHeader(HEADER_DISPOSITION_NOTIFICATION_TO, false, + m_dntRrt); + if (m_dntRrt.IsEmpty()) + m_headers->ExtractHeader(HEADER_RETURN_RECEIPT_TO, false, + m_dntRrt); + if (!m_dntRrt.IsEmpty() && ProcessSendMode() && ValidateReturnPath()) + { + if (!m_autoSend) + { + *needToAskUser = true; + rv = NS_OK; + } + else + { + *needToAskUser = false; + rv = UserAgreed(); + } + } + } + return rv; +} + +NS_IMETHODIMP nsMsgMdnGenerator::Process(EDisposeType type, + nsIMsgWindow *aWindow, + nsIMsgFolder *folder, + nsMsgKey key, + nsIMimeHeaders *headers, + bool autoAction, + bool *_retval) +{ + DEBUG_MDN("nsMsgMdnGenerator::Process"); + NS_ENSURE_ARG_POINTER(folder); + NS_ENSURE_ARG_POINTER(headers); + NS_ENSURE_ARG_POINTER(aWindow); + NS_ENSURE_TRUE(key != nsMsgKey_None, NS_ERROR_INVALID_ARG); + m_disposeType = type; + m_autoAction = autoAction; + m_window = aWindow; + m_folder = folder; + m_headers = headers; + m_key = key; + + mozilla::DebugOnly<nsresult> rv = InitAndProcess(_retval); + NS_ASSERTION(NS_SUCCEEDED(rv), "InitAndProcess failed"); + return NS_OK; +} + +NS_IMETHODIMP nsMsgMdnGenerator::UserAgreed() +{ + DEBUG_MDN("nsMsgMdnGenerator::UserAgreed"); + (void) NoteMDNRequestHandled(); + return CreateMdnMsg(); +} + +NS_IMETHODIMP nsMsgMdnGenerator::UserDeclined() +{ + DEBUG_MDN("nsMsgMdnGenerator::UserDeclined"); + return NoteMDNRequestHandled(); +} + +/** + * Set/clear flags appropriately so we won't ask user again about MDN + * request for this message. + */ +nsresult nsMsgMdnGenerator::NoteMDNRequestHandled() +{ + nsresult rv = StoreMDNSentFlag(m_folder, m_key); + NS_ASSERTION(NS_SUCCEEDED(rv), "StoreMDNSentFlag failed"); + rv = ClearMDNNeededFlag(m_folder, m_key); + NS_ASSERTION(NS_SUCCEEDED(rv), "ClearMDNNeededFlag failed"); + return rv; +} + +NS_IMETHODIMP nsMsgMdnGenerator::OnStartRunningUrl(nsIURI *url) +{ + DEBUG_MDN("nsMsgMdnGenerator::OnStartRunningUrl"); + return NS_OK; +} + +NS_IMETHODIMP nsMsgMdnGenerator::OnStopRunningUrl(nsIURI *url, + nsresult aExitCode) +{ + nsresult rv; + + DEBUG_MDN("nsMsgMdnGenerator::OnStopRunningUrl"); + if (m_file) + m_file->Remove(false); + + if (NS_SUCCEEDED(aExitCode)) + return NS_OK; + + const char16_t* exitString; + + switch (aExitCode) + { + case NS_ERROR_UNKNOWN_HOST: + case NS_ERROR_UNKNOWN_PROXY_HOST: + exitString = u"smtpSendFailedUnknownServer"; + break; + case NS_ERROR_CONNECTION_REFUSED: + case NS_ERROR_PROXY_CONNECTION_REFUSED: + exitString = u"smtpSendRequestRefused"; + break; + case NS_ERROR_NET_INTERRUPT: + case NS_ERROR_ABORT: // we have no proper string for error code NS_ERROR_ABORT in compose bundle + exitString = u"smtpSendInterrupted"; + break; + case NS_ERROR_NET_TIMEOUT: + case NS_ERROR_NET_RESET: + exitString = u"smtpSendTimeout"; + break; + default: + exitString = errorStringNameForErrorCode(aExitCode); + break; + } + + nsCOMPtr<nsISmtpService> smtpService(do_GetService(NS_SMTPSERVICE_CONTRACTID, &rv)); + NS_ENSURE_SUCCESS(rv,rv); + + // Get the smtp hostname and format the string. + nsCString smtpHostName; + nsCOMPtr<nsISmtpServer> smtpServer; + rv = smtpService->GetServerByIdentity(m_identity, getter_AddRefs(smtpServer)); + if (NS_SUCCEEDED(rv)) + smtpServer->GetHostname(smtpHostName); + + nsAutoString hostStr; + CopyASCIItoUTF16(smtpHostName, hostStr); + const char16_t *params[] = { hostStr.get() }; + + nsCOMPtr<nsIStringBundle> bundle; + nsCOMPtr<nsIStringBundleService> bundleService = + mozilla::services::GetStringBundleService(); + NS_ENSURE_TRUE(bundleService, NS_ERROR_UNEXPECTED); + + rv = bundleService->CreateBundle( + "chrome://messenger/locale/messengercompose/composeMsgs.properties", + getter_AddRefs(bundle)); + NS_ENSURE_SUCCESS(rv, rv); + + nsString failed_msg, dialogTitle; + + bundle->FormatStringFromName(exitString, params, 1, getter_Copies(failed_msg)); + bundle->GetStringFromName(u"sendMessageErrorTitle", getter_Copies(dialogTitle)); + + nsCOMPtr<nsIPrompt> dialog; + rv = m_window->GetPromptDialog(getter_AddRefs(dialog)); + if (NS_SUCCEEDED(rv)) + dialog->Alert(dialogTitle.get(),failed_msg.get()); + + return NS_OK; +} diff --git a/mailnews/extensions/mdn/src/nsMsgMdnGenerator.h b/mailnews/extensions/mdn/src/nsMsgMdnGenerator.h new file mode 100644 index 000000000..54b6efab4 --- /dev/null +++ b/mailnews/extensions/mdn/src/nsMsgMdnGenerator.h @@ -0,0 +1,90 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef _nsMsgMdnGenerator_H_ +#define _nsMsgMdnGenerator_H_ + +#include "nsIMsgMdnGenerator.h" +#include "nsCOMPtr.h" +#include "nsIUrlListener.h" +#include "nsIMsgIncomingServer.h" +#include "nsIOutputStream.h" +#include "nsIFile.h" +#include "nsIMsgIdentity.h" +#include "nsIMsgWindow.h" +#include "nsIMimeHeaders.h" +#include "nsStringGlue.h" +#include "MailNewsTypes2.h" + +#define eNeverSendOp ((int32_t) 0) +#define eAutoSendOp ((int32_t) 1) +#define eAskMeOp ((int32_t) 2) +#define eDeniedOp ((int32_t) 3) + +class nsMsgMdnGenerator : public nsIMsgMdnGenerator, public nsIUrlListener +{ +public: + NS_DECL_ISUPPORTS + NS_DECL_NSIMSGMDNGENERATOR + NS_DECL_NSIURLLISTENER + + nsMsgMdnGenerator(); + +private: + virtual ~nsMsgMdnGenerator(); + + // Sanity Check methods + bool ProcessSendMode(); // must called prior ValidateReturnPath + bool ValidateReturnPath(); + bool NotInToOrCc(); + bool MailAddrMatch(const char *addr1, const char *addr2); + + nsresult StoreMDNSentFlag(nsIMsgFolder *folder, nsMsgKey key); + nsresult ClearMDNNeededFlag(nsIMsgFolder *folder, nsMsgKey key); + nsresult NoteMDNRequestHandled(); + + nsresult CreateMdnMsg(); + nsresult CreateFirstPart(); + nsresult CreateSecondPart(); + nsresult CreateThirdPart(); + nsresult SendMdnMsg(); + + // string bundle helper methods + nsresult GetStringFromName(const char16_t *aName, char16_t **aResultString); + nsresult FormatStringFromName(const char16_t *aName, + const char16_t *aString, + char16_t **aResultString); + + // other helper methods + nsresult InitAndProcess(bool *needToAskUser); + nsresult OutputAllHeaders(); + nsresult WriteString(const char *str); + +private: + EDisposeType m_disposeType; + nsCOMPtr<nsIMsgWindow> m_window; + nsCOMPtr<nsIOutputStream> m_outputStream; + nsCOMPtr<nsIFile> m_file; + nsCOMPtr<nsIMsgIdentity> m_identity; + nsMsgKey m_key; + nsCString m_charset; + nsCString m_email; + nsCString m_mimeSeparator; + nsCString m_messageId; + nsCOMPtr<nsIMsgFolder> m_folder; + nsCOMPtr<nsIMsgIncomingServer> m_server; + nsCOMPtr<nsIMimeHeaders> m_headers; + nsCString m_dntRrt; + int32_t m_notInToCcOp; + int32_t m_outsideDomainOp; + int32_t m_otherOp; + bool m_reallySendMdn; + bool m_autoSend; + bool m_autoAction; + bool m_mdnEnabled; +}; + +#endif // _nsMsgMdnGenerator_H_ + diff --git a/mailnews/extensions/moz.build b/mailnews/extensions/moz.build new file mode 100644 index 000000000..846e47871 --- /dev/null +++ b/mailnews/extensions/moz.build @@ -0,0 +1,19 @@ +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# These extensions are not optional. +DIRS += [ + 'mdn', + 'mailviews/public', + 'mailviews/src', + 'mailviews/content', + 'bayesian-spam-filter', + 'offline-startup', + 'newsblog', + 'fts3/public', + 'fts3/src', + 'smime', +] + diff --git a/mailnews/extensions/newsblog/content/Feed.js b/mailnews/extensions/newsblog/content/Feed.js new file mode 100644 index 000000000..7e47260a8 --- /dev/null +++ b/mailnews/extensions/newsblog/content/Feed.js @@ -0,0 +1,620 @@ +/* -*- Mode: JavaScript; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// Cache for all of the feeds currently being downloaded, indexed by URL, +// so the load event listener can access the Feed objects after it finishes +// downloading the feed. +var FeedCache = +{ + mFeeds: {}, + + putFeed: function (aFeed) + { + this.mFeeds[this.normalizeHost(aFeed.url)] = aFeed; + }, + + getFeed: function (aUrl) + { + let index = this.normalizeHost(aUrl); + if (index in this.mFeeds) + return this.mFeeds[index]; + + return null; + }, + + removeFeed: function (aUrl) + { + let index = this.normalizeHost(aUrl); + if (index in this.mFeeds) + delete this.mFeeds[index]; + }, + + normalizeHost: function (aUrl) + { + try + { + let normalizedUrl = Services.io.newURI(aUrl, null, null); + normalizedUrl.host = normalizedUrl.host.toLowerCase(); + return normalizedUrl.spec + } + catch (ex) + { + return aUrl; + } + } +}; + +function Feed(aResource, aRSSServer) +{ + this.resource = aResource.QueryInterface(Ci.nsIRDFResource); + this.server = aRSSServer; +} + +Feed.prototype = +{ + description: null, + author: null, + request: null, + server: null, + downloadCallback: null, + resource: null, + items: new Array(), + itemsStored: 0, + mFolder: null, + mInvalidFeed: false, + mFeedType: null, + mLastModified: null, + + get folder() + { + return this.mFolder; + }, + + set folder (aFolder) + { + this.mFolder = aFolder; + }, + + get name() + { + // Used for the feed's title in Subcribe dialog and opml export. + let name = this.title || this.description || this.url; + return name.replace(/[\n\r\t]+/g, " ").replace(/[\x00-\x1F]+/g, ""); + }, + + get folderName() + { + if (this.mFolderName) + return this.mFolderName; + + // Get a unique sanitized name. Use title or description as a base; + // these are mandatory by spec. Length of 80 is plenty. + let folderName = (this.title || this.description || "").substr(0,80); + let defaultName = FeedUtils.strings.GetStringFromName("ImportFeedsNew"); + return this.mFolderName = FeedUtils.getSanitizedFolderName(this.server.rootMsgFolder, + folderName, + defaultName, + true); + }, + + download: function(aParseItems, aCallback) + { + // May be null. + this.downloadCallback = aCallback; + + // Whether or not to parse items when downloading and parsing the feed. + // Defaults to true, but setting to false is useful for obtaining + // just the title of the feed when the user subscribes to it. + this.parseItems = aParseItems == null ? true : aParseItems ? true : false; + + // Before we do anything, make sure the url is an http url. This is just + // a sanity check so we don't try opening mailto urls, imap urls, etc. that + // the user may have tried to subscribe to as an rss feed. + if (!FeedUtils.isValidScheme(this.url)) + { + // Simulate an invalid feed error. + FeedUtils.log.info("Feed.download: invalid protocol for - " + this.url); + this.onParseError(this); + return; + } + + // Before we try to download the feed, make sure we aren't already + // processing the feed by looking up the url in our feed cache. + if (FeedCache.getFeed(this.url)) + { + if (this.downloadCallback) + this.downloadCallback.downloaded(this, FeedUtils.kNewsBlogFeedIsBusy); + // Return, the feed is already in use. + return; + } + + if (Services.io.offline) { + // If offline and don't want to go online, just add the feed subscription; + // it can be verified later (the folder name will be the url if not adding + // to an existing folder). Only for subscribe actions; passive biff and + // active get new messages are handled prior to getting here. + let win = Services.wm.getMostRecentWindow("mail:3pane"); + if (!win.MailOfflineMgr.getNewMail()) { + this.storeNextItem(); + return; + } + } + + this.request = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"]. + createInstance(Ci.nsIXMLHttpRequest); + // Must set onProgress before calling open. + this.request.onprogress = this.onProgress; + this.request.open("GET", this.url, true); + this.request.channel.loadFlags |= Ci.nsIRequest.LOAD_BYPASS_CACHE | + Ci.nsIRequest.INHIBIT_CACHING; + + // Some servers, if sent If-Modified-Since, will send 304 if subsequently + // not sent If-Modified-Since, as in the case of an unsubscribe and new + // subscribe. Send start of century date to force a download; some servers + // will 304 on older dates (such as epoch 1970). + let lastModified = this.lastModified || "Sat, 01 Jan 2000 00:00:00 GMT"; + this.request.setRequestHeader("If-Modified-Since", lastModified); + + // Only order what you're going to eat... + this.request.responseType = "document"; + this.request.overrideMimeType("text/xml"); + this.request.setRequestHeader("Accept", FeedUtils.REQUEST_ACCEPT); + this.request.timeout = FeedUtils.REQUEST_TIMEOUT; + this.request.onload = this.onDownloaded; + this.request.onerror = this.onDownloadError; + this.request.ontimeout = this.onDownloadError; + FeedCache.putFeed(this); + this.request.send(null); + }, + + onDownloaded: function(aEvent) + { + let request = aEvent.target; + let isHttp = request.channel.originalURI.scheme.startsWith("http"); + let url = request.channel.originalURI.spec; + if (isHttp && (request.status < 200 || request.status >= 300)) + { + Feed.prototype.onDownloadError(aEvent); + return; + } + + FeedUtils.log.debug("Feed.onDownloaded: got a download - " + url); + let feed = FeedCache.getFeed(url); + if (!feed) + throw new Error("Feed.onDownloaded: error - couldn't retrieve feed " + + "from cache"); + + // If the server sends a Last-Modified header, store the property on the + // feed so we can use it when making future requests, to avoid downloading + // and parsing feeds that have not changed. Don't update if merely checking + // the url, as for subscribe move/copy, as a subsequent refresh may get a 304. + // Save the response and persist it only upon successful completion of the + // refresh cycle (i.e. not if the request is cancelled). + let lastModifiedHeader = request.getResponseHeader("Last-Modified"); + feed.mLastModified = (lastModifiedHeader && feed.parseItems) ? + lastModifiedHeader : null; + + // The download callback is called asynchronously when parse() is done. + feed.parse(); + }, + + onProgress: function(aEvent) + { + let request = aEvent.target; + let url = request.channel.originalURI.spec; + let feed = FeedCache.getFeed(url); + + if (feed.downloadCallback) + feed.downloadCallback.onProgress(feed, aEvent.loaded, aEvent.total, + aEvent.lengthComputable); + }, + + onDownloadError: function(aEvent) + { + let request = aEvent.target; + let url = request.channel.originalURI.spec; + let feed = FeedCache.getFeed(url); + if (feed.downloadCallback) + { + // Generic network or 'not found' error initially. + let error = FeedUtils.kNewsBlogRequestFailure; + + if (request.status == 304) { + // If the http status code is 304, the feed has not been modified + // since we last downloaded it and does not need to be parsed. + error = FeedUtils.kNewsBlogNoNewItems; + } + else { + let [errType, errName] = FeedUtils.createTCPErrorFromFailedXHR(request); + FeedUtils.log.info("Feed.onDownloaded: request errType:errName:statusCode - " + + errType + ":" + errName + ":" + request.status); + if (errType == "SecurityCertificate") + // This is the code for nsINSSErrorsService.ERROR_CLASS_BAD_CERT + // overrideable security certificate errors. + error = FeedUtils.kNewsBlogBadCertError; + + if (request.status == 401 || request.status == 403) + // Unauthorized or Forbidden. + error = FeedUtils.kNewsBlogNoAuthError; + } + + feed.downloadCallback.downloaded(feed, error); + } + + FeedCache.removeFeed(url); + }, + + onParseError: function(aFeed) + { + if (!aFeed) + return; + + aFeed.mInvalidFeed = true; + if (aFeed.downloadCallback) + aFeed.downloadCallback.downloaded(aFeed, FeedUtils.kNewsBlogInvalidFeed); + + FeedCache.removeFeed(aFeed.url); + }, + + onUrlChange: function(aFeed, aOldUrl) + { + if (!aFeed) + return; + + // Simulate a cancel after a url update; next cycle will check the new url. + aFeed.mInvalidFeed = true; + if (aFeed.downloadCallback) + aFeed.downloadCallback.downloaded(aFeed, FeedUtils.kNewsBlogCancel); + + FeedCache.removeFeed(aOldUrl); + }, + + get url() + { + let ds = FeedUtils.getSubscriptionsDS(this.server); + let url = ds.GetTarget(this.resource, FeedUtils.DC_IDENTIFIER, true); + if (url) + url = url.QueryInterface(Ci.nsIRDFLiteral).Value; + else + url = this.resource.ValueUTF8; + + return url; + }, + + get title() + { + let ds = FeedUtils.getSubscriptionsDS(this.server); + let title = ds.GetTarget(this.resource, FeedUtils.DC_TITLE, true); + if (title) + title = title.QueryInterface(Ci.nsIRDFLiteral).Value; + + return title; + }, + + set title (aNewTitle) + { + if (!aNewTitle) + return; + + let ds = FeedUtils.getSubscriptionsDS(this.server); + aNewTitle = FeedUtils.rdf.GetLiteral(aNewTitle); + let old_title = ds.GetTarget(this.resource, FeedUtils.DC_TITLE, true); + if (old_title) + ds.Change(this.resource, FeedUtils.DC_TITLE, old_title, aNewTitle); + else + ds.Assert(this.resource, FeedUtils.DC_TITLE, aNewTitle, true); + }, + + get lastModified() + { + let ds = FeedUtils.getSubscriptionsDS(this.server); + let lastModified = ds.GetTarget(this.resource, + FeedUtils.DC_LASTMODIFIED, + true); + if (lastModified) + lastModified = lastModified.QueryInterface(Ci.nsIRDFLiteral).Value; + + return lastModified; + }, + + set lastModified(aLastModified) + { + let ds = FeedUtils.getSubscriptionsDS(this.server); + aLastModified = FeedUtils.rdf.GetLiteral(aLastModified); + let old_lastmodified = ds.GetTarget(this.resource, + FeedUtils.DC_LASTMODIFIED, + true); + if (old_lastmodified) + ds.Change(this.resource, FeedUtils.DC_LASTMODIFIED, + old_lastmodified, aLastModified); + else + ds.Assert(this.resource, FeedUtils.DC_LASTMODIFIED, aLastModified, true); + }, + + get quickMode () + { + let ds = FeedUtils.getSubscriptionsDS(this.server); + let quickMode = ds.GetTarget(this.resource, FeedUtils.FZ_QUICKMODE, true); + if (quickMode) + { + quickMode = quickMode.QueryInterface(Ci.nsIRDFLiteral); + quickMode = quickMode.Value == "true"; + } + + return quickMode; + }, + + set quickMode (aNewQuickMode) + { + let ds = FeedUtils.getSubscriptionsDS(this.server); + aNewQuickMode = FeedUtils.rdf.GetLiteral(aNewQuickMode); + let old_quickMode = ds.GetTarget(this.resource, + FeedUtils.FZ_QUICKMODE, + true); + if (old_quickMode) + ds.Change(this.resource, FeedUtils.FZ_QUICKMODE, + old_quickMode, aNewQuickMode); + else + ds.Assert(this.resource, FeedUtils.FZ_QUICKMODE, + aNewQuickMode, true); + }, + + get options () + { + let ds = FeedUtils.getSubscriptionsDS(this.server); + let options = ds.GetTarget(this.resource, FeedUtils.FZ_OPTIONS, true); + if (options) + return JSON.parse(options.QueryInterface(Ci.nsIRDFLiteral).Value); + + return null; + }, + + set options (aOptions) + { + let newOptions = aOptions ? FeedUtils.newOptions(aOptions) : + FeedUtils._optionsDefault; + let ds = FeedUtils.getSubscriptionsDS(this.server); + newOptions = FeedUtils.rdf.GetLiteral(JSON.stringify(newOptions)); + let oldOptions = ds.GetTarget(this.resource, FeedUtils.FZ_OPTIONS, true); + if (oldOptions) + ds.Change(this.resource, FeedUtils.FZ_OPTIONS, oldOptions, newOptions); + else + ds.Assert(this.resource, FeedUtils.FZ_OPTIONS, newOptions, true); + }, + + categoryPrefs: function () + { + let categoryPrefsAcct = FeedUtils.getOptionsAcct(this.server).category; + if (!this.options) + return categoryPrefsAcct; + + return this.options.category; + }, + + get link () + { + let ds = FeedUtils.getSubscriptionsDS(this.server); + let link = ds.GetTarget(this.resource, FeedUtils.RSS_LINK, true); + if (link) + link = link.QueryInterface(Ci.nsIRDFLiteral).Value; + + return link; + }, + + set link (aNewLink) + { + if (!aNewLink) + return; + + let ds = FeedUtils.getSubscriptionsDS(this.server); + aNewLink = FeedUtils.rdf.GetLiteral(aNewLink); + let old_link = ds.GetTarget(this.resource, FeedUtils.RSS_LINK, true); + if (old_link) + ds.Change(this.resource, FeedUtils.RSS_LINK, old_link, aNewLink); + else + ds.Assert(this.resource, FeedUtils.RSS_LINK, aNewLink, true); + }, + + parse: function() + { + // Create a feed parser which will parse the feed. + let parser = new FeedParser(); + this.itemsToStore = parser.parseFeed(this, this.request.responseXML); + parser = null; + + if (this.mInvalidFeed) + { + this.request = null; + this.mInvalidFeed = false; + return; + } + + // storeNextItem() will iterate through the parsed items, storing each one. + this.itemsToStoreIndex = 0; + this.itemsStored = 0; + this.storeNextItem(); + }, + + invalidateItems: function () + { + let ds = FeedUtils.getItemsDS(this.server); + FeedUtils.log.debug("Feed.invalidateItems: for url - " + this.url); + let items = ds.GetSources(FeedUtils.FZ_FEED, this.resource, true); + let item; + + while (items.hasMoreElements()) + { + item = items.getNext(); + item = item.QueryInterface(Ci.nsIRDFResource); + FeedUtils.log.trace("Feed.invalidateItems: item - " + item.Value); + let valid = ds.GetTarget(item, FeedUtils.FZ_VALID, true); + if (valid) + ds.Unassert(item, FeedUtils.FZ_VALID, valid, true); + } + }, + + removeInvalidItems: function(aDeleteFeed) + { + let ds = FeedUtils.getItemsDS(this.server); + FeedUtils.log.debug("Feed.removeInvalidItems: for url - " + this.url); + let items = ds.GetSources(FeedUtils.FZ_FEED, this.resource, true); + let item; + let currentTime = new Date().getTime(); + while (items.hasMoreElements()) + { + item = items.getNext(); + item = item.QueryInterface(Ci.nsIRDFResource); + + if (ds.HasAssertion(item, FeedUtils.FZ_VALID, + FeedUtils.RDF_LITERAL_TRUE, true)) + continue; + + let lastSeenTime = ds.GetTarget(item, FeedUtils.FZ_LAST_SEEN_TIMESTAMP, true); + if (lastSeenTime) + lastSeenTime = parseInt(lastSeenTime.QueryInterface(Ci.nsIRDFLiteral).Value) + else + lastSeenTime = 0; + + if ((currentTime - lastSeenTime) < FeedUtils.INVALID_ITEM_PURGE_DELAY && + !aDeleteFeed) + // Don't immediately purge items in active feeds; do so for deleted feeds. + continue; + + FeedUtils.log.trace("Feed.removeInvalidItems: item - " + item.Value); + ds.Unassert(item, FeedUtils.FZ_FEED, this.resource, true); + if (ds.hasArcOut(item, FeedUtils.FZ_FEED)) + FeedUtils.log.debug("Feed.removeInvalidItems: " + item.Value + + " is from more than one feed; only the reference to" + + " this feed removed"); + else + FeedUtils.removeAssertions(ds, item); + } + }, + + createFolder: function() + { + if (this.folder) + return; + + try { + this.folder = this.server.rootMsgFolder + .QueryInterface(Ci.nsIMsgLocalMailFolder) + .createLocalSubfolder(this.folderName); + } + catch (ex) { + // An error creating. + FeedUtils.log.info("Feed.createFolder: error creating folder - '"+ + this.folderName+"' in parent folder "+ + this.server.rootMsgFolder.filePath.path + " -- "+ex); + // But its remnants are still there, clean up. + let xfolder = this.server.rootMsgFolder.getChildNamed(this.folderName); + this.server.rootMsgFolder.propagateDelete(xfolder, true, null); + } + }, + + // Gets the next item from itemsToStore and forces that item to be stored + // to the folder. If more items are left to be stored, fires a timer for + // the next one, otherwise triggers a download done notification to the UI. + storeNextItem: function() + { + if (FeedUtils.CANCEL_REQUESTED) + { + FeedUtils.CANCEL_REQUESTED = false; + this.cleanupParsingState(this, FeedUtils.kNewsBlogCancel); + return; + } + + if (!this.itemsToStore || !this.itemsToStore.length) + { + let code = FeedUtils.kNewsBlogSuccess; + this.createFolder(); + if (!this.folder) + code = FeedUtils.kNewsBlogFileError; + this.cleanupParsingState(this, code); + return; + } + + let item = this.itemsToStore[this.itemsToStoreIndex]; + + if (item.store()) + this.itemsStored++; + + if (!this.folder) + { + this.cleanupParsingState(this, FeedUtils.kNewsBlogFileError); + return; + } + + this.itemsToStoreIndex++; + + // If the listener is tracking progress for each item, report it here. + if (item.feed.downloadCallback && item.feed.downloadCallback.onFeedItemStored) + item.feed.downloadCallback.onFeedItemStored(item.feed, + this.itemsToStoreIndex, + this.itemsToStore.length); + + // Eventually we'll report individual progress here. + + if (this.itemsToStoreIndex < this.itemsToStore.length) + { + if (!this.storeItemsTimer) + this.storeItemsTimer = Cc["@mozilla.org/timer;1"]. + createInstance(Ci.nsITimer); + this.storeItemsTimer.initWithCallback(this, 50, Ci.nsITimer.TYPE_ONE_SHOT); + } + else + { + // We have just finished downloading one or more feed items into the + // destination folder; if the folder is still listed as having new + // messages in it, then we should set the biff state on the folder so the + // right RDF UI changes happen in the folder pane to indicate new mail. + if (item.feed.folder.hasNewMessages) + { + item.feed.folder.biffState = Ci.nsIMsgFolder.nsMsgBiffState_NewMail; + // Run the bayesian spam filter, if enabled. + item.feed.folder.callFilterPlugins(null); + } + + this.cleanupParsingState(this, FeedUtils.kNewsBlogSuccess); + } + }, + + cleanupParsingState: function(aFeed, aCode) + { + // Now that we are done parsing the feed, remove the feed from the cache. + FeedCache.removeFeed(aFeed.url); + + if (aFeed.parseItems) + { + // Do this only if we're in parse/store mode. + aFeed.removeInvalidItems(false); + + if (aCode == FeedUtils.kNewsBlogSuccess && aFeed.mLastModified) + aFeed.lastModified = aFeed.mLastModified; + + // Flush any feed item changes to disk. + let ds = FeedUtils.getItemsDS(aFeed.server); + ds.Flush(); + FeedUtils.log.debug("Feed.cleanupParsingState: items stored - " + this.itemsStored); + } + + // Force the xml http request to go away. This helps reduce some nasty + // assertions on shut down. + this.request = null; + this.itemsToStore = ""; + this.itemsToStoreIndex = 0; + this.itemsStored = 0; + this.storeItemsTimer = null; + + if (aFeed.downloadCallback) + aFeed.downloadCallback.downloaded(aFeed, aCode); + }, + + // nsITimerCallback + notify: function(aTimer) + { + this.storeNextItem(); + } +}; diff --git a/mailnews/extensions/newsblog/content/FeedItem.js b/mailnews/extensions/newsblog/content/FeedItem.js new file mode 100644 index 000000000..09e4eb861 --- /dev/null +++ b/mailnews/extensions/newsblog/content/FeedItem.js @@ -0,0 +1,490 @@ +/* -*- Mode: JavaScript; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +function FeedItem() +{ + this.mDate = FeedUtils.getValidRFC5322Date(); + this.mUnicodeConverter = Cc["@mozilla.org/intl/scriptableunicodeconverter"]. + createInstance(Ci.nsIScriptableUnicodeConverter); + this.mParserUtils = Cc["@mozilla.org/parserutils;1"]. + getService(Ci.nsIParserUtils); +} + +FeedItem.prototype = +{ + // Only for IETF Atom. + xmlContentBase: null, + id: null, + feed: null, + description: null, + content: null, + enclosures: [], + title: null, + author: "anonymous", + inReplyTo: "", + keywords: [], + mURL: null, + characterSet: "UTF-8", + + ENCLOSURE_BOUNDARY_PREFIX: "--------------", // 14 dashes + ENCLOSURE_HEADER_BOUNDARY_PREFIX: "------------", // 12 dashes + MESSAGE_TEMPLATE: '\n' + + '<html>\n' + + ' <head>\n' + + ' <title>%TITLE%</title>\n' + + ' <base href="%BASE%">\n' + + ' </head>\n' + + ' <body id="msgFeedSummaryBody" selected="false">\n' + + ' %CONTENT%\n' + + ' </body>\n' + + '</html>\n', + + get url() + { + return this.mURL; + }, + + set url(aVal) + { + try + { + this.mURL = Services.io.newURI(aVal, null, null).spec; + } + catch(ex) + { + // The url as published or constructed can be a non url. It's used as a + // feeditem identifier in feeditems.rdf, as a messageId, and as an href + // and for the content-base header. Save as is; ensure not null. + this.mURL = aVal ? aVal : ""; + } + }, + + get date() + { + return this.mDate; + }, + + set date (aVal) + { + this.mDate = aVal; + }, + + get identity () + { + return this.feed.name + ": " + this.title + " (" + this.id + ")" + }, + + normalizeMessageID: function(messageID) + { + // Escape occurrences of message ID meta characters <, >, and @. + messageID.replace(/</g, "%3C"); + messageID.replace(/>/g, "%3E"); + messageID.replace(/@/g, "%40"); + messageID = "<" + messageID.trim() + "@" + "localhost.localdomain" + ">"; + + FeedUtils.log.trace("FeedItem.normalizeMessageID: messageID - " + messageID); + return messageID; + }, + + get itemUniqueURI() + { + return this.createURN(this.id); + }, + + get contentBase() + { + if(this.xmlContentBase) + return this.xmlContentBase + else + return this.mURL; + }, + + store: function() + { + // this.title and this.content contain HTML. + // this.mUrl and this.contentBase contain plain text. + + let stored = false; + let resource = this.findStoredResource(); + if (!this.feed.folder) + return stored; + + if (resource == null) + { + resource = FeedUtils.rdf.GetResource(this.itemUniqueURI); + if (!this.content) + { + FeedUtils.log.trace("FeedItem.store: " + this.identity + + " no content; storing description or title"); + this.content = this.description || this.title; + } + + let content = this.MESSAGE_TEMPLATE; + content = content.replace(/%TITLE%/, this.title); + content = content.replace(/%BASE%/, this.htmlEscape(this.contentBase)); + content = content.replace(/%CONTENT%/, this.content); + this.content = content; + this.writeToFolder(); + this.markStored(resource); + stored = true; + } + this.markValid(resource); + return stored; + }, + + findStoredResource: function() + { + // Checks to see if the item has already been stored in its feed's + // message folder. + FeedUtils.log.trace("FeedItem.findStoredResource: checking if stored - " + + this.identity); + + let server = this.feed.server; + let folder = this.feed.folder; + + if (!folder) + { + FeedUtils.log.debug("FeedItem.findStoredResource: folder '" + + this.feed.folderName + + "' doesn't exist; creating as child of " + + server.rootMsgFolder.prettyName + "\n"); + this.feed.createFolder(); + return null; + } + + let ds = FeedUtils.getItemsDS(server); + let itemURI = this.itemUniqueURI; + let itemResource = FeedUtils.rdf.GetResource(itemURI); + + let downloaded = ds.GetTarget(itemResource, FeedUtils.FZ_STORED, true); + + if (!downloaded || + downloaded.QueryInterface(Ci.nsIRDFLiteral).Value == "false") + { + FeedUtils.log.trace("FeedItem.findStoredResource: not stored"); + return null; + } + + FeedUtils.log.trace("FeedItem.findStoredResource: already stored"); + return itemResource; + }, + + markValid: function(resource) + { + let ds = FeedUtils.getItemsDS(this.feed.server); + + let newTimeStamp = FeedUtils.rdf.GetLiteral(new Date().getTime()); + let currentTimeStamp = ds.GetTarget(resource, + FeedUtils.FZ_LAST_SEEN_TIMESTAMP, + true); + if (currentTimeStamp) + ds.Change(resource, FeedUtils.FZ_LAST_SEEN_TIMESTAMP, + currentTimeStamp, newTimeStamp); + else + ds.Assert(resource, FeedUtils.FZ_LAST_SEEN_TIMESTAMP, + newTimeStamp, true); + + if (!ds.HasAssertion(resource, FeedUtils.FZ_FEED, + FeedUtils.rdf.GetResource(this.feed.url), true)) + ds.Assert(resource, FeedUtils.FZ_FEED, + FeedUtils.rdf.GetResource(this.feed.url), true); + + if (ds.hasArcOut(resource, FeedUtils.FZ_VALID)) + { + let currentValue = ds.GetTarget(resource, FeedUtils.FZ_VALID, true); + ds.Change(resource, FeedUtils.FZ_VALID, + currentValue, FeedUtils.RDF_LITERAL_TRUE); + } + else + ds.Assert(resource, FeedUtils.FZ_VALID, FeedUtils.RDF_LITERAL_TRUE, true); + }, + + markStored: function(resource) + { + let ds = FeedUtils.getItemsDS(this.feed.server); + + if (!ds.HasAssertion(resource, FeedUtils.FZ_FEED, + FeedUtils.rdf.GetResource(this.feed.url), true)) + ds.Assert(resource, FeedUtils.FZ_FEED, + FeedUtils.rdf.GetResource(this.feed.url), true); + + let currentValue; + if (ds.hasArcOut(resource, FeedUtils.FZ_STORED)) + { + currentValue = ds.GetTarget(resource, FeedUtils.FZ_STORED, true); + ds.Change(resource, FeedUtils.FZ_STORED, + currentValue, FeedUtils.RDF_LITERAL_TRUE); + } + else + ds.Assert(resource, FeedUtils.FZ_STORED, + FeedUtils.RDF_LITERAL_TRUE, true); + }, + + mimeEncodeSubject: function(aSubject, aCharset) + { + // This routine sometimes throws exceptions for mis-encoded data so + // wrap it with a try catch for now. + let newSubject; + try + { + newSubject = mailServices.mimeConverter.encodeMimePartIIStr_UTF8(aSubject, + false, + aCharset, 9, 72); + } + catch (ex) + { + newSubject = aSubject; + } + + return newSubject; + }, + + writeToFolder: function() + { + FeedUtils.log.trace("FeedItem.writeToFolder: " + this.identity + + " writing to message folder " + this.feed.name); + // Convert the title to UTF-16 before performing our HTML entity + // replacement reg expressions. + let title = this.title; + + // The subject may contain HTML entities. Convert these to their unencoded + // state. i.e. & becomes '&'. + title = this.mParserUtils.convertToPlainText( + title, + Ci.nsIDocumentEncoder.OutputSelectionOnly | + Ci.nsIDocumentEncoder.OutputAbsoluteLinks, + 0); + + // Compress white space in the subject to make it look better. Trim + // leading/trailing spaces to prevent mbox header folding issue at just + // the right subject length. + title = title.replace(/[\t\r\n]+/g, " ").trim(); + + this.title = this.mimeEncodeSubject(title, this.characterSet); + + // If the date looks like it's in W3C-DTF format, convert it into + // an IETF standard date. Otherwise assume it's in IETF format. + if (this.mDate.search(/^\d\d\d\d/) != -1) + this.mDate = new Date(this.mDate).toUTCString(); + + // If there is an inreplyto value, create the headers. + let inreplytoHdrsStr = this.inReplyTo ? + ("References: " + this.inReplyTo + "\n" + + "In-Reply-To: " + this.inReplyTo + "\n") : ""; + + // If there are keywords (categories), create the headers. In the case of + // a longer than RFC5322 recommended line length, create multiple folded + // lines (easier to parse than multiple Keywords headers). + let keywordsStr = ""; + if (this.keywords.length) + { + let HEADER = "Keywords: "; + let MAXLEN = 78; + keywordsStr = HEADER; + let keyword; + let keywords = [].concat(this.keywords); + let lines = []; + while (keywords.length) + { + keyword = keywords.shift(); + if (keywordsStr.length + keyword.length > MAXLEN) + { + lines.push(keywordsStr) + keywordsStr = " ".repeat(HEADER.length); + } + keywordsStr += keyword + ","; + } + keywordsStr = keywordsStr.replace(/,$/,"\n"); + lines.push(keywordsStr) + keywordsStr = lines.join("\n"); + } + + // Escape occurrences of "From " at the beginning of lines of + // content per the mbox standard, since "From " denotes a new + // message, and add a line break so we know the last line has one. + this.content = this.content.replace(/([\r\n]+)(>*From )/g, "$1>$2"); + this.content += "\n"; + + // The opening line of the message, mandated by standards to start + // with "From ". It's useful to construct this separately because + // we not only need to write it into the message, we also need to + // use it to calculate the offset of the X-Mozilla-Status lines from + // the front of the message for the statusOffset property of the + // DB header object. + let openingLine = 'From - ' + this.mDate + '\n'; + + let source = + openingLine + + 'X-Mozilla-Status: 0000\n' + + 'X-Mozilla-Status2: 00000000\n' + + 'X-Mozilla-Keys: ' + " ".repeat(80) + '\n' + + 'Received: by localhost; ' + FeedUtils.getValidRFC5322Date() + '\n' + + 'Date: ' + this.mDate + '\n' + + 'Message-Id: ' + this.normalizeMessageID(this.id) + '\n' + + 'From: ' + this.author + '\n' + + 'MIME-Version: 1.0\n' + + 'Subject: ' + this.title + '\n' + + inreplytoHdrsStr + + keywordsStr + + 'Content-Transfer-Encoding: 8bit\n' + + 'Content-Base: ' + this.mURL + '\n'; + + if (this.enclosures.length) + { + let boundaryID = source.length; + source += 'Content-Type: multipart/mixed; boundary="' + + this.ENCLOSURE_HEADER_BOUNDARY_PREFIX + boundaryID + '"' + '\n\n' + + 'This is a multi-part message in MIME format.\n' + + this.ENCLOSURE_BOUNDARY_PREFIX + boundaryID + '\n' + + 'Content-Type: text/html; charset=' + this.characterSet + '\n' + + 'Content-Transfer-Encoding: 8bit\n' + + this.content; + + this.enclosures.forEach(function(enclosure) { + source += enclosure.convertToAttachment(boundaryID); + }); + + source += this.ENCLOSURE_BOUNDARY_PREFIX + boundaryID + '--' + '\n\n\n'; + } + else + source += 'Content-Type: text/html; charset=' + this.characterSet + '\n' + + this.content; + + FeedUtils.log.trace("FeedItem.writeToFolder: " + this.identity + + " is " + source.length + " characters long"); + + // Get the folder and database storing the feed's messages and headers. + let folder = this.feed.folder.QueryInterface(Ci.nsIMsgLocalMailFolder); + let msgFolder = folder.QueryInterface(Ci.nsIMsgFolder); + msgFolder.gettingNewMessages = true; + // Source is a unicode string, we want to save a char * string in + // the original charset. So convert back. + this.mUnicodeConverter.charset = this.characterSet; + let msgDBHdr = folder.addMessage(this.mUnicodeConverter.ConvertFromUnicode(source)); + msgDBHdr.OrFlags(Ci.nsMsgMessageFlags.FeedMsg); + msgFolder.gettingNewMessages = false; + this.tagItem(msgDBHdr, this.keywords); + }, + +/** + * Autotag messages. + * + * @param nsIMsgDBHdr aMsgDBHdr - message to tag + * @param array aKeywords - keywords (tags) + */ + tagItem: function(aMsgDBHdr, aKeywords) + { + let categoryPrefs = this.feed.categoryPrefs(); + if (!aKeywords.length || !categoryPrefs.enabled) + return; + + let msgArray = Cc["@mozilla.org/array;1"].createInstance(Ci.nsIMutableArray); + msgArray.appendElement(aMsgDBHdr, false); + + let prefix = categoryPrefs.prefixEnabled ? categoryPrefs.prefix : ""; + let rtl = Services.prefs.getIntPref("bidi.direction") == 2; + + let keys = []; + for (let keyword of aKeywords) + { + keyword = rtl ? keyword + prefix : prefix + keyword; + let keyForTag = MailServices.tags.getKeyForTag(keyword); + if (!keyForTag) + { + // Add the tag if it doesn't exist. + MailServices.tags.addTag(keyword, "", FeedUtils.AUTOTAG); + keyForTag = MailServices.tags.getKeyForTag(keyword); + } + + // Add the tag key to the keys array. + keys.push(keyForTag); + } + + if (keys.length) + // Add the keys to the message. + aMsgDBHdr.folder.addKeywordsToMessages(msgArray, keys.join(" ")); + }, + + htmlEscape: function(s) + { + s = s.replace(/&/g, "&"); + s = s.replace(/>/g, ">"); + s = s.replace(/</g, "<"); + s = s.replace(/'/g, "'"); + s = s.replace(/"/g, """); + return s; + }, + + createURN: function(aName) + { + // Returns name as a URN in the 'feeditem' namespace. The returned URN is + // (or is intended to be) RFC2141 compliant. + // The builtin encodeURI provides nearly the exact encoding functionality + // required by the RFC. The exceptions are that NULL characters should not + // appear, and that #, /, ?, &, and ~ should be escaped. + // NULL characters are removed before encoding. + + let name = aName.replace(/\0/g, ""); + let encoded = encodeURI(name); + encoded = encoded.replace(/\#/g, "%23"); + encoded = encoded.replace(/\//g, "%2f"); + encoded = encoded.replace(/\?/g, "%3f"); + encoded = encoded.replace(/\&/g, "%26"); + encoded = encoded.replace(/\~/g, "%7e"); + + return FeedUtils.FZ_ITEM_NS + encoded; + } +}; + + +// A feed enclosure is to RSS what an attachment is for e-mail. We make +// enclosures look like attachments in the UI. +function FeedEnclosure(aURL, aContentType, aLength, aTitle) +{ + this.mURL = aURL; + // Store a reasonable mimetype if content-type is not present. + this.mContentType = aContentType || "application/unknown"; + this.mLength = aLength; + this.mTitle = aTitle; + + // Generate a fileName from the URL. + if (this.mURL) + { + try + { + this.mFileName = Services.io.newURI(this.mURL, null, null). + QueryInterface(Ci.nsIURL). + fileName; + } + catch(ex) + { + this.mFileName = this.mURL; + } + } +} + +FeedEnclosure.prototype = +{ + mURL: "", + mContentType: "", + mLength: 0, + mFileName: "", + mTitle: "", + ENCLOSURE_BOUNDARY_PREFIX: "--------------", // 14 dashes + + // Returns a string that looks like an e-mail attachment which represents + // the enclosure. + convertToAttachment: function(aBoundaryID) + { + return '\n' + + this.ENCLOSURE_BOUNDARY_PREFIX + aBoundaryID + '\n' + + 'Content-Type: ' + this.mContentType + + '; name="' + (this.mTitle || this.mFileName) + + (this.mLength ? '"; size=' + this.mLength : '"') + '\n' + + 'X-Mozilla-External-Attachment-URL: ' + this.mURL + '\n' + + 'Content-Disposition: attachment; filename="' + this.mFileName + '"\n\n' + + FeedUtils.strings.GetStringFromName("externalAttachmentMsg") + '\n'; + } +}; diff --git a/mailnews/extensions/newsblog/content/FeedUtils.jsm b/mailnews/extensions/newsblog/content/FeedUtils.jsm new file mode 100644 index 000000000..6d5e64dd2 --- /dev/null +++ b/mailnews/extensions/newsblog/content/FeedUtils.jsm @@ -0,0 +1,1608 @@ +/* -*- Mode: JavaScript; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +this.EXPORTED_SYMBOLS = ["Feed", "FeedItem", "FeedParser", "FeedUtils"]; + +var {classes: Cc, interfaces: Ci, utils: Cu} = Components; + +Cu.import("resource:///modules/gloda/log4moz.js"); +Cu.import("resource:///modules/mailServices.js"); +Cu.import("resource:///modules/MailUtils.js"); +Cu.import("resource:///modules/jsmime.jsm"); +Cu.import("resource://gre/modules/FileUtils.jsm"); +Cu.import("resource://gre/modules/Services.jsm"); +Cu.import("resource://gre/modules/XPCOMUtils.jsm"); + +Services.scriptloader.loadSubScript("chrome://messenger-newsblog/content/Feed.js"); +Services.scriptloader.loadSubScript("chrome://messenger-newsblog/content/FeedItem.js"); +Services.scriptloader.loadSubScript("chrome://messenger-newsblog/content/feed-parser.js"); + +var FeedUtils = { + MOZ_PARSERERROR_NS: "http://www.mozilla.org/newlayout/xml/parsererror.xml", + + RDF_SYNTAX_NS: "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + RDF_SYNTAX_TYPE: "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", + get RDF_TYPE() { return this.rdf.GetResource(this.RDF_SYNTAX_TYPE) }, + + RSS_090_NS: "http://my.netscape.com/rdf/simple/0.9/", + + RSS_NS: "http://purl.org/rss/1.0/", + get RSS_CHANNEL() { return this.rdf.GetResource(this.RSS_NS + "channel") }, + get RSS_TITLE() { return this.rdf.GetResource(this.RSS_NS + "title") }, + get RSS_DESCRIPTION() { return this.rdf.GetResource(this.RSS_NS + "description") }, + get RSS_ITEMS() { return this.rdf.GetResource(this.RSS_NS + "items") }, + get RSS_ITEM() { return this.rdf.GetResource(this.RSS_NS + "item") }, + get RSS_LINK() { return this.rdf.GetResource(this.RSS_NS + "link") }, + + RSS_CONTENT_NS: "http://purl.org/rss/1.0/modules/content/", + get RSS_CONTENT_ENCODED() { + return this.rdf.GetResource(this.RSS_CONTENT_NS + "encoded"); + }, + + DC_NS: "http://purl.org/dc/elements/1.1/", + get DC_CREATOR() { return this.rdf.GetResource(this.DC_NS + "creator") }, + get DC_SUBJECT() { return this.rdf.GetResource(this.DC_NS + "subject") }, + get DC_DATE() { return this.rdf.GetResource(this.DC_NS + "date") }, + get DC_TITLE() { return this.rdf.GetResource(this.DC_NS + "title") }, + get DC_LASTMODIFIED() { return this.rdf.GetResource(this.DC_NS + "lastModified") }, + get DC_IDENTIFIER() { return this.rdf.GetResource(this.DC_NS + "identifier") }, + + MRSS_NS: "http://search.yahoo.com/mrss/", + FEEDBURNER_NS: "http://rssnamespace.org/feedburner/ext/1.0", + ITUNES_NS: "http://www.itunes.com/dtds/podcast-1.0.dtd", + + FZ_NS: "urn:forumzilla:", + FZ_ITEM_NS: "urn:feeditem:", + get FZ_ROOT() { return this.rdf.GetResource(this.FZ_NS + "root") }, + get FZ_FEEDS() { return this.rdf.GetResource(this.FZ_NS + "feeds") }, + get FZ_FEED() { return this.rdf.GetResource(this.FZ_NS + "feed") }, + get FZ_QUICKMODE() { return this.rdf.GetResource(this.FZ_NS + "quickMode") }, + get FZ_DESTFOLDER() { return this.rdf.GetResource(this.FZ_NS + "destFolder") }, + get FZ_STORED() { return this.rdf.GetResource(this.FZ_NS + "stored") }, + get FZ_VALID() { return this.rdf.GetResource(this.FZ_NS + "valid") }, + get FZ_OPTIONS() { return this.rdf.GetResource(this.FZ_NS + "options"); }, + get FZ_LAST_SEEN_TIMESTAMP() { + return this.rdf.GetResource(this.FZ_NS + "last-seen-timestamp"); + }, + + get RDF_LITERAL_TRUE() { return this.rdf.GetLiteral("true") }, + get RDF_LITERAL_FALSE() { return this.rdf.GetLiteral("false") }, + + // Atom constants + ATOM_03_NS: "http://purl.org/atom/ns#", + ATOM_IETF_NS: "http://www.w3.org/2005/Atom", + ATOM_THREAD_NS: "http://purl.org/syndication/thread/1.0", + + // Accept content mimetype preferences for feeds. + REQUEST_ACCEPT: "application/atom+xml," + + "application/rss+xml;q=0.9," + + "application/rdf+xml;q=0.8," + + "application/xml;q=0.7,text/xml;q=0.7," + + "*/*;q=0.1", + // Timeout for nonresponse to request, 30 seconds. + REQUEST_TIMEOUT: 30 * 1000, + + // The approximate amount of time, specified in milliseconds, to leave an + // item in the RDF cache after the item has dissappeared from feeds. + // The delay is currently one day. + INVALID_ITEM_PURGE_DELAY: 24 * 60 * 60 * 1000, + + kBiffMinutesDefault: 100, + kNewsBlogSuccess: 0, + // Usually means there was an error trying to parse the feed. + kNewsBlogInvalidFeed: 1, + // Generic networking failure when trying to download the feed. + kNewsBlogRequestFailure: 2, + kNewsBlogFeedIsBusy: 3, + // For 304 Not Modified; There are no new articles for this feed. + kNewsBlogNoNewItems: 4, + kNewsBlogCancel: 5, + kNewsBlogFileError: 6, + // Invalid certificate, for overridable user exception errors. + kNewsBlogBadCertError: 7, + // For 401 Unauthorized or 403 Forbidden. + kNewsBlogNoAuthError: 8, + + CANCEL_REQUESTED: false, + AUTOTAG: "~AUTOTAG", + +/** + * Get all rss account servers rootFolders. + * + * @return array of nsIMsgIncomingServer (empty array if none). + */ + getAllRssServerRootFolders: function() { + let rssRootFolders = []; + let allServers = MailServices.accounts.allServers; + for (let i = 0; i < allServers.length; i++) + { + let server = allServers.queryElementAt(i, Ci.nsIMsgIncomingServer); + if (server && server.type == "rss") + rssRootFolders.push(server.rootFolder); + } + + // By default, Tb sorts by hostname, ie Feeds, Feeds-1, and not by alpha + // prettyName. Do the same as a stock install to match folderpane order. + rssRootFolders.sort(function(a, b) { return a.hostname > b.hostname }); + + return rssRootFolders; + }, + +/** + * Create rss account. + * + * @param string [aName] - optional account name to override default. + * @return nsIMsgAccount. + */ + createRssAccount: function(aName) { + let userName = "nobody"; + let hostName = "Feeds"; + let hostNamePref = hostName; + let server; + let serverType = "rss"; + let defaultName = FeedUtils.strings.GetStringFromName("feeds-accountname"); + let i = 2; + while (MailServices.accounts.findRealServer(userName, hostName, serverType, 0)) + // If "Feeds" exists, try "Feeds-2", then "Feeds-3", etc. + hostName = hostNamePref + "-" + i++; + + server = MailServices.accounts.createIncomingServer(userName, hostName, serverType); + server.biffMinutes = FeedUtils.kBiffMinutesDefault; + server.prettyName = aName ? aName : defaultName; + server.valid = true; + let account = MailServices.accounts.createAccount(); + account.incomingServer = server; + + // Ensure the Trash folder db (.msf) is created otherwise folder/message + // deletes will throw until restart creates it. + server.msgStore.discoverSubFolders(server.rootMsgFolder, false); + + // Create "Local Folders" if none exist yet as it's guaranteed that + // those exist when any account exists. + let localFolders; + try { + localFolders = MailServices.accounts.localFoldersServer; + } + catch (ex) {} + + if (!localFolders) + MailServices.accounts.createLocalMailAccount(); + + // Save new accounts in case of a crash. + try { + MailServices.accounts.saveAccountInfo(); + } + catch (ex) { + this.log.error("FeedUtils.createRssAccount: error on saveAccountInfo - " + ex); + } + + this.log.debug("FeedUtils.createRssAccount: " + + account.incomingServer.rootFolder.prettyName); + + return account; + }, + +/** + * Helper routine that checks our subscriptions list array and returns + * true if the url is already in our list. This is used to prevent the + * user from subscribing to the same feed multiple times for the same server. + * + * @param string aUrl - the url. + * @param nsIMsgIncomingServer aServer - account server. + * @return boolean - true if exists else false. + */ + feedAlreadyExists: function(aUrl, aServer) { + let ds = this.getSubscriptionsDS(aServer); + let feeds = this.getSubscriptionsList(ds); + let resource = this.rdf.GetResource(aUrl); + if (feeds.IndexOf(resource) == -1) + return false; + + let folder = ds.GetTarget(resource, FeedUtils.FZ_DESTFOLDER, true) + .QueryInterface(Ci.nsIRDFResource).ValueUTF8; + this.log.info("FeedUtils.feedAlreadyExists: feed url " + aUrl + + " subscribed in folder url " + decodeURI(folder)); + + return true; + }, + +/** + * Download a feed url on biff or get new messages. + * + * @param nsIMsgFolder aFolder - folder + * @param nsIUrlListener aUrlListener - feed url + * @param bool aIsBiff - true if biff, false if manual get + * @param nsIDOMWindow aMsgWindow - window + */ + downloadFeed: function(aFolder, aUrlListener, aIsBiff, aMsgWindow) { + if (Services.io.offline) + return; + + // We don't yet support the ability to check for new articles while we are + // in the middle of subscribing to a feed. For now, abort the check for + // new feeds. + if (FeedUtils.progressNotifier.mSubscribeMode) + { + FeedUtils.log.warn("downloadFeed: Aborting RSS New Mail Check. " + + "Feed subscription in progress\n"); + return; + } + + let allFolders = Cc["@mozilla.org/array;1"].createInstance(Ci.nsIMutableArray); + if (!aFolder.isServer) { + // Add the base folder; it does not get returned by ListDescendants. Do not + // add the account folder as it doesn't have the feedUrl property or even + // a msgDatabase necessarily. + allFolders.appendElement(aFolder, false); + } + + aFolder.ListDescendants(allFolders); + + let folder; + function* feeder() { + let numFolders = allFolders.length; + for (let i = 0; i < numFolders; i++) { + folder = allFolders.queryElementAt(i, Ci.nsIMsgFolder); + FeedUtils.log.debug("downloadFeed: START x/# foldername:uri - " + + (i+1) + "/" + numFolders + " " + + folder.name + ":" + folder.URI); + + // Ensure folder's msgDatabase is openable for new message processing. + // If not, reparse. After the async reparse the folder will be ready + // for the next cycle; don't bother with a listener. Continue with + // the next folder, as attempting to add a message to a folder with + // an unavailable msgDatabase will throw later. + if (!FeedUtils.isMsgDatabaseOpenable(folder, true)) + continue; + + let feedUrlArray = FeedUtils.getFeedUrlsInFolder(folder); + // Continue if there are no feedUrls for the folder in the feeds + // database. All folders in Trash are skipped. + if (!feedUrlArray) + continue; + + FeedUtils.log.debug("downloadFeed: CONTINUE foldername:urlArray - " + + folder.name + ":" + feedUrlArray); + + FeedUtils.progressNotifier.init(aMsgWindow, false); + + // We need to kick off a download for each feed. + let id, feed; + for (let url of feedUrlArray) + { + id = FeedUtils.rdf.GetResource(url); + feed = new Feed(id, folder.server); + feed.folder = folder; + // Bump our pending feed download count. + FeedUtils.progressNotifier.mNumPendingFeedDownloads++; + feed.download(true, FeedUtils.progressNotifier); + FeedUtils.log.debug("downloadFeed: DOWNLOAD feed url - " + url); + + Services.tm.mainThread.dispatch(function() { + try { + let done = getFeed.next().done; + if (done) { + // Finished with all feeds in base folder and its subfolders. + FeedUtils.log.debug("downloadFeed: Finished with folder - " + + aFolder.name); + folder = null; + allFolders = null; + } + } + catch (ex) { + FeedUtils.log.error("downloadFeed: error - " + ex); + FeedUtils.progressNotifier.downloaded({name: folder.name}, 0); + } + }, Ci.nsIThread.DISPATCH_NORMAL); + + yield undefined; + } + } + } + + let getFeed = feeder(); + try { + let done = getFeed.next().done; + if (done) { + // Nothing to do. + FeedUtils.log.debug("downloadFeed: Nothing to do in folder - " + + aFolder.name); + folder = null; + allFolders = null; + } + } + catch (ex) { + FeedUtils.log.error("downloadFeed: error - " + ex); + FeedUtils.progressNotifier.downloaded({name: aFolder.name}, 0); + } + }, + +/** + * Subscribe a new feed url. + * + * @param string aUrl - feed url + * @param nsIMsgFolder aFolder - folder + * @param nsIDOMWindow aMsgWindow - window + */ + subscribeToFeed: function(aUrl, aFolder, aMsgWindow) { + // We don't support the ability to subscribe to several feeds at once yet. + // For now, abort the subscription if we are already in the middle of + // subscribing to a feed via drag and drop. + if (FeedUtils.progressNotifier.mNumPendingFeedDownloads) + { + FeedUtils.log.warn("subscribeToFeed: Aborting RSS subscription. " + + "Feed downloads already in progress\n"); + return; + } + + // If aFolder is null, then use the root folder for the first RSS account. + if (!aFolder) + aFolder = FeedUtils.getAllRssServerRootFolders()[0]; + + // If the user has no Feeds account yet, create one. + if (!aFolder) + aFolder = FeedUtils.createRssAccount().incomingServer.rootFolder; + + if (!aMsgWindow) + { + let wlist = Services.wm.getEnumerator("mail:3pane"); + if (wlist.hasMoreElements()) + { + let win = wlist.getNext().QueryInterface(Ci.nsIDOMWindow); + win.focus(); + aMsgWindow = win.msgWindow; + } + else + { + // If there are no open windows, open one, pass it the URL, and + // during opening it will subscribe to the feed. + let arg = Cc["@mozilla.org/supports-string;1"]. + createInstance(Ci.nsISupportsString); + arg.data = aUrl; + Services.ww.openWindow(null, "chrome://messenger/content/", + "_blank", "chrome,dialog=no,all", arg); + return; + } + } + + // If aUrl is a feed url, then it is either of the form + // feed://example.org/feed.xml or feed:https://example.org/feed.xml. + // Replace feed:// with http:// per the spec, then strip off feed: + // for the second case. + aUrl = aUrl.replace(/^feed:\x2f\x2f/i, "http://"); + aUrl = aUrl.replace(/^feed:/i, ""); + + // Make sure we aren't already subscribed to this feed before we attempt + // to subscribe to it. + if (FeedUtils.feedAlreadyExists(aUrl, aFolder.server)) + { + aMsgWindow.statusFeedback.showStatusString( + FeedUtils.strings.GetStringFromName("subscribe-feedAlreadySubscribed")); + return; + } + + let itemResource = FeedUtils.rdf.GetResource(aUrl); + let feed = new Feed(itemResource, aFolder.server); + feed.quickMode = feed.server.getBoolValue("quickMode"); + feed.options = FeedUtils.getOptionsAcct(feed.server); + + // If the root server, create a new folder for the feed. The user must + // want us to add this subscription url to an existing RSS folder. + if (!aFolder.isServer) + feed.folder = aFolder; + + FeedUtils.progressNotifier.init(aMsgWindow, true); + FeedUtils.progressNotifier.mNumPendingFeedDownloads++; + feed.download(true, FeedUtils.progressNotifier); + }, + +/** + * Add a feed record to the feeds.rdf database and update the folder's feedUrl + * property. + * + * @param object aFeed - our feed object + */ + addFeed: function(aFeed) { + let ds = this.getSubscriptionsDS(aFeed.folder.server); + let feeds = this.getSubscriptionsList(ds); + + // Generate a unique ID for the feed. + let id = aFeed.url; + let i = 1; + while (feeds.IndexOf(this.rdf.GetResource(id)) != -1 && ++i < 1000) + id = aFeed.url + i; + if (i == 1000) + throw new Error("FeedUtils.addFeed: couldn't generate a unique ID " + + "for feed " + aFeed.url); + + // Add the feed to the list. + id = this.rdf.GetResource(id); + feeds.AppendElement(id); + ds.Assert(id, this.RDF_TYPE, this.FZ_FEED, true); + ds.Assert(id, this.DC_IDENTIFIER, this.rdf.GetLiteral(aFeed.url), true); + if (aFeed.title) + ds.Assert(id, this.DC_TITLE, this.rdf.GetLiteral(aFeed.title), true); + ds.Assert(id, this.FZ_DESTFOLDER, aFeed.folder, true); + ds.Flush(); + + // Update folderpane. + this.setFolderPaneProperty(aFeed.folder, "favicon", null, "row"); + }, + +/** + * Delete a feed record from the feeds.rdf database and update the folder's + * feedUrl property. + * + * @param nsIRDFResource aId - feed url as rdf resource. + * @param nsIMsgIncomingServer aServer - folder's account server. + * @param nsIMsgFolder aParentFolder - owning folder. + */ + deleteFeed: function(aId, aServer, aParentFolder) { + let feed = new Feed(aId, aServer); + let ds = this.getSubscriptionsDS(aServer); + + if (!feed || !ds) + return; + + // Remove the feed from the subscriptions ds. + let feeds = this.getSubscriptionsList(ds); + let index = feeds.IndexOf(aId); + if (index != -1) + feeds.RemoveElementAt(index, false); + + // Remove all assertions about the feed from the subscriptions database. + this.removeAssertions(ds, aId); + ds.Flush(); + + // Remove all assertions about items in the feed from the items database. + let itemds = this.getItemsDS(aServer); + feed.invalidateItems(); + feed.removeInvalidItems(true); + itemds.Flush(); + + // Update folderpane. + this.setFolderPaneProperty(aParentFolder, "favicon", null, "row"); + }, + +/** + * Change an existing feed's url, as identified by FZ_FEED resource in the + * feeds.rdf subscriptions database. + * + * @param obj aFeed - the feed object + * @param string aNewUrl - new url + * @return bool - true if successful, else false + */ + changeUrlForFeed: function(aFeed, aNewUrl) { + if (!aFeed || !aFeed.folder || !aNewUrl) + return false; + + if (this.feedAlreadyExists(aNewUrl, aFeed.folder.server)) + { + this.log.info("FeedUtils.changeUrlForFeed: new feed url " + aNewUrl + + " already subscribed in account " + aFeed.folder.server.prettyName); + return false; + } + + let title = aFeed.title; + let link = aFeed.link; + let quickMode = aFeed.quickMode; + let options = aFeed.options; + + this.deleteFeed(this.rdf.GetResource(aFeed.url), + aFeed.folder.server, aFeed.folder); + aFeed.resource = this.rdf.GetResource(aNewUrl) + .QueryInterface(Ci.nsIRDFResource); + aFeed.title = title; + aFeed.link = link; + aFeed.quickMode = quickMode; + aFeed.options = options; + this.addFeed(aFeed); + + let win = Services.wm.getMostRecentWindow("Mail:News-BlogSubscriptions"); + if (win) + win.FeedSubscriptions.refreshSubscriptionView(aFeed.folder, aNewUrl); + + return true; + }, + +/** + * Get the list of feed urls for a folder, as identified by the FZ_DESTFOLDER + * tag, directly from the primary feeds.rdf subscriptions database. + * + * @param nsIMsgFolder - the folder. + * @return array of urls, or null if none. + */ + getFeedUrlsInFolder: function(aFolder) { + if (aFolder.isServer || aFolder.server.type != "rss" || + aFolder.getFlag(Ci.nsMsgFolderFlags.Trash) || + aFolder.getFlag(Ci.nsMsgFolderFlags.Virtual) || + !aFolder.filePath.exists()) + // There are never any feedUrls in the account/non-feed/trash/virtual + // folders or in a ghost folder (nonexistant on disk yet found in + // aFolder.subFolders). + return null; + + let feedUrlArray = []; + + // Get the list from the feeds database. + try { + let ds = this.getSubscriptionsDS(aFolder.server); + let enumerator = ds.GetSources(this.FZ_DESTFOLDER, aFolder, true); + while (enumerator.hasMoreElements()) + { + let containerArc = enumerator.getNext(); + let uri = containerArc.QueryInterface(Ci.nsIRDFResource).ValueUTF8; + feedUrlArray.push(uri); + } + } + catch(ex) + { + this.log.error("getFeedUrlsInFolder: feeds.rdf db error - " + ex); + this.log.error("getFeedUrlsInFolder: feeds.rdf db error for account - " + + aFolder.server.serverURI + " : " + aFolder.server.prettyName); + } + + return feedUrlArray.length ? feedUrlArray : null; + }, + +/** + * Check if the folder's msgDatabase is openable, reparse if desired. + * + * @param nsIMsgFolder aFolder - the folder + * @param boolean aReparse - reparse if true + * @return boolean - true if msgDb is available, else false + */ + isMsgDatabaseOpenable: function(aFolder, aReparse) { + let msgDb; + try { + msgDb = Cc["@mozilla.org/msgDatabase/msgDBService;1"] + .getService(Ci.nsIMsgDBService).openFolderDB(aFolder, true); + } + catch (ex) {} + + if (msgDb) + return true; + + if (!aReparse) + return false; + + // Force a reparse. + FeedUtils.log.debug("checkMsgDb: rebuild msgDatabase for " + + aFolder.name + " - " + aFolder.filePath.path); + try { + // Ignore error returns. + aFolder.QueryInterface(Ci.nsIMsgLocalMailFolder) + .getDatabaseWithReparse(null, null); + } + catch (ex) {} + + return false; + }, + +/** + * Update a folderpane cached property. + * + * @param nsIMsgFolder aFolder - folder + * @param string aProperty - property + * @param string aValue - value + * @param string aInvalidate - "row" = folder's row. + * "all" = all rows. + */ + setFolderPaneProperty: function(aFolder, aProperty, aValue, aInvalidate) { + let win = Services.wm.getMostRecentWindow("mail:3pane"); + if (!aFolder || !aProperty || !win || !("gFolderTreeView" in win)) + return; + + win.gFolderTreeView.setFolderCacheProperty(aFolder, aProperty, aValue); + + if (aInvalidate == "all") { + win.gFolderTreeView._tree.invalidate(); + } + if (aInvalidate == "row") { + let row = win.gFolderTreeView.getIndexOfFolder(aFolder); + win.gFolderTreeView._tree.invalidateRow(row); + } + }, + +/** + * Get the favicon for a feed folder subscription url (first one) or a feed + * message url. The favicon service caches it in memory if places history is + * not enabled. + * + * @param nsIMsgFolder aFolder - the feed folder or null if aUrl + * @param string aUrl - a url (feed, message, other) or null if aFolder + * @param string aIconUrl - the icon url if already determined, else null + * @param nsIDOMWindow aWindow - null if requesting url without setting it + * @param function aCallback - null or callback + * @return string - the favicon url or empty string + */ + getFavicon: function(aFolder, aUrl, aIconUrl, aWindow, aCallback) { + // On any error, cache an empty string to show the default favicon, and + // don't try anymore in this session. + let useDefaultFavicon = (() => { + if (aCallback) + aCallback(""); + return ""; + }); + + if (!Services.prefs.getBoolPref("browser.chrome.site_icons") || + !Services.prefs.getBoolPref("browser.chrome.favicons")) + return useDefaultFavicon(); + + if (aIconUrl != null) + return aIconUrl; + + let onLoadSuccess = (aEvent => { + let iconUri = Services.io.newURI(aEvent.target.src, null, null); + aWindow.specialTabs.mFaviconService.setAndFetchFaviconForPage( + uri, iconUri, false, + aWindow.specialTabs.mFaviconService.FAVICON_LOAD_NON_PRIVATE, + null, Services.scriptSecurityManager.getSystemPrincipal()); + + if (aCallback) + aCallback(iconUri.spec); + }); + + let onLoadError = (aEvent => { + useDefaultFavicon(); + let url = aEvent.target.src; + aWindow.specialTabs.getFaviconFromPage(url, aCallback); + }); + + let url = aUrl; + if (!url) + { + // Get the proposed iconUrl from the folder's first subscribed feed's + // <link>. + if (!aFolder) + return useDefaultFavicon(); + + let feedUrls = this.getFeedUrlsInFolder(aFolder); + url = feedUrls ? feedUrls[0] : null; + if (!url) + return useDefaultFavicon(); + } + + if (aFolder) + { + let ds = this.getSubscriptionsDS(aFolder.server); + let resource = this.rdf.GetResource(url).QueryInterface(Ci.nsIRDFResource); + let feedLinkUrl = ds.GetTarget(resource, this.RSS_LINK, true); + feedLinkUrl = feedLinkUrl ? + feedLinkUrl.QueryInterface(Ci.nsIRDFLiteral).Value : null; + url = feedLinkUrl && feedLinkUrl.startsWith("http") ? feedLinkUrl : url; + } + + let uri, iconUri; + try { + uri = Services.io.newURI(url, null, null); + iconUri = Services.io.newURI(uri.prePath + "/favicon.ico", null, null); + } + catch (ex) { + return useDefaultFavicon(); + } + + if (!aWindow) + return iconUri.spec; + + aWindow.specialTabs.loadFaviconImageNode(onLoadSuccess, onLoadError, + iconUri.spec); + // Cache the favicon url initially. + if (aCallback) + aCallback(iconUri.spec); + + return iconUri.spec; + }, + +/** + * Update the feeds.rdf database for rename and move/copy folder name changes. + * + * @param nsIMsgFolder aFolder - the folder, new if rename or target of + * move/copy folder (new parent) + * @param nsIMsgFolder aOrigFolder - original folder + * @param string aAction - "move" or "copy" or "rename" + */ + updateSubscriptionsDS: function(aFolder, aOrigFolder, aAction) { + this.log.debug("FeedUtils.updateSubscriptionsDS: " + + "\nfolder changed - " + aAction + + "\nnew folder - " + aFolder.filePath.path + + "\norig folder - " + aOrigFolder.filePath.path); + + if (aFolder.server.type != "rss" || FeedUtils.isInTrash(aOrigFolder)) + // Target not a feed account folder; nothing to do, or move/rename in + // trash; no subscriptions already. + return; + + let newFolder = aFolder; + let newParentURI = aFolder.URI; + let origParentURI = aOrigFolder.URI; + if (aAction == "move" || aAction == "copy") + { + // Get the new folder. Don't process the entire parent (new dest folder)! + newFolder = aFolder.getChildNamed(aOrigFolder.name); + origParentURI = aOrigFolder.parent ? aOrigFolder.parent.URI : + aOrigFolder.rootFolder.URI; + } + + this.updateFolderChangeInFeedsDS(newFolder, aOrigFolder, null, null); + + // There may be subfolders, but we only get a single notification; iterate + // over all descendent folders of the folder whose location has changed. + let newSubFolders = Cc["@mozilla.org/array;1"].createInstance(Ci.nsIMutableArray); + newFolder.ListDescendants(newSubFolders); + for (let i = 0; i < newSubFolders.length; i++) + { + let newSubFolder = newSubFolders.queryElementAt(i, Ci.nsIMsgFolder); + FeedUtils.updateFolderChangeInFeedsDS(newSubFolder, aOrigFolder, + newParentURI, origParentURI) + } + }, + +/** + * Update the feeds.rdf database with the new folder's or subfolder's location + * for rename and move/copy name changes. The feeds.rdf subscriptions db is + * also synced on cross account folder copies. Note that if a copied folder's + * url exists in the new account, its active subscription will be switched to + * the folder being copied, to enforce the one unique url per account design. + * + * @param nsIMsgFolder aFolder - new folder + * @param nsIMsgFolder aOrigFolder - original folder + * @param string aNewAncestorURI - for subfolders, ancestor new folder + * @param string aOrigAncestorURI - for subfolders, ancestor original folder + */ + updateFolderChangeInFeedsDS: function(aFolder, aOrigFolder, + aNewAncestorURI, aOrigAncestorURI) { + this.log.debug("updateFolderChangeInFeedsDS: " + + "\naFolder - " + aFolder.URI + + "\naOrigFolder - " + aOrigFolder.URI + + "\naOrigAncestor - " + aOrigAncestorURI + + "\naNewAncestor - " + aNewAncestorURI); + + // Get the original folder's URI. + let folderURI = aFolder.URI; + let origURI = aNewAncestorURI && aOrigAncestorURI ? + folderURI.replace(aNewAncestorURI, aOrigAncestorURI) : + aOrigFolder.URI; + let origFolderRes = this.rdf.GetResource(origURI); + this.log.debug("updateFolderChangeInFeedsDS: urls origURI - " + origURI); + // Get the original folder's url list from the feeds database. + let feedUrlArray = []; + let dsSrc = this.getSubscriptionsDS(aOrigFolder.server); + try { + let enumerator = dsSrc.GetSources(this.FZ_DESTFOLDER, origFolderRes, true); + while (enumerator.hasMoreElements()) + { + let containerArc = enumerator.getNext(); + let uri = containerArc.QueryInterface(Ci.nsIRDFResource).ValueUTF8; + feedUrlArray.push(uri); + } + } + catch(ex) + { + this.log.error("updateFolderChangeInFeedsDS: feeds.rdf db error for account - " + + aOrigFolder.server.prettyName + " : " + ex); + } + + if (!feedUrlArray.length) + { + this.log.debug("updateFolderChangeInFeedsDS: no feedUrls in this folder"); + return; + } + + let id, resource, node; + let ds = this.getSubscriptionsDS(aFolder.server); + for (let feedUrl of feedUrlArray) + { + this.log.debug("updateFolderChangeInFeedsDS: feedUrl - " + feedUrl); + + id = this.rdf.GetResource(feedUrl); + // If move to trash, unsubscribe. + if (this.isInTrash(aFolder)) + { + this.deleteFeed(id, aFolder.server, aFolder); + } + else + { + resource = this.rdf.GetResource(aFolder.URI); + // Get the node for the current folder URI. + node = ds.GetTarget(id, this.FZ_DESTFOLDER, true); + if (node) + { + ds.Change(id, this.FZ_DESTFOLDER, node, resource); + } + else + { + // If adding a new feed it's a cross account action; make sure to + // preserve all properties from the original datasource where + // available. Otherwise use the new folder's name and default server + // quickMode; preserve link and options. + let feedTitle = dsSrc.GetTarget(id, this.DC_TITLE, true); + feedTitle = feedTitle ? feedTitle.QueryInterface(Ci.nsIRDFLiteral).Value : + resource.name; + let link = dsSrc.GetTarget(id, FeedUtils.RSS_LINK, true); + link = link ? link.QueryInterface(Ci.nsIRDFLiteral).Value : ""; + let quickMode = dsSrc.GetTarget(id, this.FZ_QUICKMODE, true); + quickMode = quickMode ? quickMode.QueryInterface(Ci.nsIRDFLiteral).Value : + null; + quickMode = quickMode == "true" ? true : + quickMode == "false" ? false : + aFeed.folder.server.getBoolValue("quickMode"); + let options = dsSrc.GetTarget(id, this.FZ_OPTIONS, true); + options = options ? JSON.parse(options.QueryInterface(Ci.nsIRDFLiteral).Value) : + this.optionsTemplate; + + let feed = new Feed(id, aFolder.server); + feed.folder = aFolder; + feed.title = feedTitle; + feed.link = link; + feed.quickMode = quickMode; + feed.options = options; + this.addFeed(feed); + } + } + } + + ds.Flush(); + }, + +/** + * When subscribing to feeds by dnd on, or adding a url to, the account + * folder (only), or creating folder structure via opml import, a subfolder is + * autocreated and thus the derived/given name must be sanitized to prevent + * filesystem errors. Hashing invalid chars based on OS rather than filesystem + * is not strictly correct. + * + * @param nsIMsgFolder aParentFolder - parent folder + * @param string aProposedName - proposed name + * @param string aDefaultName - default name if proposed sanitizes to + * blank, caller ensures sane value + * @param bool aUnique - if true, return a unique indexed name. + * @return string - sanitized unique name + */ + getSanitizedFolderName: function(aParentFolder, aProposedName, aDefaultName, aUnique) { + // Clean up the name for the strictest fs (fat) and to ensure portability. + // 1) Replace line breaks and tabs '\n\r\t' with a space. + // 2) Remove nonprintable ascii. + // 3) Remove invalid win chars '* | \ / : < > ? "'. + // 4) Remove all '.' as starting/ending with one is trouble on osx/win. + // 5) No leading/trailing spaces. + let folderName = aProposedName.replace(/[\n\r\t]+/g, " ") + .replace(/[\x00-\x1F]+/g, "") + .replace(/[*|\\\/:<>?"]+/g, "") + .replace(/[\.]+/g, "") + .trim(); + + // Prefix with __ if name is: + // 1) a reserved win filename. + // 2) an undeletable/unrenameable special folder name (bug 259184). + if (folderName.toUpperCase() + .match(/^COM\d$|^LPT\d$|^CON$|PRN$|^AUX$|^NUL$|^CLOCK\$/) || + folderName.toUpperCase() + .match(/^INBOX$|^OUTBOX$|^UNSENT MESSAGES$|^TRASH$/)) + folderName = "__" + folderName; + + // Use a default if no name is found. + if (!folderName) + folderName = aDefaultName; + + if (!aUnique) + return folderName; + + // Now ensure the folder name is not a dupe; if so append index. + let folderNameBase = folderName; + let i = 2; + while (aParentFolder.containsChildNamed(folderName)) + { + folderName = folderNameBase + "-" + i++; + } + + return folderName; + }, + +/** + * This object will contain all feed specific properties. + */ + _optionsDefault: { + version: 1, + // Autotag and <category> handling options. + category: { + enabled: false, + prefixEnabled: false, + prefix: null, + } + }, + + get optionsTemplate() + { + // Copy the object. + return JSON.parse(JSON.stringify(this._optionsDefault)); + }, + + getOptionsAcct: function(aServer) + { + let optionsAcctPref = "mail.server." + aServer.key + ".feed_options"; + try { + return JSON.parse(Services.prefs.getCharPref(optionsAcctPref)); + } + catch (ex) { + this.setOptionsAcct(aServer, this._optionsDefault); + return JSON.parse(Services.prefs.getCharPref(optionsAcctPref)); + } + }, + + setOptionsAcct: function(aServer, aOptions) + { + let optionsAcctPref = "mail.server." + aServer.key + ".feed_options"; + let newOptions = this.newOptions(aOptions); + Services.prefs.setCharPref(optionsAcctPref, JSON.stringify(newOptions)); + }, + + newOptions: function(aOptions) + { + // TODO: Clean options, so that only keys in the active template are stored. + return aOptions; + }, + + getSubscriptionsDS: function(aServer) { + if (this[aServer.serverURI] && this[aServer.serverURI]["FeedsDS"]) + return this[aServer.serverURI]["FeedsDS"]; + + let file = this.getSubscriptionsFile(aServer); + let url = Services.io.getProtocolHandler("file"). + QueryInterface(Ci.nsIFileProtocolHandler). + getURLSpecFromFile(file); + + // GetDataSourceBlocking has a cache, so it's cheap to do this again + // once we've already done it once. + let ds = this.rdf.GetDataSourceBlocking(url); + + if (!ds) + throw new Error("FeedUtils.getSubscriptionsDS: can't get feed " + + "subscriptions data source - " + url); + + if (!this[aServer.serverURI]) + this[aServer.serverURI] = {}; + return this[aServer.serverURI]["FeedsDS"] = + ds.QueryInterface(Ci.nsIRDFRemoteDataSource); + }, + + getSubscriptionsList: function(aDataSource) { + let list = aDataSource.GetTarget(this.FZ_ROOT, this.FZ_FEEDS, true); + list = list.QueryInterface(Ci.nsIRDFResource); + list = this.rdfContainerUtils.MakeSeq(aDataSource, list); + return list; + }, + + getSubscriptionsFile: function(aServer) { + aServer.QueryInterface(Ci.nsIRssIncomingServer); + let file = aServer.subscriptionsDataSourcePath; + + // If the file doesn't exist, create it. + if (!file.exists()) + this.createFile(file, this.FEEDS_TEMPLATE); + + return file; + }, + + FEEDS_TEMPLATE: '<?xml version="1.0"?>\n' + + '<RDF:RDF xmlns:dc="http://purl.org/dc/elements/1.1/"\n' + + ' xmlns:fz="urn:forumzilla:"\n' + + ' xmlns:RDF="http://www.w3.org/1999/02/22-rdf-syntax-ns#">\n' + + ' <RDF:Description about="urn:forumzilla:root">\n' + + ' <fz:feeds>\n' + + ' <RDF:Seq>\n' + + ' </RDF:Seq>\n' + + ' </fz:feeds>\n' + + ' </RDF:Description>\n' + + '</RDF:RDF>\n', + + getItemsDS: function(aServer) { + if (this[aServer.serverURI] && this[aServer.serverURI]["FeedItemsDS"]) + return this[aServer.serverURI]["FeedItemsDS"]; + + let file = this.getItemsFile(aServer); + let url = Services.io.getProtocolHandler("file"). + QueryInterface(Ci.nsIFileProtocolHandler). + getURLSpecFromFile(file); + + // GetDataSourceBlocking has a cache, so it's cheap to do this again + // once we've already done it once. + let ds = this.rdf.GetDataSourceBlocking(url); + if (!ds) + throw new Error("FeedUtils.getItemsDS: can't get feed items " + + "data source - " + url); + + // Note that it this point the datasource may not be loaded yet. + // You have to QueryInterface it to nsIRDFRemoteDataSource and check + // its "loaded" property to be sure. You can also attach an observer + // which will get notified when the load is complete. + if (!this[aServer.serverURI]) + this[aServer.serverURI] = {}; + return this[aServer.serverURI]["FeedItemsDS"] = + ds.QueryInterface(Ci.nsIRDFRemoteDataSource); + }, + + getItemsFile: function(aServer) { + aServer.QueryInterface(Ci.nsIRssIncomingServer); + let file = aServer.feedItemsDataSourcePath; + + // If the file doesn't exist, create it. + if (!file.exists()) { + this.createFile(file, this.FEEDITEMS_TEMPLATE); + return file; + } + + // If feeditems.rdf is not sane, duplicate messages will occur repeatedly + // until the file is corrected; check that the file is valid XML. This is + // done lazily only once in a session. + let fileUrl = Services.io.getProtocolHandler("file") + .QueryInterface(Ci.nsIFileProtocolHandler) + .getURLSpecFromFile(file); + let request = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"] + .createInstance(Ci.nsIXMLHttpRequest); + request.open("GET", fileUrl, false); + request.responseType = "document"; + request.send(); + let dom = request.responseXML; + if (dom instanceof Ci.nsIDOMXMLDocument && + dom.documentElement.namespaceURI != this.MOZ_PARSERERROR_NS) + return file; + + // Error on the file. Rename it and create a new one. + this.log.debug("FeedUtils.getItemsFile: error in feeditems.rdf"); + let errName = "feeditems_error_" + + (new Date().toISOString()).replace(/\D/g, "") + ".rdf"; + file.moveTo(file.parent, errName); + file = aServer.feedItemsDataSourcePath; + this.createFile(file, this.FEEDITEMS_TEMPLATE); + this.log.error("FeedUtils.getItemsFile: error in feeditems.rdf in account '" + + aServer.prettyName + "'; the file has been moved to " + + errName + " and a new file has been created. Recent messages " + + "may be duplicated."); + return file; + }, + + FEEDITEMS_TEMPLATE: '<?xml version="1.0"?>\n' + + '<RDF:RDF xmlns:dc="http://purl.org/dc/elements/1.1/"\n' + + ' xmlns:fz="urn:forumzilla:"\n' + + ' xmlns:RDF="http://www.w3.org/1999/02/22-rdf-syntax-ns#">\n' + + '</RDF:RDF>\n', + + createFile: function(aFile, aTemplate) { + let fos = FileUtils.openSafeFileOutputStream(aFile); + fos.write(aTemplate, aTemplate.length); + FileUtils.closeSafeFileOutputStream(fos); + }, + + getParentTargetForChildResource: function(aChildResource, aParentTarget, + aServer) { + // Generic get feed property, based on child value. Assumes 1 unique + // child value with 1 unique parent, valid for feeds.rdf structure. + let ds = this.getSubscriptionsDS(aServer); + let childRes = this.rdf.GetResource(aChildResource); + let parent = null; + + let arcsIn = ds.ArcLabelsIn(childRes); + while (arcsIn.hasMoreElements()) + { + let arc = arcsIn.getNext(); + if (arc instanceof Ci.nsIRDFResource) + { + parent = ds.GetSource(arc, childRes, true); + parent = parent.QueryInterface(Ci.nsIRDFResource); + break; + } + } + + if (parent) + { + let resource = this.rdf.GetResource(parent.Value); + return ds.GetTarget(resource, aParentTarget, true); + } + + return null; + }, + + removeAssertions: function(aDataSource, aResource) { + let properties = aDataSource.ArcLabelsOut(aResource); + let property; + while (properties.hasMoreElements()) + { + property = properties.getNext(); + let values = aDataSource.GetTargets(aResource, property, true); + let value; + while (values.hasMoreElements()) + { + value = values.getNext(); + aDataSource.Unassert(aResource, property, value, true); + } + } + }, + +/** + * Dragging something from somewhere. It may be a nice x-moz-url or from a + * browser or app that provides a less nice dataTransfer object in the event. + * Extract the url and if it passes the scheme test, try to subscribe. + * + * @param nsIDOMDataTransfer aDataTransfer - the dnd event's dataTransfer. + * @return nsIURI uri - a uri if valid, null if none. + */ + getFeedUriFromDataTransfer: function(aDataTransfer) { + let dt = aDataTransfer; + let types = ["text/x-moz-url-data", "text/x-moz-url"]; + let validUri = false; + let uri = Cc["@mozilla.org/network/standard-url;1"]. + createInstance(Ci.nsIURI); + + if (dt.getData(types[0])) + { + // The url is the data. + uri.spec = dt.mozGetDataAt(types[0], 0); + validUri = this.isValidScheme(uri); + this.log.trace("getFeedUriFromDataTransfer: dropEffect:type:value - " + + dt.dropEffect + " : " + types[0] + " : " + uri.spec); + } + else if (dt.getData(types[1])) + { + // The url is the first part of the data, the second part is random. + uri.spec = dt.mozGetDataAt(types[1], 0).split("\n")[0]; + validUri = this.isValidScheme(uri); + this.log.trace("getFeedUriFromDataTransfer: dropEffect:type:value - " + + dt.dropEffect + " : " + types[0] + " : " + uri.spec); + } + else + { + // Go through the types and see if there's a url; get the first one. + for (let i = 0; i < dt.types.length; i++) { + let spec = dt.mozGetDataAt(dt.types[i], 0); + this.log.trace("getFeedUriFromDataTransfer: dropEffect:index:type:value - " + + dt.dropEffect + " : " + i + " : " + dt.types[i] + " : "+spec); + try { + uri.spec = spec; + validUri = this.isValidScheme(uri); + } + catch(ex) {} + + if (validUri) + break; + }; + } + + return validUri ? uri : null; + }, + + /** + * Returns security/certificate/network error details for an XMLHTTPRequest. + * + * @param XMLHTTPRequest xhr - The xhr request. + * @return array [string errType, string errName] (null if not determined). + */ + createTCPErrorFromFailedXHR: function(xhr) { + let status = xhr.channel.QueryInterface(Ci.nsIRequest).status; + + let errType = null; + let errName = null; + if ((status & 0xff0000) === 0x5a0000) { + // Security module. + const nsINSSErrorsService = Ci.nsINSSErrorsService; + let nssErrorsService = Cc["@mozilla.org/nss_errors_service;1"] + .getService(nsINSSErrorsService); + let errorClass; + + // getErrorClass()) will throw a generic NS_ERROR_FAILURE if the error + // code is somehow not in the set of covered errors. + try { + errorClass = nssErrorsService.getErrorClass(status); + } + catch (ex) { + // Catch security protocol exception. + errorClass = "SecurityProtocol"; + } + + if (errorClass == nsINSSErrorsService.ERROR_CLASS_BAD_CERT) { + errType = "SecurityCertificate"; + } + else { + errType = "SecurityProtocol"; + } + + // NSS_SEC errors (happen below the base value because of negative vals). + if ((status & 0xffff) < Math.abs(nsINSSErrorsService.NSS_SEC_ERROR_BASE)) { + // The bases are actually negative, so in our positive numeric space, + // we need to subtract the base off our value. + let nssErr = Math.abs(nsINSSErrorsService.NSS_SEC_ERROR_BASE) - (status & 0xffff); + + switch (nssErr) { + case 11: // SEC_ERROR_EXPIRED_CERTIFICATE, sec(11) + errName = "SecurityExpiredCertificateError"; + break; + case 12: // SEC_ERROR_REVOKED_CERTIFICATE, sec(12) + errName = "SecurityRevokedCertificateError"; + break; + + // Per bsmith, we will be unable to tell these errors apart very soon, + // so it makes sense to just folder them all together already. + case 13: // SEC_ERROR_UNKNOWN_ISSUER, sec(13) + case 20: // SEC_ERROR_UNTRUSTED_ISSUER, sec(20) + case 21: // SEC_ERROR_UNTRUSTED_CERT, sec(21) + case 36: // SEC_ERROR_CA_CERT_INVALID, sec(36) + errName = "SecurityUntrustedCertificateIssuerError"; + break; + case 90: // SEC_ERROR_INADEQUATE_KEY_USAGE, sec(90) + errName = "SecurityInadequateKeyUsageError"; + break; + case 176: // SEC_ERROR_CERT_SIGNATURE_ALGORITHM_DISABLED, sec(176) + errName = "SecurityCertificateSignatureAlgorithmDisabledError"; + break; + default: + errName = "SecurityError"; + break; + } + } + else { + // Calculating the difference. + let sslErr = Math.abs(nsINSSErrorsService.NSS_SSL_ERROR_BASE) - (status & 0xffff); + + switch (sslErr) { + case 3: // SSL_ERROR_NO_CERTIFICATE, ssl(3) + errName = "SecurityNoCertificateError"; + break; + case 4: // SSL_ERROR_BAD_CERTIFICATE, ssl(4) + errName = "SecurityBadCertificateError"; + break; + case 8: // SSL_ERROR_UNSUPPORTED_CERTIFICATE_TYPE, ssl(8) + errName = "SecurityUnsupportedCertificateTypeError"; + break; + case 9: // SSL_ERROR_UNSUPPORTED_VERSION, ssl(9) + errName = "SecurityUnsupportedTLSVersionError"; + break; + case 12: // SSL_ERROR_BAD_CERT_DOMAIN, ssl(12) + errName = "SecurityCertificateDomainMismatchError"; + break; + default: + errName = "SecurityError"; + break; + } + } + } + else { + errType = "Network"; + switch (status) { + // Connect to host:port failed. + case 0x804B000C: // NS_ERROR_CONNECTION_REFUSED, network(13) + errName = "ConnectionRefusedError"; + break; + // network timeout error. + case 0x804B000E: // NS_ERROR_NET_TIMEOUT, network(14) + errName = "NetworkTimeoutError"; + break; + // Hostname lookup failed. + case 0x804B001E: // NS_ERROR_UNKNOWN_HOST, network(30) + errName = "DomainNotFoundError"; + break; + case 0x804B0047: // NS_ERROR_NET_INTERRUPT, network(71) + errName = "NetworkInterruptError"; + break; + default: + errName = "NetworkError"; + break; + } + } + + return [errType, errName]; + }, + +/** + * Returns if a uri/url is valid to subscribe. + * + * @param nsIURI aUri or string aUrl - the Uri/Url. + * @return boolean - true if a valid scheme, false if not. + */ + _validSchemes: ["http", "https"], + isValidScheme: function(aUri) { + if (!(aUri instanceof Ci.nsIURI)) { + try { + aUri = Services.io.newURI(aUri, null, null); + } + catch (ex) { + return false; + } + } + + return (this._validSchemes.indexOf(aUri.scheme) != -1); + }, + +/** + * Is a folder Trash or in Trash. + * + * @param nsIMsgFolder aFolder - the folder. + * @return boolean - true if folder is Trash else false. + */ + isInTrash: function(aFolder) { + let trashFolder = + aFolder.rootFolder.getFolderWithFlags(Ci.nsMsgFolderFlags.Trash); + if (trashFolder && + (trashFolder == aFolder || trashFolder.isAncestorOf(aFolder))) + return true; + return false; + }, + +/** + * Return a folder path string constructed from individual folder UTF8 names + * stored as properties (not possible hashes used to construct disk foldername). + * + * @param nsIMsgFolder aFolder - the folder. + * @return string prettyName | null - name or null if not a disk folder. + */ + getFolderPrettyPath: function(aFolder) { + let msgFolder = MailUtils.getFolderForURI(aFolder.URI, true); + if (!msgFolder) + // Not a real folder uri. + return null; + + if (msgFolder.URI == msgFolder.server.serverURI) + return msgFolder.server.prettyName; + + // Server part first. + let pathParts = [msgFolder.server.prettyName]; + let rawPathParts = msgFolder.URI.split(msgFolder.server.serverURI + "/"); + let folderURI = msgFolder.server.serverURI; + rawPathParts = rawPathParts[1].split("/"); + for (let i = 0; i < rawPathParts.length - 1; i++) + { + // Two or more folders deep parts here. + folderURI += "/" + rawPathParts[i]; + msgFolder = MailUtils.getFolderForURI(folderURI, true); + pathParts.push(msgFolder.name); + } + + // Leaf folder last. + pathParts.push(aFolder.name); + return pathParts.join("/"); + }, + +/** + * Date validator for feeds. + * + * @param string aDate - date string + * @return boolean - true if passes regex test, false if not + */ + isValidRFC822Date: function(aDate) + { + const FZ_RFC822_RE = "^(((Mon)|(Tue)|(Wed)|(Thu)|(Fri)|(Sat)|(Sun)), *)?\\d\\d?" + + " +((Jan)|(Feb)|(Mar)|(Apr)|(May)|(Jun)|(Jul)|(Aug)|(Sep)|(Oct)|(Nov)|(Dec))" + + " +\\d\\d(\\d\\d)? +\\d\\d:\\d\\d(:\\d\\d)? +(([+-]?\\d\\d\\d\\d)|(UT)|(GMT)" + + "|(EST)|(EDT)|(CST)|(CDT)|(MST)|(MDT)|(PST)|(PDT)|\\w)$"; + let regex = new RegExp(FZ_RFC822_RE); + return regex.test(aDate); + }, + +/** + * Create rfc5322 date. + * + * @param [string] aDateString - optional date string; if null or invalid + * date, get the current datetime. + * @return string - an rfc5322 date string + */ + getValidRFC5322Date: function(aDateString) + { + let d = new Date(aDateString || new Date().getTime()); + d = isNaN(d.getTime()) ? new Date() : d; + return jsmime.headeremitter.emitStructuredHeader("Date", d, {}).substring(6).trim(); + }, + + // Progress glue code. Acts as a go between the RSS back end and the mail + // window front end determined by the aMsgWindow parameter passed into + // nsINewsBlogFeedDownloader. + progressNotifier: { + mSubscribeMode: false, + mMsgWindow: null, + mStatusFeedback: null, + mFeeds: {}, + // Keeps track of the total number of feeds we have been asked to download. + // This number may not reflect the # of entries in our mFeeds array because + // not all feeds may have reported in for the first time. + mNumPendingFeedDownloads: 0, + + init: function(aMsgWindow, aSubscribeMode) + { + if (!this.mNumPendingFeedDownloads) + { + // If we aren't already in the middle of downloading feed items. + this.mStatusFeedback = aMsgWindow ? aMsgWindow.statusFeedback : null; + this.mSubscribeMode = aSubscribeMode; + this.mMsgWindow = aMsgWindow; + + if (this.mStatusFeedback) + { + this.mStatusFeedback.startMeteors(); + this.mStatusFeedback.showStatusString( + FeedUtils.strings.GetStringFromName( + aSubscribeMode ? "subscribe-validating-feed" : + "newsblog-getNewMsgsCheck")); + } + } + }, + + downloaded: function(feed, aErrorCode) + { + let location = feed.folder ? feed.folder.filePath.path : ""; + FeedUtils.log.debug("downloaded: "+ + (this.mSubscribeMode ? "Subscribe " : "Update ") + + "errorCode:feedName:folder - " + + aErrorCode + " : " + feed.name + " : " + location); + if (this.mSubscribeMode) + { + if (aErrorCode == FeedUtils.kNewsBlogSuccess) + { + // Add the feed to the databases. + FeedUtils.addFeed(feed); + + // Nice touch: select the folder that now contains the newly subscribed + // feed. This is particularly nice if we just finished subscribing + // to a feed URL that the operating system gave us. + this.mMsgWindow.windowCommands.selectFolder(feed.folder.URI); + + // Check for an existing feed subscriptions window and update it. + let subscriptionsWindow = + Services.wm.getMostRecentWindow("Mail:News-BlogSubscriptions"); + if (subscriptionsWindow) + subscriptionsWindow.FeedSubscriptions. + FolderListener.folderAdded(feed.folder); + } + else + { + // Non success. Remove intermediate traces from the feeds database. + if (feed && feed.url && feed.server) + FeedUtils.deleteFeed(FeedUtils.rdf.GetResource(feed.url), + feed.server, + feed.server.rootFolder); + } + } + + if (feed.folder && aErrorCode != FeedUtils.kNewsBlogFeedIsBusy) + // Free msgDatabase after new mail biff is set; if busy let the next + // result do the freeing. Otherwise new messages won't be indicated. + feed.folder.msgDatabase = null; + + let message = ""; + if (feed.folder) + location = FeedUtils.getFolderPrettyPath(feed.folder) + " -> "; + switch (aErrorCode) { + case FeedUtils.kNewsBlogSuccess: + case FeedUtils.kNewsBlogFeedIsBusy: + message = ""; + break; + case FeedUtils.kNewsBlogNoNewItems: + message = feed.url+". " + + FeedUtils.strings.GetStringFromName( + "newsblog-noNewArticlesForFeed"); + break; + case FeedUtils.kNewsBlogInvalidFeed: + message = FeedUtils.strings.formatStringFromName( + "newsblog-feedNotValid", [feed.url], 1); + break; + case FeedUtils.kNewsBlogRequestFailure: + message = FeedUtils.strings.formatStringFromName( + "newsblog-networkError", [feed.url], 1); + break; + case FeedUtils.kNewsBlogFileError: + message = FeedUtils.strings.GetStringFromName( + "subscribe-errorOpeningFile"); + break; + case FeedUtils.kNewsBlogBadCertError: + let host = Services.io.newURI(feed.url, null, null).host; + message = FeedUtils.strings.formatStringFromName( + "newsblog-badCertError", [host], 1); + break; + case FeedUtils.kNewsBlogNoAuthError: + message = FeedUtils.strings.formatStringFromName( + "newsblog-noAuthError", [feed.url], 1); + break; + } + if (message) + FeedUtils.log.info("downloaded: " + + (this.mSubscribeMode ? "Subscribe: " : "Update: ") + + location + message); + + if (this.mStatusFeedback) + { + this.mStatusFeedback.showStatusString(message); + this.mStatusFeedback.stopMeteors(); + } + + if (!--this.mNumPendingFeedDownloads) + { + FeedUtils.getSubscriptionsDS(feed.server).Flush(); + this.mFeeds = {}; + this.mSubscribeMode = false; + FeedUtils.log.debug("downloaded: all pending downloads finished"); + + // Should we do this on a timer so the text sticks around for a little + // while? It doesnt look like we do it on a timer for newsgroups so + // we'll follow that model. Don't clear the status text if we just + // dumped an error to the status bar! + if (aErrorCode == FeedUtils.kNewsBlogSuccess && this.mStatusFeedback) + this.mStatusFeedback.showStatusString(""); + } + + feed = null; + }, + + // This gets called after the RSS parser finishes storing a feed item to + // disk. aCurrentFeedItems is an integer corresponding to how many feed + // items have been downloaded so far. aMaxFeedItems is an integer + // corresponding to the total number of feed items to download + onFeedItemStored: function (feed, aCurrentFeedItems, aMaxFeedItems) + { + // We currently don't do anything here. Eventually we may add status + // text about the number of new feed articles received. + + if (this.mSubscribeMode && this.mStatusFeedback) + { + // If we are subscribing to a feed, show feed download progress. + this.mStatusFeedback.showStatusString( + FeedUtils.strings.formatStringFromName("subscribe-gettingFeedItems", + [aCurrentFeedItems, aMaxFeedItems], 2)); + this.onProgress(feed, aCurrentFeedItems, aMaxFeedItems); + } + }, + + onProgress: function(feed, aProgress, aProgressMax, aLengthComputable) + { + if (feed.url in this.mFeeds) + // Have we already seen this feed? + this.mFeeds[feed.url].currentProgress = aProgress; + else + this.mFeeds[feed.url] = {currentProgress: aProgress, + maxProgress: aProgressMax}; + + this.updateProgressBar(); + }, + + updateProgressBar: function() + { + let currentProgress = 0; + let maxProgress = 0; + for (let index in this.mFeeds) + { + currentProgress += this.mFeeds[index].currentProgress; + maxProgress += this.mFeeds[index].maxProgress; + } + + // If we start seeing weird "jumping" behavior where the progress bar + // goes below a threshold then above it again, then we can factor a + // fudge factor here based on the number of feeds that have not reported + // yet and the avg progress we've already received for existing feeds. + // Fortunately the progressmeter is on a timer and only updates every so + // often. For the most part all of our request have initial progress + // before the UI actually picks up a progress value. + if (this.mStatusFeedback) + { + let progress = (currentProgress * 100) / maxProgress; + this.mStatusFeedback.showProgress(progress); + } + } + } +}; + +XPCOMUtils.defineLazyGetter(FeedUtils, "log", function() { + return Log4Moz.getConfiguredLogger("Feeds"); +}); + +XPCOMUtils.defineLazyGetter(FeedUtils, "strings", function() { + return Services.strings.createBundle( + "chrome://messenger-newsblog/locale/newsblog.properties"); +}); + +XPCOMUtils.defineLazyGetter(FeedUtils, "rdf", function() { + return Cc["@mozilla.org/rdf/rdf-service;1"]. + getService(Ci.nsIRDFService); +}); + +XPCOMUtils.defineLazyGetter(FeedUtils, "rdfContainerUtils", function() { + return Cc["@mozilla.org/rdf/container-utils;1"]. + getService(Ci.nsIRDFContainerUtils); +}); diff --git a/mailnews/extensions/newsblog/content/am-newsblog.js b/mailnews/extensions/newsblog/content/am-newsblog.js new file mode 100644 index 000000000..674280f81 --- /dev/null +++ b/mailnews/extensions/newsblog/content/am-newsblog.js @@ -0,0 +1,63 @@ +/* -*- Mode: JavaScript; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +Components.utils.import("resource:///modules/FeedUtils.jsm"); + +var gServer, autotagEnable, autotagUsePrefix, autotagPrefix; + +function onInit(aPageId, aServerId) +{ + var accountName = document.getElementById("server.prettyName"); + var title = document.getElementById("am-newsblog-title"); + var defaultTitle = title.getAttribute("defaultTitle"); + + var titleValue; + if (accountName.value) + titleValue = defaultTitle + " - <" + accountName.value + ">"; + else + titleValue = defaultTitle; + + title.setAttribute("title", titleValue); + document.title = titleValue; + + onCheckItem("server.biffMinutes", ["server.doBiff"]); + + autotagEnable = document.getElementById("autotagEnable"); + autotagUsePrefix = document.getElementById("autotagUsePrefix"); + autotagPrefix = document.getElementById("autotagPrefix"); + + let categoryPrefsAcct = FeedUtils.getOptionsAcct(gServer).category; + autotagEnable.checked = categoryPrefsAcct.enabled; + autotagUsePrefix.disabled = !autotagEnable.checked; + autotagUsePrefix.checked = categoryPrefsAcct.prefixEnabled; + autotagPrefix.disabled = autotagUsePrefix.disabled || !autotagUsePrefix.checked; + autotagPrefix.value = categoryPrefsAcct.prefix; +} + +function onPreInit(account, accountValues) +{ + gServer = account.incomingServer; +} + +function setCategoryPrefs(aNode) +{ + let options = FeedUtils.getOptionsAcct(gServer); + switch (aNode.id) { + case "autotagEnable": + options.category.enabled = aNode.checked; + autotagUsePrefix.disabled = !aNode.checked; + autotagPrefix.disabled = !aNode.checked || !autotagUsePrefix.checked; + break; + case "autotagUsePrefix": + options.category.prefixEnabled = aNode.checked; + autotagPrefix.disabled = aNode.disabled || !aNode.checked; + break; + case "autotagPrefix": + options.category.prefix = aNode.value; + break; + } + + FeedUtils.setOptionsAcct(gServer, options) +} diff --git a/mailnews/extensions/newsblog/content/am-newsblog.xul b/mailnews/extensions/newsblog/content/am-newsblog.xul new file mode 100644 index 000000000..19173d803 --- /dev/null +++ b/mailnews/extensions/newsblog/content/am-newsblog.xul @@ -0,0 +1,155 @@ +<?xml version="1.0"?> +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> + +<?xml-stylesheet href="chrome://messenger/skin/accountManage.css" type="text/css"?> +<?xml-stylesheet href="chrome://messenger-newsblog/skin/feed-subscriptions.css" type="text/css"?> + +<!DOCTYPE page [ +<!ENTITY % newsblogDTD SYSTEM "chrome://messenger-newsblog/locale/am-newsblog.dtd" > +%newsblogDTD; +<!ENTITY % feedDTD SYSTEM "chrome://messenger-newsblog/locale/feed-subscriptions.dtd" > +%feedDTD; +<!ENTITY % accountNoIdentDTD SYSTEM "chrome://messenger/locale/am-serverwithnoidentities.dtd" > +%accountNoIdentDTD; +<!ENTITY % accountServerTopDTD SYSTEM "chrome://messenger/locale/am-server-top.dtd"> +%accountServerTopDTD; +]> + +<page xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul" + class="color-dialog" + title="&accountTitle.label;" + onload="parent.onPanelLoaded('am-newsblog.xul');"> + + <script type="application/javascript" + src="chrome://messenger/content/AccountManager.js"/> + <script type="application/javascript" + src="chrome://messenger-newsblog/content/am-newsblog.js"/> + <script type="application/javascript" + src="chrome://messenger-newsblog/content/newsblogOverlay.js"/> + <script type="application/javascript" + src="chrome://messenger/content/amUtils.js"/> + <script type="application/javascript" + src="chrome://messenger/content/am-prefs.js"/> + + <vbox flex="1" style="overflow: auto;"> + + <dialogheader id="am-newsblog-title" defaultTitle="&accountTitle.label;"/> + + <description class="secDesc">&accountSettingsDesc.label;</description> + + <hbox align="center"> + <label value="&accountName.label;" + accesskey="&accountName.accesskey;" + control="server.prettyName"/> + <textbox id="server.prettyName" + wsm_persist="true" + size="30" + prefstring="mail.server.%serverkey%.name"/> + </hbox> + + <separator class="thin"/> + + <groupbox> + <caption label="&serverSettings.label;"/> + + <checkbox id="server.loginAtStartUp" + wsm_persist="true" + label="&loginAtStartup.label;" + accesskey="&loginAtStartup.accesskey;" + prefattribute="value" + prefstring="mail.server.%serverkey%.login_at_startup"/> + + <hbox align="center"> + <checkbox id="server.doBiff" + wsm_persist="true" + label="&biffStart.label;" + accesskey="&biffStart.accesskey;" + oncommand="onCheckItem('server.biffMinutes', [this.id]);" + prefattribute="value" + prefstring="mail.server.%serverkey%.check_new_mail"/> + <textbox id="server.biffMinutes" + wsm_persist="true" + type="number" + size="3" + min="1" + increment="1" + preftype="int" + prefstring="mail.server.%serverkey%.check_time" + aria-labelledby="server.doBiff server.biffMinutes biffEnd"/> + <label id="biffEnd" + value="&biffEnd.label;" + control="server.biffMinutes"/> + </hbox> + + <checkbox id="server.quickMode" + wsm_persist="true" + genericattr="true" + label="&useQuickMode.label;" + accesskey="&useQuickMode.accesskey;" + preftype="bool" + prefattribute="value" + prefstring="mail.server.%serverkey%.quickMode"/> + + <checkbox id="autotagEnable" + accesskey="&autotagEnable.accesskey;" + label="&autotagEnable.label;" + oncommand="setCategoryPrefs(this)"/> + <hbox> + <checkbox id="autotagUsePrefix" + class="indent" + accesskey="&autotagUsePrefix.accesskey;" + label="&autotagUsePrefix.label;" + oncommand="setCategoryPrefs(this)"/> + <textbox id="autotagPrefix" + placeholder="&autoTagPrefix.placeholder;" + clickSelectsAll="true" + onchange="setCategoryPrefs(this)"/> + </hbox> + </groupbox> + + <separator class="thin"/> + + <groupbox> + <caption label="&messageStorage.label;"/> + + <checkbox id="server.emptyTrashOnExit" + wsm_persist="true" + label="&emptyTrashOnExit.label;" + accesskey="&emptyTrashOnExit.accesskey;" + prefattribute="value" + prefstring="mail.server.%serverkey%.empty_trash_on_exit"/> + + <separator class="thin"/> + + <vbox> + <label value="&localPath.label;" control="server.localPath"/> + <hbox align="center"> + <textbox readonly="true" + wsm_persist="true" + flex="1" + id="server.localPath" + datatype="nsIFile" + prefstring="mail.server.%serverkey%.directory" + class="uri-element"/> + <button id="browseForLocalFolder" + label="&browseFolder.label;" + filepickertitle="&localFolderPicker.label;" + accesskey="&browseFolder.accesskey;" + oncommand="BrowseForLocalFolders();"/> + </hbox> + </vbox> + + </groupbox> + + <separator class="thin"/> + + <hbox align="center"> + <spacer flex="1"/> + <button label="&manageSubscriptions.label;" + accesskey="&manageSubscriptions.accesskey;" + oncommand="openSubscriptionsDialog(gServer.rootFolder);"/> + </hbox> + </vbox> +</page> diff --git a/mailnews/extensions/newsblog/content/feed-parser.js b/mailnews/extensions/newsblog/content/feed-parser.js new file mode 100644 index 000000000..660333422 --- /dev/null +++ b/mailnews/extensions/newsblog/content/feed-parser.js @@ -0,0 +1,1034 @@ +/* -*- Mode: JavaScript; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// The feed parser depends on FeedItem.js, Feed.js. +function FeedParser() { + this.mSerializer = Cc["@mozilla.org/xmlextras/xmlserializer;1"]. + createInstance(Ci.nsIDOMSerializer); +} + +FeedParser.prototype = +{ + // parseFeed() returns an array of parsed items ready for processing. It is + // currently a synchronous operation. If there is an error parsing the feed, + // parseFeed returns an empty feed in addition to calling aFeed.onParseError. + parseFeed: function (aFeed, aDOM) + { + if (!(aDOM instanceof Ci.nsIDOMXMLDocument)) + { + // No xml doc. + return aFeed.onParseError(aFeed); + } + + let doc = aDOM.documentElement; + if (doc.namespaceURI == FeedUtils.MOZ_PARSERERROR_NS) + { + // Gecko caught a basic parsing error. + let errStr = doc.firstChild.textContent + "\n" + + doc.firstElementChild.textContent; + FeedUtils.log.info("FeedParser.parseFeed: - " + errStr); + return aFeed.onParseError(aFeed); + } + else if (aDOM.querySelector("redirect")) + { + // Check for RSS2.0 redirect document. + let channel = aDOM.querySelector("redirect"); + if (this.isPermanentRedirect(aFeed, channel, null, null)) + return; + + return aFeed.onParseError(aFeed); + } + else if (doc.namespaceURI == FeedUtils.RDF_SYNTAX_NS && + doc.getElementsByTagNameNS(FeedUtils.RSS_NS, "channel")[0]) + { + aFeed.mFeedType = "RSS_1.xRDF" + FeedUtils.log.debug("FeedParser.parseFeed: type:url - " + + aFeed.mFeedType +" : " +aFeed.url); + // aSource can be misencoded (XMLHttpRequest converts to UTF-8 by default), + // but the DOM is almost always right because it uses the hints in the + // XML file. This is slower, but not noticably so. Mozilla doesn't have + // the XMLHttpRequest.responseBody property that IE has, which provides + // access to the unencoded response. + let xmlString = this.mSerializer.serializeToString(doc); + return this.parseAsRSS1(aFeed, xmlString, aFeed.request.channel.URI); + } + else if (doc.namespaceURI == FeedUtils.ATOM_03_NS) + { + aFeed.mFeedType = "ATOM_0.3" + FeedUtils.log.debug("FeedParser.parseFeed: type:url - " + + aFeed.mFeedType +" : " +aFeed.url); + return this.parseAsAtom(aFeed, aDOM); + } + else if (doc.namespaceURI == FeedUtils.ATOM_IETF_NS) + { + aFeed.mFeedType = "ATOM_IETF" + FeedUtils.log.debug("FeedParser.parseFeed: type:url - " + + aFeed.mFeedType +" : " +aFeed.url); + return this.parseAsAtomIETF(aFeed, aDOM); + } + else if (doc.getElementsByTagNameNS(FeedUtils.RSS_090_NS, "channel")[0]) + { + aFeed.mFeedType = "RSS_0.90" + FeedUtils.log.debug("FeedParser.parseFeed: type:url - " + + aFeed.mFeedType +" : " +aFeed.url); + return this.parseAsRSS2(aFeed, aDOM); + } + else + { + // Parse as RSS 0.9x. In theory even RSS 1.0 feeds could be parsed by + // the 0.9x parser if the RSS namespace were the default. + let rssVer = doc.localName == "rss" ? doc.getAttribute("version") : null; + if (rssVer) + aFeed.mFeedType = "RSS_" + rssVer; + else + aFeed.mFeedType = "RSS_0.9x?"; + FeedUtils.log.debug("FeedParser.parseFeed: type:url - " + + aFeed.mFeedType +" : " +aFeed.url); + return this.parseAsRSS2(aFeed, aDOM); + } + }, + + parseAsRSS2: function (aFeed, aDOM) + { + // Get the first channel (assuming there is only one per RSS File). + let parsedItems = new Array(); + + let channel = aDOM.querySelector("channel"); + if (!channel) + return aFeed.onParseError(aFeed); + + // Usually the empty string, unless this is RSS .90. + let nsURI = channel.namespaceURI || ""; + FeedUtils.log.debug("FeedParser.parseAsRSS2: channel nsURI - " + nsURI); + + if (this.isPermanentRedirect(aFeed, null, channel, null)) + return; + + let tags = this.childrenByTagNameNS(channel, nsURI, "title"); + aFeed.title = aFeed.title || this.getNodeValue(tags ? tags[0] : null); + tags = this.childrenByTagNameNS(channel, nsURI, "description"); + aFeed.description = this.getNodeValueFormatted(tags ? tags[0] : null); + tags = this.childrenByTagNameNS(channel, nsURI, "link"); + aFeed.link = this.validLink(this.getNodeValue(tags ? tags[0] : null)); + + if (!(aFeed.title || aFeed.description) || !aFeed.link) + { + FeedUtils.log.error("FeedParser.parseAsRSS2: missing mandatory element " + + "<title> and <description>, or <link>"); + return aFeed.onParseError(aFeed); + } + + if (!aFeed.parseItems) + return parsedItems; + + aFeed.invalidateItems(); + // XXX use getElementsByTagNameNS for now; childrenByTagNameNS would be + // better, but RSS .90 is still with us. + let itemNodes = aDOM.getElementsByTagNameNS(nsURI, "item"); + itemNodes = itemNodes ? itemNodes : []; + FeedUtils.log.debug("FeedParser.parseAsRSS2: items to parse - " + + itemNodes.length); + + for (let itemNode of itemNodes) + { + if (!itemNode.childElementCount) + continue; + let item = new FeedItem(); + item.feed = aFeed; + item.enclosures = []; + item.keywords = []; + + tags = this.childrenByTagNameNS(itemNode, FeedUtils.FEEDBURNER_NS, "origLink"); + let link = this.validLink(this.getNodeValue(tags ? tags[0] : null)); + if (!link) + { + tags = this.childrenByTagNameNS(itemNode, nsURI, "link"); + link = this.validLink(this.getNodeValue(tags ? tags[0] : null)); + } + tags = this.childrenByTagNameNS(itemNode, nsURI, "guid"); + let guidNode = tags ? tags[0] : null; + + let guid; + let isPermaLink = false; + if (guidNode) + { + guid = this.getNodeValue(guidNode); + // isPermaLink is true if the value is "true" or if the attribute is + // not present; all other values, including "false" and "False" and + // for that matter "TRuE" and "meatcake" are false. + if (!guidNode.hasAttribute("isPermaLink") || + guidNode.getAttribute("isPermaLink") == "true") + isPermaLink = true; + // If attribute isPermaLink is missing, it is good to check the validity + // of <guid> value as an URL to avoid linking to non-URL strings. + if (!guidNode.hasAttribute("isPermaLink")) + { + try + { + Services.io.newURI(guid, null, null); + if (Services.io.extractScheme(guid) == "tag") + isPermaLink = false; + } + catch (ex) + { + isPermaLink = false; + } + } + + item.id = guid; + } + + let guidLink = this.validLink(guid); + item.url = isPermaLink && guidLink ? guidLink : link ? link : null; + tags = this.childrenByTagNameNS(itemNode, nsURI, "description"); + item.description = this.getNodeValueFormatted(tags ? tags[0] : null); + tags = this.childrenByTagNameNS(itemNode, nsURI, "title"); + item.title = this.getNodeValue(tags ? tags[0] : null); + if (!(item.title || item.description)) + { + FeedUtils.log.info("FeedParser.parseAsRSS2: <item> missing mandatory " + + "element, either <title> or <description>; skipping"); + continue; + } + + if (!item.id) + { + // At this point, if there is no guid, uniqueness cannot be guaranteed + // by any of link or date (optional) or title (optional unless there + // is no description). Use a big chunk of description; minimize dupes + // with url and title if present. + item.id = (item.url || item.feed.url) + "#" + item.title + "#" + + (this.stripTags(item.description ? + item.description.substr(0, 150) : null) || + item.title); + item.id = item.id.replace(/[\n\r\t\s]+/g, " "); + } + + // Escape html entities in <title>, which are unescaped as textContent + // values. If the title is used as content, it will remain escaped; if + // it is used as the title, it will be unescaped upon store. Bug 1240603. + // The <description> tag must follow escaping examples found in + // http://www.rssboard.org/rss-encoding-examples, i.e. single escape angle + // brackets for tags, which are removed if used as title, and double + // escape entities for presentation in title. + // Better: always use <title>. Best: use Atom. + if (!item.title) + item.title = this.stripTags(item.description).substr(0, 150); + else + item.title = item.htmlEscape(item.title); + + tags = this.childrenByTagNameNS(itemNode, nsURI, "author"); + if (!tags) + tags = this.childrenByTagNameNS(itemNode, FeedUtils.DC_NS, "creator"); + item.author = this.getNodeValue(tags ? tags[0] : null) || + aFeed.title || + item.author; + + tags = this.childrenByTagNameNS(itemNode, nsURI, "pubDate"); + if (!tags || !this.getNodeValue(tags[0])) + tags = this.childrenByTagNameNS(itemNode, FeedUtils.DC_NS, "date"); + item.date = this.getNodeValue(tags ? tags[0] : null) || item.date; + + // If the date is invalid, users will see the beginning of the epoch + // unless we reset it here, so they'll see the current time instead. + // This is typical aggregator behavior. + if (item.date) + { + item.date = item.date.trim(); + if (!FeedUtils.isValidRFC822Date(item.date)) + { + // XXX Use this on the other formats as well. + item.date = this.dateRescue(item.date); + } + } + + tags = this.childrenByTagNameNS(itemNode, FeedUtils.RSS_CONTENT_NS, "encoded"); + item.content = this.getNodeValueFormatted(tags ? tags[0] : null); + + // Handle <enclosures> and <media:content>, which may be in a + // <media:group> (if present). + tags = this.childrenByTagNameNS(itemNode, nsURI, "enclosure"); + let encUrls = []; + if (tags) + for (let tag of tags) + { + let url = this.validLink(tag.getAttribute("url")); + if (url && encUrls.indexOf(url) == -1) + { + let type = this.removeUnprintableASCII(tag.getAttribute("type")); + let length = this.removeUnprintableASCII(tag.getAttribute("length")); + item.enclosures.push(new FeedEnclosure(url, type, length)); + encUrls.push(url); + } + } + + tags = itemNode.getElementsByTagNameNS(FeedUtils.MRSS_NS, "content"); + if (tags) + for (let tag of tags) + { + let url = this.validLink(tag.getAttribute("url")); + if (url && encUrls.indexOf(url) == -1) + { + let type = this.removeUnprintableASCII(tag.getAttribute("type")); + let fileSize = this.removeUnprintableASCII(tag.getAttribute("fileSize")); + item.enclosures.push(new FeedEnclosure(url, type, fileSize)); + } + } + + // The <origEnclosureLink> tag has no specification, especially regarding + // whether more than one tag is allowed and, if so, how tags would + // relate to previously declared (and well specified) enclosure urls. + // The common usage is to include 1 origEnclosureLink, in addition to + // the specified enclosure tags for 1 enclosure. Thus, we will replace the + // first enclosure's, if found, url with the first <origEnclosureLink> + // url only or else add the <origEnclosureLink> url. + tags = this.childrenByTagNameNS(itemNode, FeedUtils.FEEDBURNER_NS, "origEnclosureLink"); + let origEncUrl = this.validLink(this.getNodeValue(tags ? tags[0] : null)); + if (origEncUrl) + { + if (item.enclosures.length) + item.enclosures[0].mURL = origEncUrl; + else + item.enclosures.push(new FeedEnclosure(origEncUrl)); + } + + // Support <category> and autotagging. + tags = this.childrenByTagNameNS(itemNode, nsURI, "category"); + if (tags) + { + for (let tag of tags) + { + let term = this.getNodeValue(tag); + term = term ? this.xmlUnescape(term.replace(/,/g, ";")) : null; + if (term && item.keywords.indexOf(term) == -1) + item.keywords.push(term); + } + } + + parsedItems.push(item); + } + + return parsedItems; + }, + + parseAsRSS1 : function(aFeed, aSource, aBaseURI) + { + let parsedItems = new Array(); + + // RSS 1.0 is valid RDF, so use the RDF parser/service to extract data. + // Create a new RDF data source and parse the feed into it. + let ds = Cc["@mozilla.org/rdf/datasource;1?name=in-memory-datasource"]. + createInstance(Ci.nsIRDFDataSource); + + let rdfparser = Cc["@mozilla.org/rdf/xml-parser;1"]. + createInstance(Ci.nsIRDFXMLParser); + rdfparser.parseString(ds, aBaseURI, aSource); + + // Get information about the feed as a whole. + let channel = ds.GetSource(FeedUtils.RDF_TYPE, FeedUtils.RSS_CHANNEL, true); + if (!channel) + return aFeed.onParseError(aFeed); + + if (this.isPermanentRedirect(aFeed, null, channel, ds)) + return; + + aFeed.title = aFeed.title || + this.getRDFTargetValue(ds, channel, FeedUtils.RSS_TITLE) || + aFeed.url; + aFeed.description = this.getRDFTargetValueFormatted(ds, channel, FeedUtils.RSS_DESCRIPTION) || + ""; + aFeed.link = this.validLink(this.getRDFTargetValue(ds, channel, FeedUtils.RSS_LINK)) || + aFeed.url; + + if (!(aFeed.title || aFeed.description) || !aFeed.link) + { + FeedUtils.log.error("FeedParser.parseAsRSS1: missing mandatory element " + + "<title> and <description>, or <link>"); + return aFeed.onParseError(aFeed); + } + + if (!aFeed.parseItems) + return parsedItems; + + aFeed.invalidateItems(); + + // Ignore the <items> list and just get the <item>s. + let items = ds.GetSources(FeedUtils.RDF_TYPE, FeedUtils.RSS_ITEM, true); + + let index = 0; + while (items.hasMoreElements()) + { + let itemResource = items.getNext().QueryInterface(Ci.nsIRDFResource); + let item = new FeedItem(); + item.feed = aFeed; + + // Prefer the value of the link tag to the item URI since the URI could be + // a relative URN. + let uri = itemResource.ValueUTF8; + let link = this.validLink(this.getRDFTargetValue(ds, itemResource, FeedUtils.RSS_LINK)); + item.url = link || uri; + item.description = this.getRDFTargetValueFormatted(ds, itemResource, + FeedUtils.RSS_DESCRIPTION); + item.title = this.getRDFTargetValue(ds, itemResource, FeedUtils.RSS_TITLE) || + this.getRDFTargetValue(ds, itemResource, FeedUtils.DC_SUBJECT) || + (item.description ? + (this.stripTags(item.description).substr(0, 150)) : null); + if (!item.url || !item.title) + { + FeedUtils.log.info("FeedParser.parseAsRSS1: <item> missing mandatory " + + "element <item rdf:about> and <link>, or <title> and " + + "no <description>; skipping"); + continue; + } + + item.id = item.url; + item.url = this.validLink(item.url); + + item.author = this.getRDFTargetValue(ds, itemResource, FeedUtils.DC_CREATOR) || + this.getRDFTargetValue(ds, channel, FeedUtils.DC_CREATOR) || + aFeed.title || + item.author; + item.date = this.getRDFTargetValue(ds, itemResource, FeedUtils.DC_DATE) || + item.date; + item.content = this.getRDFTargetValueFormatted(ds, itemResource, + FeedUtils.RSS_CONTENT_ENCODED); + + parsedItems[index++] = item; + } + FeedUtils.log.debug("FeedParser.parseAsRSS1: items parsed - " + index); + + return parsedItems; + }, + + parseAsAtom: function(aFeed, aDOM) + { + let parsedItems = new Array(); + + // Get the first channel (assuming there is only one per Atom File). + let channel = aDOM.querySelector("feed"); + if (!channel) + return aFeed.onParseError(aFeed); + + if (this.isPermanentRedirect(aFeed, null, channel, null)) + return; + + let tags = this.childrenByTagNameNS(channel, FeedUtils.ATOM_03_NS, "title"); + aFeed.title = aFeed.title || + this.stripTags(this.getNodeValue(tags ? tags[0] : null)); + tags = this.childrenByTagNameNS(channel, FeedUtils.ATOM_03_NS, "tagline"); + aFeed.description = this.getNodeValueFormatted(tags ? tags[0] : null); + tags = this.childrenByTagNameNS(channel, FeedUtils.ATOM_03_NS, "link"); + aFeed.link = this.validLink(this.findAtomLink("alternate", tags)); + + if (!aFeed.title) + { + FeedUtils.log.error("FeedParser.parseAsAtom: missing mandatory element " + + "<title>"); + return aFeed.onParseError(aFeed); + } + + if (!aFeed.parseItems) + return parsedItems; + + aFeed.invalidateItems(); + let items = this.childrenByTagNameNS(channel, FeedUtils.ATOM_03_NS, "entry"); + items = items ? items : []; + FeedUtils.log.debug("FeedParser.parseAsAtom: items to parse - " + + items.length); + + for (let itemNode of items) + { + if (!itemNode.childElementCount) + continue; + let item = new FeedItem(); + item.feed = aFeed; + + tags = this.childrenByTagNameNS(itemNode, FeedUtils.ATOM_03_NS, "link"); + item.url = this.validLink(this.findAtomLink("alternate", tags)); + + tags = this.childrenByTagNameNS(itemNode, FeedUtils.ATOM_03_NS, "id"); + item.id = this.getNodeValue(tags ? tags[0] : null); + tags = this.childrenByTagNameNS(itemNode, FeedUtils.ATOM_03_NS, "summary"); + item.description = this.getNodeValueFormatted(tags ? tags[0] : null); + tags = this.childrenByTagNameNS(itemNode, FeedUtils.ATOM_03_NS, "title"); + item.title = this.getNodeValue(tags ? tags[0] : null) || + (item.description ? item.description.substr(0, 150) : null); + if (!item.title || !item.id) + { + // We're lenient about other mandatory tags, but insist on these. + FeedUtils.log.info("FeedParser.parseAsAtom: <entry> missing mandatory " + + "element <id>, or <title> and no <summary>; skipping"); + continue; + } + + tags = this.childrenByTagNameNS(itemNode, FeedUtils.ATOM_03_NS, "author"); + if (!tags) + tags = this.childrenByTagNameNS(itemNode, FeedUtils.ATOM_03_NS, "contributor"); + if (!tags) + tags = this.childrenByTagNameNS(channel, FeedUtils.ATOM_03_NS, "author"); + + let authorEl = tags ? tags[0] : null; + + let author = ""; + if (authorEl) + { + tags = this.childrenByTagNameNS(authorEl, FeedUtils.ATOM_03_NS, "name"); + let name = this.getNodeValue(tags ? tags[0] : null); + tags = this.childrenByTagNameNS(authorEl, FeedUtils.ATOM_03_NS, "email"); + let email = this.getNodeValue(tags ? tags[0] : null); + if (name) + author = name + (email ? " <" + email + ">" : ""); + else if (email) + author = email; + } + + item.author = author || item.author || aFeed.title; + + tags = this.childrenByTagNameNS(itemNode, FeedUtils.ATOM_03_NS, "modified"); + if (!tags || !this.getNodeValue(tags[0])) + tags = this.childrenByTagNameNS(itemNode, FeedUtils.ATOM_03_NS, "issued"); + if (!tags || !this.getNodeValue(tags[0])) + tags = this.childrenByTagNameNS(channel, FeedUtils.ATOM_03_NS, "created"); + + item.date = this.getNodeValue(tags ? tags[0] : null) || item.date; + + // XXX We should get the xml:base attribute from the content tag as well + // and use it as the base HREF of the message. + // XXX Atom feeds can have multiple content elements; we should differentiate + // between them and pick the best one. + // Some Atom feeds wrap the content in a CTYPE declaration; others use + // a namespace to identify the tags as HTML; and a few are buggy and put + // HTML tags in without declaring their namespace so they look like Atom. + // We deal with the first two but not the third. + tags = this.childrenByTagNameNS(itemNode, FeedUtils.ATOM_03_NS, "content"); + let contentNode = tags ? tags[0] : null; + + let content; + if (contentNode) + { + content = ""; + for (let j = 0; j < contentNode.childNodes.length; j++) + { + let node = contentNode.childNodes.item(j); + if (node.nodeType == node.CDATA_SECTION_NODE) + content += node.data; + else + content += this.mSerializer.serializeToString(node); + } + + if (contentNode.getAttribute("mode") == "escaped") + { + content = content.replace(/</g, "<"); + content = content.replace(/>/g, ">"); + content = content.replace(/&/g, "&"); + } + + if (content == "") + content = null; + } + + item.content = content; + parsedItems.push(item); + } + + return parsedItems; + }, + + parseAsAtomIETF: function(aFeed, aDOM) + { + let parsedItems = new Array(); + + // Get the first channel (assuming there is only one per Atom File). + let channel = this.childrenByTagNameNS(aDOM, FeedUtils.ATOM_IETF_NS, "feed")[0]; + if (!channel) + return aFeed.onParseError(aFeed); + + if (this.isPermanentRedirect(aFeed, null, channel, null)) + return; + + let tags = this.childrenByTagNameNS(channel, FeedUtils.ATOM_IETF_NS, "title"); + aFeed.title = aFeed.title || + this.stripTags(this.serializeTextConstruct(tags ? tags[0] : null)); + + tags = this.childrenByTagNameNS(channel, FeedUtils.ATOM_IETF_NS, "subtitle"); + aFeed.description = this.serializeTextConstruct(tags ? tags[0] : null); + + tags = this.childrenByTagNameNS(channel, FeedUtils.ATOM_IETF_NS, "link"); + aFeed.link = this.findAtomLink("alternate", tags); + aFeed.link = this.validLink(aFeed.link); + + if (!aFeed.title) + { + FeedUtils.log.error("FeedParser.parseAsAtomIETF: missing mandatory element " + + "<title>"); + return aFeed.onParseError(aFeed); + } + + if (!aFeed.parseItems) + return parsedItems; + + aFeed.invalidateItems(); + let items = this.childrenByTagNameNS(channel, FeedUtils.ATOM_IETF_NS, "entry"); + items = items ? items : []; + FeedUtils.log.debug("FeedParser.parseAsAtomIETF: items to parse - " + + items.length); + + for (let itemNode of items) + { + if (!itemNode.childElementCount) + continue; + let item = new FeedItem(); + item.feed = aFeed; + item.enclosures = []; + item.keywords = []; + + tags = this.childrenByTagNameNS(itemNode, FeedUtils.FEEDBURNER_NS, "origLink"); + item.url = this.validLink(this.getNodeValue(tags ? tags[0] : null)); + if (!item.url) + { + tags = this.childrenByTagNameNS(itemNode, FeedUtils.ATOM_IETF_NS, "link"); + item.url = this.validLink(this.findAtomLink("alternate", tags)) || + aFeed.link; + } + tags = this.childrenByTagNameNS(itemNode, FeedUtils.ATOM_IETF_NS, "id"); + item.id = this.getNodeValue(tags ? tags[0] : null); + tags = this.childrenByTagNameNS(itemNode, FeedUtils.ATOM_IETF_NS, "summary"); + item.description = this.serializeTextConstruct(tags ? tags[0] : null); + tags = this.childrenByTagNameNS(itemNode, FeedUtils.ATOM_IETF_NS, "title"); + item.title = this.stripTags(this.serializeTextConstruct(tags ? tags[0] : null) || + (item.description ? + item.description.substr(0, 150) : null)); + if (!item.title || !item.id) + { + // We're lenient about other mandatory tags, but insist on these. + FeedUtils.log.info("FeedParser.parseAsAtomIETF: <entry> missing mandatory " + + "element <id>, or <title> and no <summary>; skipping"); + continue; + } + + // XXX Support multiple authors. + tags = this.childrenByTagNameNS(itemNode, FeedUtils.ATOM_IETF_NS, "source"); + let source = tags ? tags[0] : null; + + tags = this.childrenByTagNameNS(itemNode, FeedUtils.ATOM_IETF_NS, "author"); + if (!tags) + tags = this.childrenByTagNameNS(source, FeedUtils.ATOM_IETF_NS, "author"); + if (!tags) + tags = this.childrenByTagNameNS(channel, FeedUtils.ATOM_IETF_NS, "author"); + + let authorEl = tags ? tags[0] : null; + + let author = ""; + if (authorEl) + { + tags = this.childrenByTagNameNS(authorEl, FeedUtils.ATOM_IETF_NS, "name"); + let name = this.getNodeValue(tags ? tags[0] : null); + tags = this.childrenByTagNameNS(authorEl, FeedUtils.ATOM_IETF_NS, "email"); + let email = this.getNodeValue(tags ? tags[0] : null); + if (name) + author = name + (email ? " <" + email + ">" : ""); + else if (email) + author = email; + } + + item.author = author || item.author || aFeed.title; + + tags = this.childrenByTagNameNS(itemNode, FeedUtils.ATOM_IETF_NS, "updated"); + if (!tags || !this.getNodeValue(tags[0])) + tags = this.childrenByTagNameNS(itemNode, FeedUtils.ATOM_IETF_NS, "published"); + if (!tags || !this.getNodeValue(tags[0])) + tags = this.childrenByTagNameNS(source, FeedUtils.ATOM_IETF_NS, "published"); + item.date = this.getNodeValue(tags ? tags[0] : null) || item.date; + + tags = this.childrenByTagNameNS(itemNode, FeedUtils.ATOM_IETF_NS, "content"); + item.content = this.serializeTextConstruct(tags ? tags[0] : null); + + if (item.content) + item.xmlContentBase = tags ? tags[0].baseURI : null; + else if (item.description) + { + tags = this.childrenByTagNameNS(itemNode, FeedUtils.ATOM_IETF_NS, "summary"); + item.xmlContentBase = tags ? tags[0].baseURI : null; + } + else + item.xmlContentBase = itemNode.baseURI; + + item.xmlContentBase = this.validLink(item.xmlContentBase); + + // Handle <link rel="enclosure"> (if present). + tags = this.childrenByTagNameNS(itemNode, FeedUtils.ATOM_IETF_NS, "link"); + let encUrls = []; + if (tags) + for (let tag of tags) + { + let url = tag.getAttribute("rel") == "enclosure" ? + (tag.getAttribute("href") || "").trim() : null; + url = this.validLink(url); + if (url && encUrls.indexOf(url) == -1) + { + let type = this.removeUnprintableASCII(tag.getAttribute("type")); + let length = this.removeUnprintableASCII(tag.getAttribute("length")); + let title = this.removeUnprintableASCII(tag.getAttribute("title")); + item.enclosures.push(new FeedEnclosure(url, type, length, title)); + encUrls.push(url); + } + } + + tags = this.childrenByTagNameNS(itemNode, FeedUtils.FEEDBURNER_NS, "origEnclosureLink"); + let origEncUrl = this.validLink(this.getNodeValue(tags ? tags[0] : null)); + if (origEncUrl) + { + if (item.enclosures.length) + item.enclosures[0].mURL = origEncUrl; + else + item.enclosures.push(new FeedEnclosure(origEncUrl)); + } + + // Handle atom threading extension, RFC4685. There may be 1 or more tags, + // and each must contain a ref attribute with 1 Message-Id equivalent + // value. This is the only attr of interest in the spec for presentation. + tags = this.childrenByTagNameNS(itemNode, FeedUtils.ATOM_THREAD_NS, "in-reply-to"); + if (tags) + { + for (let tag of tags) + { + let ref = this.removeUnprintableASCII(tag.getAttribute("ref")); + if (ref) + item.inReplyTo += item.normalizeMessageID(ref) + " "; + } + item.inReplyTo = item.inReplyTo.trimRight(); + } + + // Support <category> and autotagging. + tags = this.childrenByTagNameNS(itemNode, FeedUtils.ATOM_IETF_NS, "category"); + if (tags) + { + for (let tag of tags) + { + let term = this.removeUnprintableASCII(tag.getAttribute("term")); + term = term ? this.xmlUnescape(term.replace(/,/g, ";")).trim() : null; + if (term && item.keywords.indexOf(term) == -1) + item.keywords.push(term); + } + } + + parsedItems.push(item); + } + + return parsedItems; + }, + + isPermanentRedirect: function(aFeed, aRedirDocChannel, aFeedChannel, aDS) + { + // If subscribing to a new feed, do not check redirect tags. + if (!aFeed.downloadCallback || aFeed.downloadCallback.mSubscribeMode) + return false; + + let tags, tagName, newUrl; + let oldUrl = aFeed.url; + + // Check for RSS2.0 redirect document <newLocation> tag. + if (aRedirDocChannel) + { + tagName = "newLocation"; + tags = this.childrenByTagNameNS(aRedirDocChannel, "", tagName); + newUrl = this.getNodeValue(tags ? tags[0] : null); + } + + // Check for <itunes:new-feed-url> tag. + if (aFeedChannel) + { + tagName = "new-feed-url"; + if (aDS) + { + tags = FeedUtils.rdf.GetResource(FeedUtils.ITUNES_NS + tagName); + newUrl = this.getRDFTargetValue(aDS, aFeedChannel, tags); + } + else + { + tags = this.childrenByTagNameNS(aFeedChannel, FeedUtils.ITUNES_NS, tagName); + newUrl = this.getNodeValue(tags ? tags[0] : null); + } + tagName = "itunes:" + tagName; + } + + if (newUrl && newUrl != oldUrl && FeedUtils.isValidScheme(newUrl) && + FeedUtils.changeUrlForFeed(aFeed, newUrl)) + { + FeedUtils.log.info("FeedParser.isPermanentRedirect: found <" + tagName + + "> tag; updated feed url from: " + oldUrl + " to: " + newUrl + + " in folder: " + FeedUtils.getFolderPrettyPath(aFeed.folder)); + aFeed.onUrlChange(aFeed, oldUrl); + return true; + } + + return false; + }, + + serializeTextConstruct: function(textElement) + { + let content = ""; + if (textElement) + { + let textType = textElement.getAttribute("type"); + + // Atom spec says consider it "text" if not present. + if (!textType) + textType = "text"; + + // There could be some strange content type we don't handle. + if (textType != "text" && textType != "html" && textType != "xhtml") + return null; + + for (let j = 0; j < textElement.childNodes.length; j++) + { + let node = textElement.childNodes.item(j); + if (node.nodeType == node.CDATA_SECTION_NODE) + content += this.xmlEscape(node.data); + else + content += this.mSerializer.serializeToString(node); + } + + if (textType == "html") + content = this.xmlUnescape(content); + + content = content.trim(); + } + + // Other parts of the code depend on this being null if there's no content. + return content ? content : null; + }, + + getRDFTargetValue: function(ds, source, property) + { + let nodeValue = this.getRDFTargetValueRaw(ds, source, property); + if (!nodeValue) + return null; + + nodeValue = nodeValue.replace(/[\n\r\t]+/g, " "); + return this.removeUnprintableASCII(nodeValue); + + }, + + getRDFTargetValueFormatted: function(ds, source, property) + { + let nodeValue = this.getRDFTargetValueRaw(ds, source, property); + if (!nodeValue) + return null; + + return this.removeUnprintableASCIIexCRLFTAB(nodeValue); + + }, + + getRDFTargetValueRaw: function(ds, source, property) + { + let node = ds.GetTarget(source, property, true); + if (node) + { + try + { + node = node.QueryInterface(Ci.nsIRDFLiteral); + if (node) + return node.Value.trim(); + } + catch (e) + { + // If the RDF was bogus, do nothing. Rethrow if it's some other problem. + if (!((e instanceof Ci.nsIXPCException) && + e.result == Cr.NS_ERROR_NO_INTERFACE)) + throw new Error("FeedParser.getRDFTargetValue: " + e); + } + } + + return null; + }, + + /** + * Return a cleaned up node value. This is intended for values that are not + * multiline and not formatted. A sequence of tab or newline is converted to + * a space and unprintable ascii is removed. + * + * @param {Node} node - A DOM node. + * @return {String} - A clean string value or null. + */ + getNodeValue: function(node) + { + let nodeValue = this.getNodeValueRaw(node); + if (!nodeValue) + return null; + + nodeValue = nodeValue.replace(/[\n\r\t]+/g, " "); + return this.removeUnprintableASCII(nodeValue); + }, + + /** + * Return a cleaned up formatted node value, meaning CR/LF/TAB are retained + * while all other unprintable ascii is removed. This is intended for values + * that are multiline and formatted, such as content or description tags. + * + * @param {Node} node - A DOM node. + * @return {String} - A clean string value or null. + */ + getNodeValueFormatted: function(node) + { + let nodeValue = this.getNodeValueRaw(node); + if (!nodeValue) + return null; + + return this.removeUnprintableASCIIexCRLFTAB(nodeValue); + }, + + /** + * Return a raw node value, as received. This should be sanitized as + * appropriate. + * + * @param {Node} node - A DOM node. + * @return {String} - A string value or null. + */ + getNodeValueRaw: function(node) + { + if (node && node.textContent) + return node.textContent.trim(); + + if (node && node.firstChild) + { + let ret = ""; + for (let child = node.firstChild; child; child = child.nextSibling) + { + let value = this.getNodeValueRaw(child); + if (value) + ret += value; + } + + if (ret) + return ret.trim(); + } + + return null; + }, + + // Finds elements that are direct children of the first arg. + childrenByTagNameNS: function(aElement, aNamespace, aTagName) + { + if (!aElement) + return null; + + let matches = aElement.getElementsByTagNameNS(aNamespace, aTagName); + let matchingChildren = new Array(); + for (let match of matches) + { + if (match.parentNode == aElement) + matchingChildren.push(match) + } + + return matchingChildren.length ? matchingChildren : null; + }, + + /** + * Ensure <link> type tags start with http[s]://, ftp:// or magnet: + * for values stored in mail headers (content-base and remote enclosures), + * particularly to prevent data: uris, javascript, and other spoofing. + * + * @param {String} link - An intended http url string. + * @return {String} - A clean string starting with http, ftp or magnet, + * else null. + */ + validLink: function(link) + { + if (/^((https?|ftp):\/\/|magnet:)/.test(link)) + return this.removeUnprintableASCII(link.trim()); + + return null; + }, + + findAtomLink: function(linkRel, linkElements) + { + if (!linkElements) + return null; + + // XXX Need to check for MIME type and hreflang. + for (let alink of linkElements) { + if (alink && + // If there's a link rel. + ((alink.getAttribute("rel") && alink.getAttribute("rel") == linkRel) || + // If there isn't, assume 'alternate'. + (!alink.getAttribute("rel") && (linkRel == "alternate"))) && + alink.getAttribute("href")) + { + // Atom links are interpreted relative to xml:base. + try { + return Services.io.newURI(alink.baseURI, null, null). + resolve(alink.getAttribute("href")); + } + catch (ex) {} + } + } + + return null; + }, + + /** + * Remove unprintable ascii, particularly CR/LF, for non formatted tag values. + * + * @param {String} s - String to clean. + * @return {String} + */ + removeUnprintableASCII: function(s) + { + return s ? s.replace(/[\x00-\x1F\x7F]+/g, "") : ""; + }, + + /** + * Remove unprintable ascii, except CR/LF/TAB, for formatted tag values. + * + * @param {String} s - String to clean. + * @return {String} + */ + removeUnprintableASCIIexCRLFTAB: function(s) + { + return s ? s.replace(/[\x00-\x08\x0B-\x0C\x0E-\x1F\x7F]+/g, "") : ""; + }, + + stripTags: function(someHTML) + { + return someHTML ? someHTML.replace(/<[^>]+>/g, "") : someHTML; + }, + + xmlUnescape: function(s) + { + s = s.replace(/</g, "<"); + s = s.replace(/>/g, ">"); + s = s.replace(/&/g, "&"); + return s; + }, + + xmlEscape: function(s) + { + s = s.replace(/&/g, "&"); + s = s.replace(/>/g, ">"); + s = s.replace(/</g, "<"); + return s; + }, + + dateRescue: function(dateString) + { + // Deal with various kinds of invalid dates. + if (!isNaN(parseInt(dateString))) + { + // It's an integer, so maybe it's a timestamp. + let d = new Date(parseInt(dateString) * 1000); + let now = new Date(); + let yeardiff = now.getFullYear() - d.getFullYear(); + FeedUtils.log.trace("FeedParser.dateRescue: Rescue Timestamp date - " + + d.toString() + " ,year diff - " + yeardiff); + if (yeardiff >= 0 && yeardiff < 3) + // It's quite likely the correct date. + return d.toString(); + } + + // Could be an ISO8601/W3C date. If not, get the current time. + return FeedUtils.getValidRFC5322Date(dateString); + } +}; diff --git a/mailnews/extensions/newsblog/content/feed-subscriptions.js b/mailnews/extensions/newsblog/content/feed-subscriptions.js new file mode 100644 index 000000000..2b77e60c4 --- /dev/null +++ b/mailnews/extensions/newsblog/content/feed-subscriptions.js @@ -0,0 +1,2703 @@ +/* -*- Mode: JavaScript; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +Components.utils.import("resource:///modules/FeedUtils.jsm"); +Components.utils.import("resource:///modules/gloda/log4moz.js"); +Components.utils.import("resource:///modules/mailServices.js"); +Components.utils.import("resource://gre/modules/FileUtils.jsm"); +Components.utils.import("resource://gre/modules/PluralForm.jsm"); +Components.utils.import("resource://gre/modules/Services.jsm"); + +var {classes: Cc, interfaces: Ci} = Components; + +var FeedSubscriptions = { + get mMainWin() { return Services.wm.getMostRecentWindow("mail:3pane"); }, + + get mTree() { return document.getElementById("rssSubscriptionsList"); }, + + mFeedContainers: [], + mRSSServer : null, + mActionMode : null, + kSubscribeMode : 1, + kUpdateMode : 2, + kMoveMode : 3, + kCopyMode : 4, + kImportingOPML : 5, + kVerifyUrlMode : 6, + + get FOLDER_ACTIONS() + { + return Ci.nsIMsgFolderNotificationService.folderAdded | + Ci.nsIMsgFolderNotificationService.folderDeleted | + Ci.nsIMsgFolderNotificationService.folderRenamed | + Ci.nsIMsgFolderNotificationService.folderMoveCopyCompleted; + }, + + onLoad: function () + { + // Extract the folder argument. + let folder; + if (window.arguments && window.arguments[0].folder) + folder = window.arguments[0].folder; + + // Ensure dialog is fully loaded before selecting, to get visible row. + setTimeout(function() { + FeedSubscriptions.refreshSubscriptionView(folder) + }, 100); + let message = FeedUtils.strings.GetStringFromName("subscribe-loading"); + this.updateStatusItem("statusText", message); + + FeedUtils.CANCEL_REQUESTED = false; + + if (this.mMainWin) + { + this.mMainWin.FeedFolderNotificationService = MailServices.mfn; + this.mMainWin.FeedFolderNotificationService + .addListener(this.FolderListener, this.FOLDER_ACTIONS); + } + }, + + onClose: function () + { + let dismissDialog = true; + + // If we are in the middle of subscribing to a feed, inform the user that + // dismissing the dialog right now will abort the feed subscription. + if (this.mActionMode == this.kSubscribeMode) + { + let pTitle = FeedUtils.strings.GetStringFromName( + "subscribe-cancelSubscriptionTitle"); + let pMessage = FeedUtils.strings.GetStringFromName( + "subscribe-cancelSubscription"); + dismissDialog = + !(Services.prompt.confirmEx(window, pTitle, pMessage, + Ci.nsIPromptService.STD_YES_NO_BUTTONS, + null, null, null, null, { })); + } + + if (dismissDialog) + { + FeedUtils.CANCEL_REQUESTED = this.mActionMode == this.kSubscribeMode; + if (this.mMainWin) + { + this.mMainWin.FeedFolderNotificationService + .removeListener(this.FolderListener, this.FOLDER_ACTIONS); + delete this.mMainWin.FeedFolderNotificationService; + } + } + + return dismissDialog; + }, + + refreshSubscriptionView: function(aSelectFolder, aSelectFeedUrl) + { + let item = this.mView.currentItem; + this.loadSubscriptions(); + this.mTree.view = this.mView; + + if (aSelectFolder && !aSelectFeedUrl) + this.selectFolder(aSelectFolder); + else + { + // If no folder to select, try to select the pre rebuild selection, in + // an existing window. For folderpane changes in a feed account. + if (item) + { + let rootFolder = item.container ? item.folder.rootFolder : + item.parentFolder.rootFolder; + if (item.container) + { + if (!this.selectFolder(item.folder, { open: item.open })) + // The item no longer exists, an ancestor folder was deleted or + // renamed/moved. + this.selectFolder(rootFolder); + } + else { + let url = item.parentFolder == aSelectFolder ? aSelectFeedUrl : + item.url; + this.selectFeed({ folder: rootFolder, url: url }, null); + } + } + } + + this.mView.treeBox.ensureRowIsVisible(this.mView.selection.currentIndex); + this.clearStatusInfo(); + }, + + mView: + { + kRowIndexUndefined: -1, + + get currentItem() { + // Get the current selection, if any. + let seln = this.selection; + let currentSelectionIndex = seln ? seln.currentIndex : null; + let item; + if (currentSelectionIndex != null) + item = this.getItemAtIndex(currentSelectionIndex); + + return item; + }, + + /* nsITreeView */ + treeBox: null, + + mRowCount: 0, + get rowCount() { return this.mRowCount; }, + + _selection: null, + get selection () { return this._selection; }, + set selection (val) { return this._selection = val; }, + + setTree: function(aTreebox) { this.treeBox = aTreebox; }, + isSeparator: function(aRow) { return false; }, + isSorted: function() { return false; }, + isSelectable: function(aRow, aColumn) { return false; }, + isEditable: function (aRow, aColumn) { return false; }, + + getProgressMode : function(aRow, aCol) {}, + cycleHeader: function(aCol) {}, + cycleCell: function(aRow, aCol) {}, + selectionChanged: function() {}, + performAction: function(aAction) {}, + performActionOnRow: function (aAction, aRow) {}, + performActionOnCell: function(aAction, aRow, aCol) {}, + getRowProperties: function(aRow) { return ""; }, + getColumnProperties: function(aCol) { return ""; }, + getCellValue: function (aRow, aColumn) {}, + setCellValue: function (aRow, aColumn, aValue) {}, + setCellText: function (aRow, aColumn, aValue) {}, + + getCellProperties: function (aRow, aColumn) { + let item = this.getItemAtIndex(aRow); + let folder = item && item.folder ? item.folder : null; +#ifdef MOZ_THUNDERBIRD + let properties = ["folderNameCol"]; + let hasFeeds = folder ? FeedUtils.getFeedUrlsInFolder(folder) : false; + let prop = !folder ? "isFeed-true" : + hasFeeds ? "isFeedFolder-true" : + folder.isServer ? "serverType-rss isServer-true" : null; + if (prop) + properties.push(prop); + return properties.join(" "); +#else + return !folder ? "serverType-rss" : + folder.isServer ? "serverType-rss isServer-true" : "livemark"; +#endif + }, + + isContainer: function (aRow) + { + let item = this.getItemAtIndex(aRow); + return item ? item.container : false; + }, + + isContainerOpen: function (aRow) + { + let item = this.getItemAtIndex(aRow); + return item ? item.open : false; + }, + + isContainerEmpty: function (aRow) + { + let item = this.getItemAtIndex(aRow); + if (!item) + return false; + + return item.children.length == 0; + }, + + getItemAtIndex: function (aRow) + { + if (aRow < 0 || aRow >= FeedSubscriptions.mFeedContainers.length) + return null; + + return FeedSubscriptions.mFeedContainers[aRow]; + }, + + getItemInViewIndex: function(aFolder) + { + if (!aFolder || !(aFolder instanceof Ci.nsIMsgFolder)) + return null; + + for (let index = 0; index < this.rowCount; index++) + { + // Find the visible folder in the view. + let item = this.getItemAtIndex(index); + if (item && item.container && item.url == aFolder.URI) + return index; + } + + return null; + }, + + removeItemAtIndex: function (aRow, aNoSelect) + { + let itemToRemove = this.getItemAtIndex(aRow); + if (!itemToRemove) + return; + + if (itemToRemove.container && itemToRemove.open) + // Close it, if open container. + this.toggleOpenState(aRow); + + let parentIndex = this.getParentIndex(aRow); + let hasNextSibling = this.hasNextSibling(aRow, aRow); + if (parentIndex != this.kRowIndexUndefined) + { + let parent = this.getItemAtIndex(parentIndex); + if (parent) + { + for (let index = 0; index < parent.children.length; index++) + if (parent.children[index] == itemToRemove) + { + parent.children.splice(index, 1); + break; + } + } + } + + // Now remove it from our view. + FeedSubscriptions.mFeedContainers.splice(aRow, 1); + + // Now invalidate the correct tree rows. + this.mRowCount--; + this.treeBox.rowCountChanged(aRow, -1); + + // Now update the selection position, unless noSelect (selection is + // done later or not at all). If the item is the last child, select the + // parent. Otherwise select the next sibling. + if (!aNoSelect) { + if (aRow <= FeedSubscriptions.mFeedContainers.length) + this.selection.select(hasNextSibling ? aRow : aRow - 1); + else + this.selection.clearSelection(); + } + + // Now refocus the tree. + FeedSubscriptions.mTree.focus(); + }, + + getCellText: function (aRow, aColumn) + { + let item = this.getItemAtIndex(aRow); + return (item && aColumn.id == "folderNameCol") ? item.name : ""; + }, + + getImageSrc: function(aRow, aCol) + { + let item = this.getItemAtIndex(aRow); + if ((item.folder && item.folder.isServer) || item.open) + return ""; + + if (item.favicon != null) + return item.favicon; + + if (item.folder && FeedSubscriptions.mMainWin && + "gFolderTreeView" in FeedSubscriptions.mMainWin) { + let favicon = FeedSubscriptions.mMainWin.gFolderTreeView + .getFolderCacheProperty(item.folder, "favicon"); + if (favicon != null) + return item.favicon = favicon; + } + + let callback = (iconUrl => { + item.favicon = iconUrl; + if (item.folder) + { + for (let child of item.children) + if (!child.container) + { + child.favicon = iconUrl; + break; + } + } + + this.selection.tree.invalidateRow(aRow); + }); + + // A closed non server folder. + if (item.folder) + { + for (let child of item.children) + { + if (!child.container) { + if (child.favicon != null) + return child.favicon; + + setTimeout(() => { + FeedUtils.getFavicon(child.parentFolder, child.url, null, + window, callback); + }, 0); + break; + } + } + } + else + { + // A feed. + setTimeout(() => { + FeedUtils.getFavicon(item.parentFolder, item.url, null, + window, callback); + }, 0); + } + + // Store empty string to return default while favicons are retrieved. + return item.favicon = ""; + }, + + canDrop: function (aRow, aOrientation) + { + let dropResult = this.extractDragData(aRow); + return aOrientation == Ci.nsITreeView.DROP_ON && dropResult.canDrop && + (dropResult.dropUrl || dropResult.dropOnIndex != this.kRowIndexUndefined); + }, + + drop: function (aRow, aOrientation) + { + let win = FeedSubscriptions; + let results = this.extractDragData(aRow); + if (!results.canDrop) + return; + + // Preselect the drop folder. + this.selection.select(aRow); + + if (results.dropUrl) + { + // Don't freeze the app that initiated the drop just because we are + // in a loop waiting for the user to dimisss the add feed dialog. + setTimeout(function() { + win.addFeed(results.dropUrl, null, true, null, win.kSubscribeMode); + }, 0); + let folderItem = this.getItemAtIndex(aRow); + FeedUtils.log.debug("drop: folder, url - " + + folderItem.folder.name + ", " + results.dropUrl); + } + else if (results.dropOnIndex != this.kRowIndexUndefined) + { + win.moveCopyFeed(results.dropOnIndex, aRow, results.dropEffect); + } + }, + + // Helper function for drag and drop. + extractDragData: function(aRow) + { + let dt = this._currentDataTransfer; + let dragDataResults = { canDrop: false, + dropUrl: null, + dropOnIndex: this.kRowIndexUndefined, + dropEffect: dt.dropEffect }; + + if (dt.getData("text/x-moz-feed-index")) + { + // Dragging a feed in the tree. + if (this.selection) + { + dragDataResults.dropOnIndex = this.selection.currentIndex; + + let curItem = this.getItemAtIndex(this.selection.currentIndex); + let newItem = this.getItemAtIndex(aRow); + let curServer = curItem && curItem.parentFolder ? + curItem.parentFolder.server : null; + let newServer = newItem && newItem.folder ? + newItem.folder.server : null; + + // No copying within the same account and no moving to the account + // folder in the same account. + if (!(curServer == newServer && + (dragDataResults.dropEffect == "copy" || + newItem.folder == curItem.parentFolder || + newItem.folder.isServer))) + dragDataResults.canDrop = true; + } + } + else + { + // Try to get a feed url. + let validUri = FeedUtils.getFeedUriFromDataTransfer(dt); + + if (validUri) + { + dragDataResults.canDrop = true; + dragDataResults.dropUrl = validUri.spec; + } + } + + return dragDataResults; + }, + + getParentIndex: function (aRow) + { + let item = this.getItemAtIndex(aRow); + + if (item) + { + for (let index = aRow; index >= 0; index--) + if (FeedSubscriptions.mFeedContainers[index].level < item.level) + return index; + } + + return this.kRowIndexUndefined; + }, + + isIndexChildOfParentIndex: function (aRow, aChildRow) + { + // For visible tree rows, not if items are children of closed folders. + let item = this.getItemAtIndex(aRow); + if (!item || aChildRow <= aRow) + return false; + + let targetLevel = this.getItemAtIndex(aRow).level; + let rows = FeedSubscriptions.mFeedContainers; + + for (let i = aRow + 1; i < rows.length; i++) { + if (this.getItemAtIndex(i).level <= targetLevel) + break; + if (aChildRow == i) + return true; + } + + return false; + }, + + hasNextSibling: function(aRow, aAfterIndex) { + let targetLevel = this.getItemAtIndex(aRow).level; + let rows = FeedSubscriptions.mFeedContainers; + for (let i = aAfterIndex + 1; i < rows.length; i++) { + if (this.getItemAtIndex(i).level == targetLevel) + return true; + if (this.getItemAtIndex(i).level < targetLevel) + return false; + } + + return false; + }, + + hasPreviousSibling: function (aRow) + { + let item = this.getItemAtIndex(aRow); + if (item && aRow) + return this.getItemAtIndex(aRow - 1).level == item.level; + else + return false; + }, + + getLevel: function (aRow) + { + let item = this.getItemAtIndex(aRow); + if (!item) + return 0; + + return item.level; + }, + + toggleOpenState: function (aRow) + { + let item = this.getItemAtIndex(aRow); + if (!item) + return; + + // Save off the current selection item. + let seln = this.selection; + let currentSelectionIndex = seln.currentIndex; + + let rowsChanged = this.toggle(aRow) + + // Now restore selection, ensuring selection is maintained on toggles. + if (currentSelectionIndex > aRow) + seln.currentIndex = currentSelectionIndex + rowsChanged; + else + seln.select(currentSelectionIndex); + + seln.selectEventsSuppressed = false; + }, + + toggle: function (aRow) + { + // Collapse the row, or build sub rows based on open states in the map. + let item = this.getItemAtIndex(aRow); + if (!item) + return null; + + let rows = FeedSubscriptions.mFeedContainers; + let rowCount = 0; + let multiplier; + + function addDescendants(aItem) + { + for (let i = 0; i < aItem.children.length; i++) + { + rowCount++; + let child = aItem.children[i]; + rows.splice(aRow + rowCount, 0, child); + if (child.open) + addDescendants(child); + } + } + + if (item.open) + { + // Close the container. Add up all subfolders and their descendants + // who may be open. + multiplier = -1; + let nextRow = aRow + 1; + let nextItem = rows[nextRow]; + while (nextItem && nextItem.level > item.level) + { + rowCount++; + nextItem = rows[++nextRow]; + } + + rows.splice(aRow + 1, rowCount); + } + else + { + // Open the container. Restore the open state of all subfolder and + // their descendants. + multiplier = 1; + addDescendants(item); + } + + let delta = multiplier * rowCount; + this.mRowCount += delta; + + item.open = !item.open; + // Suppress the select event caused by rowCountChanged. + this.selection.selectEventsSuppressed = true; + // Add or remove the children from our view. + this.treeBox.rowCountChanged(aRow, delta); + return delta; + } + }, + + makeFolderObject: function (aFolder, aCurrentLevel) + { + let defaultQuickMode = aFolder.server.getBoolValue("quickMode"); + let optionsAcct = aFolder.isServer ? FeedUtils.getOptionsAcct(aFolder.server) : + null; + let open = !aFolder.isServer && + aFolder.server == this.mRSSServer && + this.mActionMode == this.kImportingOPML ? true : false + let folderObject = { children : [], + folder : aFolder, + name : aFolder.prettyName, + level : aCurrentLevel, + url : aFolder.URI, + quickMode: defaultQuickMode, + options : optionsAcct, + open : open, + container: true, + favicon : null }; + + // If a feed has any sub folders, add them to the list of children. + let folderEnumerator = aFolder.subFolders; + + while (folderEnumerator.hasMoreElements()) + { + let folder = folderEnumerator.getNext(); + if ((folder instanceof Ci.nsIMsgFolder) && + !folder.getFlag(Ci.nsMsgFolderFlags.Trash) && + !folder.getFlag(Ci.nsMsgFolderFlags.Virtual)) + { + folderObject.children + .push(this.makeFolderObject(folder, aCurrentLevel + 1)); + } + } + + let feeds = this.getFeedsInFolder(aFolder); + for (let feed of feeds) + { + // Now add any feed urls for the folder. + folderObject.children.push(this.makeFeedObject(feed, + aFolder, + aCurrentLevel + 1)); + } + + // Finally, set the folder's quickMode based on the its first feed's + // quickMode, since that is how the view determines summary mode, and now + // quickMode is updated to be the same for all feeds in a folder. + if (feeds && feeds[0]) + folderObject.quickMode = feeds[0].quickMode; + + folderObject.children = this.folderItemSorter(folderObject.children); + + return folderObject; + }, + + folderItemSorter: function (aArray) + { + return aArray.sort(function(a, b) { return a.name.toLowerCase() > + b.name.toLowerCase() }). + sort(function(a, b) { return a.container < b.container }); + }, + + getFeedsInFolder: function (aFolder) + { + let feeds = new Array(); + let feedUrlArray = FeedUtils.getFeedUrlsInFolder(aFolder); + if (!feedUrlArray) + // No feedUrls in this folder. + return feeds; + + for (let url of feedUrlArray) + { + let feedResource = FeedUtils.rdf.GetResource(url); + let feed = new Feed(feedResource, aFolder.server); + feeds.push(feed); + } + + return feeds; + }, + + makeFeedObject: function (aFeed, aFolder, aLevel) + { + // Look inside the data source for the feed properties. + let feed = { children : [], + parentFolder: aFolder, + name : aFeed.title || aFeed.description || aFeed.url, + url : aFeed.url, + quickMode : aFeed.quickMode, + options : aFeed.options || FeedUtils.optionsTemplate, + level : aLevel, + open : false, + container : false, + favicon : null }; + return feed; + }, + + loadSubscriptions: function () + { + // Put together an array of folders. Each feed account level folder is + // included as the root. + let numFolders = 0; + let feedContainers = []; + // Get all the feed account folders. + let feedRootFolders = FeedUtils.getAllRssServerRootFolders(); + + feedRootFolders.forEach(function(rootFolder) { + feedContainers.push(this.makeFolderObject(rootFolder, 0)); + numFolders++; + }, this); + + this.mFeedContainers = feedContainers; + this.mView.mRowCount = numFolders; + + FeedSubscriptions.mTree.focus(); + }, + + /** + * Find the folder in the tree. The search may be limited to subfolders of + * a known folder, or expanded to include the entire tree. This function is + * also used to insert/remove folders without rebuilding the tree view cache + * (to avoid position/toggle state loss). + * + * @param aFolder nsIMsgFolder - the folder to find. + * @param [aParams] object - params object, containing: + * + * [parentIndex] int - index of folder to start the search; if + * null (default), the index of the folder's + * rootFolder will be used. + * [select] boolean - if true (default) the folder's ancestors + * will be opened and the folder selected. + * [open] boolean - if true (default) the folder is opened. + * [remove] boolean - delete the item from tree row cache if true, + * false (default) otherwise. + * [newFolder] nsIMsgFolder - if not null (default) the new folder, + * for add or rename. + * + * @return bool found - true if found, false if not. + */ + selectFolder: function(aFolder, aParms) + { + let folderURI = aFolder.URI; + let parentIndex = aParms && ("parentIndex" in aParms) ? aParms.parentIndex : null; + let selectIt = aParms && ("select" in aParms) ? aParms.select : true; + let openIt = aParms && ("open" in aParms) ? aParms.open : true; + let removeIt = aParms && ("remove" in aParms) ? aParms.remove : false; + let newFolder = aParms && ("newFolder" in aParms) ? aParms.newFolder : null; + let startIndex, startItem; + let found = false; + + let firstVisRow, curFirstVisRow, curLastVisRow; + if (this.mView.treeBox) + firstVisRow = this.mView.treeBox.getFirstVisibleRow(); + + if (parentIndex != null) + { + // Use the parentIndex if given. + startIndex = parentIndex; + if (aFolder.isServer) + // Fake item for account root folder. + startItem = { name: "AccountRoot", + children: [this.mView.getItemAtIndex(startIndex)], + container: true, open: false, url: null, level: -1}; + else + startItem = this.mView.getItemAtIndex(startIndex); + } + else + { + // Get the folder's root parent index. + let index = 0; + for (index; index < this.mView.rowCount; index++) + { + let item = this.mView.getItemAtIndex(index); + if (item.url == aFolder.server.rootFolder.URI) + break; + } + startIndex = index; + if (aFolder.isServer) + // Fake item for account root folder. + startItem = { name: "AccountRoot", + children: [this.mView.getItemAtIndex(startIndex)], + container: true, open: false, url: null, level: -1}; + else + startItem = this.mView.getItemAtIndex(startIndex); + } + + function containsFolder(aItem) + { + // Search for the folder. If it's found, set the open state on all + // ancestor folders. A toggle() rebuilds the view rows to match the map. + if (aItem.url == folderURI) + return found = true; + + for (let i = 0; i < aItem.children.length; i++) { + if (aItem.children[i].container && containsFolder(aItem.children[i])) + { + if (removeIt && aItem.children[i].url == folderURI) + { + // Get all occurences in the tree cache arrays. + FeedUtils.log.debug("selectFolder: delete in cache, " + + "parent:children:item:index - "+ + aItem.name + ":" + aItem.children.length + ":" + + aItem.children[i].name + ":" + i); + aItem.children.splice(i, 1); + FeedUtils.log.debug("selectFolder: deleted in cache, " + + "parent:children - " + + aItem.name + ":" + aItem.children.length); + removeIt = false; + return true; + } + if (newFolder) + { + let newItem = FeedSubscriptions.makeFolderObject(newFolder, + aItem.level + 1); + newItem.open = aItem.children[i].open; + if (newFolder.isServer) + FeedSubscriptions.mFeedContainers[startIndex] = newItem; + else + { + aItem.children[i] = newItem; + aItem.children = FeedSubscriptions.folderItemSorter(aItem.children); + } + FeedUtils.log.trace("selectFolder: parentName:newFolderName:newFolderItem - " + + aItem.name + ":" + newItem.name + ":" + newItem.toSource()); + newFolder = null; + return true; + } + if (!found) + { + // For the folder to find. + found = true; + aItem.children[i].open = openIt; + } + else + { + // For ancestor folders. + if (selectIt || openIt) + aItem.children[i].open = true; + } + + return true; + } + } + + return false; + } + + if (startItem) + { + // Find a folder with a specific parent. + containsFolder(startItem); + if (!found) + return false; + + if (!selectIt) + return true; + + if (startItem.open) + this.mView.toggle(startIndex); + this.mView.toggleOpenState(startIndex); + } + + for (let index = 0; index < this.mView.rowCount && selectIt; index++) + { + // The desired folder is now in the view. + let item = this.mView.getItemAtIndex(index); + if (!item.container) + continue; + if (item.url == folderURI) + { + if (item.children.length && + ((!item.open && openIt) || (item.open && !openIt))) + this.mView.toggleOpenState(index); + this.mView.selection.select(index); + found = true; + break; + } + } + + // Ensure tree position does not jump unnecessarily. + curFirstVisRow = this.mView.treeBox.getFirstVisibleRow(); + curLastVisRow = this.mView.treeBox.getLastVisibleRow(); + if (firstVisRow >= 0 && + this.mView.rowCount - curLastVisRow > firstVisRow - curFirstVisRow) + this.mView.treeBox.scrollToRow(firstVisRow); + else + this.mView.treeBox.ensureRowIsVisible(this.mView.rowCount - 1); + + FeedUtils.log.debug("selectFolder: curIndex:firstVisRow:" + + "curFirstVisRow:curLastVisRow:rowCount - " + + this.mView.selection.currentIndex + ":" + + firstVisRow + ":" + + curFirstVisRow + ":" + curLastVisRow + ":" + this.mView.rowCount); + + return found; + }, + + /** + * Find the feed in the tree. The search first gets the feed's folder, + * then selects the child feed. + * + * @param aFeed {Feed object} - the feed to find. + * @param [aParentIndex] integer - index to start the folder search. + * + * @return found bool - true if found, false if not. + */ + selectFeed: function(aFeed, aParentIndex) + { + let folder = aFeed.folder; + let found = false; + + if (aFeed.folder.isServer) { + // If passed the root folder, the caller wants to get the feed's folder + // from the db (for cases of an ancestor folder rename/move). + let itemResource = FeedUtils.rdf.GetResource(aFeed.url); + let ds = FeedUtils.getSubscriptionsDS(aFeed.folder.server); + folder = ds.GetTarget(itemResource, FeedUtils.FZ_DESTFOLDER, true); + } + + if (this.selectFolder(folder, { parentIndex: aParentIndex })) + { + let seln = this.mView.selection; + let item = this.mView.currentItem; + if (item) { + for (let i = seln.currentIndex + 1; i < this.mView.rowCount; i++) { + if (this.mView.getItemAtIndex(i).url == aFeed.url) { + this.mView.selection.select(i); + this.mView.treeBox.ensureRowIsVisible(i); + found = true; + break; + } + } + } + } + + return found; + }, + + updateFeedData: function (aItem) + { + if (!aItem) + return; + + let nameValue = document.getElementById("nameValue"); + let locationValue = document.getElementById("locationValue"); + let locationValidate = document.getElementById("locationValidate"); + let selectFolder = document.getElementById("selectFolder"); + let selectFolderValue = document.getElementById("selectFolderValue"); + let isServer = aItem.folder && aItem.folder.isServer; + let isFolder = aItem.folder && !aItem.folder.isServer; + let isFeed = !aItem.container; + let server, displayFolder; + + if (isFeed) + { + // A feed item. Set the feed location and title info. + nameValue.value = aItem.name; + locationValue.value = aItem.url; + locationValidate.removeAttribute("collapsed"); + + // Root the location picker to the news & blogs server. + server = aItem.parentFolder.server; + displayFolder = aItem.parentFolder; + } + else + { + // A folder/container item. + nameValue.value = ""; + nameValue.disabled = true; + locationValue.value = ""; + locationValidate.setAttribute("collapsed", true); + + server = aItem.folder.server; + displayFolder = aItem.folder; + } + + // Common to both folder and feed items. + nameValue.disabled = aItem.container; + this.setFolderPicker(displayFolder, isFeed); + + // Set quick mode value. + document.getElementById("quickMode").checked = aItem.quickMode; + + // Autotag items. + let autotagEnable = document.getElementById("autotagEnable"); + let autotagUsePrefix = document.getElementById("autotagUsePrefix"); + let autotagPrefix = document.getElementById("autotagPrefix"); + let categoryPrefsAcct = FeedUtils.getOptionsAcct(server).category; + if (isServer) + aItem.options = FeedUtils.getOptionsAcct(server); + let categoryPrefs = aItem.options ? aItem.options.category : null; + + autotagEnable.checked = categoryPrefs && categoryPrefs.enabled; + autotagUsePrefix.checked = categoryPrefs && categoryPrefs.prefixEnabled; + autotagUsePrefix.disabled = !autotagEnable.checked; + autotagPrefix.disabled = autotagUsePrefix.disabled || !autotagUsePrefix.checked; + autotagPrefix.value = categoryPrefs && categoryPrefs.prefix ? + categoryPrefs.prefix : ""; + }, + + setFolderPicker: function(aFolder, aIsFeed) + { + let editFeed = document.getElementById("editFeed"); + let folderPrettyPath = FeedUtils.getFolderPrettyPath(aFolder); + if (!folderPrettyPath) + return editFeed.disabled = true; + + let selectFolder = document.getElementById("selectFolder"); + let selectFolderPopup = document.getElementById("selectFolderPopup"); + let selectFolderValue = document.getElementById("selectFolderValue"); + + selectFolder.setAttribute("hidden", !aIsFeed); + selectFolder._folder = aFolder; + selectFolderValue.setAttribute("hidden", aIsFeed); + selectFolderValue.setAttribute("showfilepath", false); + + if (aIsFeed) + { + selectFolderPopup._ensureInitialized(); + selectFolderPopup.selectFolder(aFolder); + selectFolder.setAttribute("label", folderPrettyPath); + selectFolder.setAttribute("uri", aFolder.URI); + } + else + { + selectFolderValue.value = folderPrettyPath; + selectFolderValue.setAttribute("prettypath", folderPrettyPath); + selectFolderValue.setAttribute("filepath", aFolder.filePath.path); + } + + return editFeed.disabled = false; + }, + + onClickSelectFolderValue: function(aEvent) + { + let target = aEvent.target; + if ((("button" in aEvent) && + (aEvent.button != 0 || + aEvent.originalTarget.localName != "div" || + target.selectionStart != target.selectionEnd)) || + (aEvent.keyCode && aEvent.keyCode != aEvent.DOM_VK_RETURN)) + return; + + // Toggle between showing prettyPath and absolute filePath. + if (target.getAttribute("showfilepath") == "true") + { + target.setAttribute("showfilepath", false); + target.value = target.getAttribute("prettypath"); + } + else + { + target.setAttribute("showfilepath", true); + target.value = target.getAttribute("filepath"); + } + }, + + setNewFolder: function(aEvent) + { + aEvent.stopPropagation(); + this.setFolderPicker(aEvent.target._folder, true); + this.editFeed(); + }, + + setSummary: function(aChecked) + { + let item = this.mView.currentItem; + if (!item || !item.folder) + // Not a folder. + return; + + if (item.folder.isServer) + { + if (document.getElementById("locationValue").value) + // Intent is to add a feed/folder to the account, so return. + return; + + // An account folder. If it changes, all non feed containing subfolders + // need to be updated with the new default. + item.folder.server.setBoolValue("quickMode", aChecked); + this.FolderListener.folderAdded(item.folder); + } + else if (!FeedUtils.getFeedUrlsInFolder(item.folder)) + // Not a folder with feeds. + return; + else + { + let feedsInFolder = this.getFeedsInFolder(item.folder); + // Update the feeds database, for each feed in the folder. + feedsInFolder.forEach(function(feed) { feed.quickMode = aChecked; }); + // Update the folder's feeds properties in the tree map. + item.children.forEach(function(feed) { feed.quickMode = aChecked; }); + let ds = FeedUtils.getSubscriptionsDS(item.folder.server); + ds.Flush(); + } + + // Update the folder in the tree map. + item.quickMode = aChecked; + let message = FeedUtils.strings.GetStringFromName("subscribe-feedUpdated"); + this.updateStatusItem("statusText", message); + }, + + setCategoryPrefs: function(aNode) + { + let item = this.mView.currentItem; + if (!item) + return; + + let isServer = item.folder && item.folder.isServer; + let isFolder = item.folder && !item.folder.isServer; + let autotagEnable = document.getElementById("autotagEnable"); + let autotagUsePrefix = document.getElementById("autotagUsePrefix"); + let autotagPrefix = document.getElementById("autotagPrefix"); + if (isFolder || (isServer && document.getElementById("locationValue").value)) + { + // Intend to subscribe a feed to a folder, a value must be in the url + // field. Update states for addFeed() and return. + autotagUsePrefix.disabled = !autotagEnable.checked; + autotagPrefix.disabled = autotagUsePrefix.disabled || !autotagUsePrefix.checked; + return; + } + + switch (aNode.id) { + case "autotagEnable": + item.options.category.enabled = aNode.checked; + break; + case "autotagUsePrefix": + item.options.category.prefixEnabled = aNode.checked; + item.options.category.prefix = autotagPrefix.value; + break; + } + + if (isServer) + { + FeedUtils.setOptionsAcct(item.folder.server, item.options) + } + else + { + let feedResource = FeedUtils.rdf.GetResource(item.url); + let feed = new Feed(feedResource, item.parentFolder.server); + feed.options = item.options; + let ds = FeedUtils.getSubscriptionsDS(item.parentFolder.server); + ds.Flush(); + } + + this.updateFeedData(item); + let message = FeedUtils.strings.GetStringFromName("subscribe-feedUpdated"); + this.updateStatusItem("statusText", message); + }, + + onKeyPress: function(aEvent) + { + if (aEvent.keyCode == aEvent.DOM_VK_DELETE && + aEvent.target.id == "rssSubscriptionsList") + this.removeFeed(true); + + this.clearStatusInfo(); + }, + + onSelect: function () + { + let item = this.mView.currentItem; + this.updateFeedData(item); + this.setSummaryFocus(); + this.updateButtons(item); + }, + + updateButtons: function (aSelectedItem) + { + let item = aSelectedItem; + let isServer = item && item.folder && item.folder.isServer; + let disable = !item || !item.container || isServer || + this.mActionMode == this.kImportingOPML; + document.getElementById("addFeed").disabled = disable; + disable = !item || (item.container && !isServer) || + this.mActionMode == this.kImportingOPML; + document.getElementById("editFeed").disabled = disable; + disable = !item || item.container || + this.mActionMode == this.kImportingOPML; + document.getElementById("removeFeed").disabled = disable; + disable = !item || !isServer || + this.mActionMode == this.kImportingOPML; + document.getElementById("importOPML").disabled = disable; + document.getElementById("exportOPML").disabled = disable; + }, + + onMouseDown: function (aEvent) + { + if (aEvent.button != 0 || + aEvent.target.id == "validationText" || + aEvent.target.id == "addCertException") + return; + + this.clearStatusInfo(); + }, + + setSummaryFocus: function () + { + let item = this.mView.currentItem; + if (!item) + return; + + let locationValue = document.getElementById("locationValue"); + let quickMode = document.getElementById("quickMode"); + let autotagEnable = document.getElementById("autotagEnable"); + let autotagUsePrefix = document.getElementById("autotagUsePrefix"); + let autotagPrefix = document.getElementById("autotagPrefix"); + let isServer = item.folder && item.folder.isServer; + let isFolder = item.folder && !item.folder.isServer; + let isFeed = !item.container; + + // Enable summary/autotag by default. + quickMode.disabled = autotagEnable.disabled = false; + autotagUsePrefix.disabled = !autotagEnable.checked; + autotagPrefix.disabled = autotagUsePrefix.disabled || !autotagUsePrefix.checked; + + if (isServer) + { + let disable = locationValue.hasAttribute("focused") || locationValue.value; + document.getElementById("addFeed").disabled = !disable; + document.getElementById("editFeed").disabled = disable; + + } + else if (isFolder) + { + if (!locationValue.hasAttribute("focused") && !locationValue.value) + { + // Enabled for a folder with feeds. Autotag disabled unless intent is + // to add a feed. + quickMode.disabled = !FeedUtils.getFeedUrlsInFolder(item.folder); + autotagEnable.disabled = autotagUsePrefix.disabled = + autotagPrefix.disabled = true; + } + } + else + { + // Summary is per folder. + quickMode.disabled = true; + } + }, + + removeFeed: function (aPrompt) + { + let seln = this.mView.selection; + if (seln.count != 1) + return; + + let itemToRemove = this.mView.getItemAtIndex(seln.currentIndex); + + if (!itemToRemove || itemToRemove.container) + return; + + if (aPrompt) + { + // Confirm unsubscribe prompt. + let pTitle = FeedUtils.strings.GetStringFromName( + "subscribe-confirmFeedDeletionTitle"); + let pMessage = FeedUtils.strings.formatStringFromName( + "subscribe-confirmFeedDeletion", [itemToRemove.name], 1); + if (Services.prompt.confirmEx(window, pTitle, pMessage, + Ci.nsIPromptService.STD_YES_NO_BUTTONS, + null, null, null, null, { })) + return; + } + + FeedUtils.deleteFeed(FeedUtils.rdf.GetResource(itemToRemove.url), + itemToRemove.parentFolder.server, + itemToRemove.parentFolder); + + // Now that we have removed the feed from the datasource, it is time to + // update our view layer. Update parent folder's quickMode if necessary + // and remove the child from its parent folder object. + let parentIndex = this.mView.getParentIndex(seln.currentIndex); + let parentItem = this.mView.getItemAtIndex(parentIndex); + this.updateFolderQuickModeInView(itemToRemove, parentItem, true); + this.mView.removeItemAtIndex(seln.currentIndex, false); + let message = FeedUtils.strings.GetStringFromName("subscribe-feedRemoved"); + this.updateStatusItem("statusText", message); + }, + + + /** + * This addFeed is used by 1) Add button, 1) Update button, 3) Drop of a + * feed url on a folder (which can be an add or move). If Update, the new + * url is added and the old removed; thus aParse is false and no new messages + * are downloaded, the feed is only validated and stored in the db. If dnd, + * the drop folder is selected and the url is prefilled, so proceed just as + * though the url were entered manually. This allows a user to see the dnd + * url better in case of errors. + * + * @param [aFeedLocation] string - the feed url; get the url from the + * input field if null. + * @param [aFolder] nsIMsgFolder - folder to subscribe, current selected + * folder if null. + * @param [aParse] boolean - if true (default) parse and download + * the feed's articles. + * @param [aParams] object - additional params. + * @param [aMode] integer - action mode (default is kSubscribeMode) + * of the add. + * + * @return success boolean - true if edit checks passed and an + * async download has been initiated. + */ + addFeed: function(aFeedLocation, aFolder, aParse, aParams, aMode) + { + let message; + let parse = aParse == null ? true : aParse; + let mode = aMode == null ? this.kSubscribeMode : aMode; + let locationValue = document.getElementById("locationValue"); + let quickMode = aParams && ("quickMode" in aParams) ? + aParams.quickMode : document.getElementById("quickMode").checked; + let name = aParams && ("name" in aParams) ? + aParams.name : document.getElementById("nameValue").value; + let options = aParams && ("options" in aParams) ? + aParams.options : null; + + if (aFeedLocation) + locationValue.value = aFeedLocation; + let feedLocation = locationValue.value.trim(); + + if (!feedLocation) + { + message = locationValue.getAttribute("placeholder"); + this.updateStatusItem("statusText", message); + return false; + } + + if (!FeedUtils.isValidScheme(feedLocation)) + { + message = FeedUtils.strings.GetStringFromName("subscribe-feedNotValid"); + this.updateStatusItem("statusText", message); + return false; + } + + let addFolder; + if (aFolder) + { + // For Update or if passed a folder. + if (aFolder instanceof Ci.nsIMsgFolder) + addFolder = aFolder; + } + else + { + // A folder must be selected for Add and Drop. + let index = this.mView.selection.currentIndex; + let item = this.mView.getItemAtIndex(index); + if (item && item.container) + addFolder = item.folder; + } + + // Shouldn't happen. Or else not passed an nsIMsgFolder. + if (!addFolder) + return false; + + // Before we go any further, make sure the user is not already subscribed + // to this feed. + if (FeedUtils.feedAlreadyExists(feedLocation, addFolder.server)) + { + message = FeedUtils.strings.GetStringFromName( + "subscribe-feedAlreadySubscribed"); + this.updateStatusItem("statusText", message); + return false; + } + + if (!options) + { + // Not passed a param, get values from the ui. + options = FeedUtils.optionsTemplate; + options.category.enabled = document.getElementById("autotagEnable").checked; + options.category.prefixEnabled = document.getElementById("autotagUsePrefix").checked; + options.category.prefix = document.getElementById("autotagPrefix").value; + } + + let folderURI = addFolder.isServer ? null : addFolder.URI; + let feedProperties = { feedName : name, + feedLocation : feedLocation, + folderURI : folderURI, + server : addFolder.server, + quickMode : quickMode, + options : options }; + + let feed = this.storeFeed(feedProperties); + if (!feed) + return false; + + // Now validate and start downloading the feed. + message = FeedUtils.strings.GetStringFromName("subscribe-validating-feed"); + this.updateStatusItem("statusText", message); + this.updateStatusItem("progressMeter", 0); + document.getElementById("addFeed").setAttribute("disabled", true); + this.mActionMode = mode; + feed.download(parse, this.mFeedDownloadCallback); + return true; + }, + + // Helper routine used by addFeed and importOPMLFile. + storeFeed: function(feedProperties) + { + let itemResource = FeedUtils.rdf.GetResource(feedProperties.feedLocation); + let feed = new Feed(itemResource, feedProperties.server); + + // If the user specified a folder to add the feed to, then set it here. + if (feedProperties.folderURI) + { + let folderResource = FeedUtils.rdf.GetResource(feedProperties.folderURI); + if (folderResource) + { + let folder = folderResource.QueryInterface(Ci.nsIMsgFolder); + if (folder && !folder.isServer) + feed.folder = folder; + } + } + + feed.title = feedProperties.feedName; + feed.quickMode = feedProperties.quickMode; + feed.options = feedProperties.options; + return feed; + }, + + updateAccount: function(aItem) + { + // Check to see if the categoryPrefs custom prefix string value changed. + let editAutotagPrefix = document.getElementById("autotagPrefix").value; + if (aItem.options.category.prefix != editAutotagPrefix) + { + aItem.options.category.prefix = editAutotagPrefix; + FeedUtils.setOptionsAcct(aItem.folder.server, aItem.options) + let message = FeedUtils.strings.GetStringFromName("subscribe-feedUpdated"); + this.updateStatusItem("statusText", message); + } + }, + + editFeed: function() + { + let seln = this.mView.selection; + if (seln.count != 1) + return; + + let itemToEdit = this.mView.getItemAtIndex(seln.currentIndex); + if (itemToEdit.folder && itemToEdit.folder.isServer) + { + this.updateAccount(itemToEdit) + return; + } + + if (!itemToEdit || itemToEdit.container || !itemToEdit.parentFolder) + return; + + let resource = FeedUtils.rdf.GetResource(itemToEdit.url); + let currentFolderServer = itemToEdit.parentFolder.server; + let ds = FeedUtils.getSubscriptionsDS(currentFolderServer); + let currentFolderURI = itemToEdit.parentFolder.URI; + let feed = new Feed(resource, currentFolderServer); + feed.folder = itemToEdit.parentFolder; + + let editNameValue = document.getElementById("nameValue").value; + let editFeedLocation = document.getElementById("locationValue").value.trim(); + let selectFolder = document.getElementById("selectFolder"); + let editQuickMode = document.getElementById("quickMode").checked; + let editAutotagPrefix = document.getElementById("autotagPrefix").value; + + if (feed.url != editFeedLocation) + { + // Updating a url. We need to add the new url and delete the old, to + // ensure everything is cleaned up correctly. + this.addFeed(null, itemToEdit.parentFolder, false, null, this.kUpdateMode) + return; + } + + // Did the user change the folder URI for storing the feed? + let editFolderURI = selectFolder.getAttribute("uri"); + if (currentFolderURI != editFolderURI) + { + // Make sure the new folderpicked folder is visible. + this.selectFolder(selectFolder._folder); + // Now go back to the feed item. + this.selectFeed(feed, null); + // We need to find the index of the new parent folder. + let newParentIndex = this.mView.kRowIndexUndefined; + for (let index = 0; index < this.mView.rowCount; index++) + { + let item = this.mView.getItemAtIndex(index); + if (item && item.container && item.url == editFolderURI) + { + newParentIndex = index; + break; + } + } + + if (newParentIndex != this.mView.kRowIndexUndefined) + this.moveCopyFeed(seln.currentIndex, newParentIndex, "move"); + + return; + } + + let updated = false; + let message = ""; + // Disable the button until the update completes and we process the async + // verify response. + document.getElementById("editFeed").setAttribute("disabled", true); + + // Check to see if the title value changed, no blank title allowed. + if (feed.title != editNameValue) + { + if (!editNameValue) + { + document.getElementById("nameValue").value = feed.title; + } + else + { + feed.title = editNameValue; + itemToEdit.name = editNameValue; + seln.tree.invalidateRow(seln.currentIndex); + updated = true; + } + } + + // Check to see if the quickMode value changed. + if (feed.quickMode != editQuickMode) + { + feed.quickMode = editQuickMode; + itemToEdit.quickMode = editQuickMode; + updated = true; + } + + // Check to see if the categoryPrefs custom prefix string value changed. + if (itemToEdit.options.category.prefix != editAutotagPrefix && + itemToEdit.options.category.prefix != null && + editAutotagPrefix != "") + { + itemToEdit.options.category.prefix = editAutotagPrefix; + feed.options = itemToEdit.options; + updated = true; + } + + let verifyDelay = 0; + if (updated) { + ds.Flush(); + message = FeedUtils.strings.GetStringFromName("subscribe-feedUpdated"); + this.updateStatusItem("statusText", message); + verifyDelay = 1500; + } + + // Now we want to verify if the stored feed url still works. If it + // doesn't, show the error. Delay a bit to leave Updated message visible. + message = FeedUtils.strings.GetStringFromName("subscribe-validating-feed"); + this.mActionMode = this.kVerifyUrlMode; + setTimeout(() => { + this.updateStatusItem("statusText", message); + this.updateStatusItem("progressMeter", "?"); + feed.download(false, this.mFeedDownloadCallback); + }, verifyDelay); + }, + +/** + * Moves or copies a feed to another folder or account. + * + * @param int aOldFeedIndex - index in tree of target feed item. + * @param int aNewParentIndex - index in tree of target parent folder item. + * @param string aMoveCopy - either "move" or "copy". + */ + moveCopyFeed: function(aOldFeedIndex, aNewParentIndex, aMoveCopy) + { + let moveFeed = aMoveCopy == "move"; + let currentItem = this.mView.getItemAtIndex(aOldFeedIndex); + if (!currentItem || + this.mView.getParentIndex(aOldFeedIndex) == aNewParentIndex) + // If the new parent is the same as the current parent, then do nothing. + return; + + let currentParentIndex = this.mView.getParentIndex(aOldFeedIndex); + let currentParentItem = this.mView.getItemAtIndex(currentParentIndex); + let currentParentResource = FeedUtils.rdf.GetResource(currentParentItem.url); + let currentFolder = currentParentResource.QueryInterface(Ci.nsIMsgFolder); + + let newParentItem = this.mView.getItemAtIndex(aNewParentIndex); + let newParentResource = FeedUtils.rdf.GetResource(newParentItem.url); + let newFolder = newParentResource.QueryInterface(Ci.nsIMsgFolder); + + let ds = FeedUtils.getSubscriptionsDS(currentItem.parentFolder.server); + let resource = FeedUtils.rdf.GetResource(currentItem.url); + + let accountMoveCopy = false; + if (currentFolder.rootFolder.URI == newFolder.rootFolder.URI) + { + // Moving within the same account/feeds db. + if (newFolder.isServer || !moveFeed) + // No moving to account folder if already in the account; can only move, + // not copy, to folder in the same account. + return; + + // Unassert the older URI, add an assertion for the new parent URI. + ds.Change(resource, FeedUtils.FZ_DESTFOLDER, + currentParentResource, newParentResource); + ds.Flush(); + + // Update folderpane favicons. + FeedUtils.setFolderPaneProperty(currentFolder, "favicon", null, "row"); + FeedUtils.setFolderPaneProperty(newFolder, "favicon", null, "row"); + } + else + { + // Moving/copying to a new account. If dropping on the account folder, + // a new subfolder is created if necessary. + accountMoveCopy = true; + let mode = moveFeed ? this.kMoveMode : this.kCopyMode; + let params = {quickMode: currentItem.quickMode, + name: currentItem.name, + options: currentItem.options}; + // Subscribe to the new folder first. If it already exists in the + // account or on error, return. + if (!this.addFeed(currentItem.url, newFolder, false, params, mode)) + return; + // Unsubscribe the feed from the old folder, if add to the new folder + // is successfull, and doing a move. + if (moveFeed) + FeedUtils.deleteFeed(FeedUtils.rdf.GetResource(currentItem.url), + currentItem.parentFolder.server, + currentItem.parentFolder); + } + + // Update local favicons. + currentParentItem.favicon = newParentItem.favicon = null; + + // Finally, update our view layer. Update old parent folder's quickMode + // and remove the old row, if move. Otherwise no change to the view. + if (moveFeed) + { + this.updateFolderQuickModeInView(currentItem, currentParentItem, true); + this.mView.removeItemAtIndex(aOldFeedIndex, true); + if (aNewParentIndex > aOldFeedIndex) + aNewParentIndex--; + } + + if (accountMoveCopy) + { + // If a cross account move/copy, download callback will update the view + // with the new location. Preselect folder/mode for callback. + this.selectFolder(newFolder, { parentIndex: aNewParentIndex }); + return; + } + + // Add the new row location to the view. + currentItem.level = newParentItem.level + 1; + currentItem.parentFolder = newFolder; + this.updateFolderQuickModeInView(currentItem, newParentItem, false); + newParentItem.children.push(currentItem); + + if (newParentItem.open) + // Close the container, selecting the feed will rebuild the view rows. + this.mView.toggle(aNewParentIndex); + + this.selectFeed({folder: newParentItem.folder, url: currentItem.url}, + aNewParentIndex); + + let message = FeedUtils.strings.GetStringFromName("subscribe-feedMoved"); + this.updateStatusItem("statusText", message); + }, + + updateFolderQuickModeInView: function (aFeedItem, aParentItem, aRemove) + { + let feedItem = aFeedItem; + let parentItem = aParentItem; + let feedUrlArray = FeedUtils.getFeedUrlsInFolder(feedItem.parentFolder); + let feedsInFolder = feedUrlArray ? feedUrlArray.length : 0; + + if (aRemove && feedsInFolder < 1) + // Removed only feed in folder; set quickMode to server default. + parentItem.quickMode = parentItem.folder.server.getBoolValue("quickMode"); + + if (!aRemove) + { + // Just added a feed to a folder. If there are already feeds in the + // folder, the feed must reflect the parent's quickMode. If it is the + // only feed, update the parent folder to the feed's quickMode. + if (feedsInFolder > 1) + { + let feedResource = FeedUtils.rdf.GetResource(feedItem.url); + let feed = new Feed(feedResource, feedItem.parentFolder.server); + feed.quickMode = parentItem.quickMode; + feedItem.quickMode = parentItem.quickMode; + } + else + parentItem.quickMode = feedItem.quickMode; + } + }, + + onDragStart: function (aEvent) + { + // Get the selected feed article (if there is one). + let seln = this.mView.selection; + if (seln.count != 1) + return; + + // Only initiate a drag if the item is a feed (ignore folders/containers). + let item = this.mView.getItemAtIndex(seln.currentIndex); + if (!item || item.container) + return; + + aEvent.dataTransfer.setData("text/x-moz-feed-index", seln.currentIndex); + aEvent.dataTransfer.effectAllowed = "copyMove"; + }, + + onDragOver: function (aEvent) + { + this.mView._currentDataTransfer = aEvent.dataTransfer; + }, + + mFeedDownloadCallback: + { + mSubscribeMode: true, + downloaded: function(feed, aErrorCode) + { + // Offline check is done in the context of 3pane, return to the subscribe + // window once the modal prompt is dispatched. + window.focus(); + // Feed is null if our attempt to parse the feed failed. + let message = ""; + let win = FeedSubscriptions; + if (aErrorCode == FeedUtils.kNewsBlogSuccess || + aErrorCode == FeedUtils.kNewsBlogNoNewItems) + { + win.updateStatusItem("progressMeter", 100); + + if (win.mActionMode == win.kVerifyUrlMode) { + // Just checking for errors, if none bye. The (non error) code + // kNewsBlogNoNewItems can only happen in verify mode. + win.mActionMode = null; + win.clearStatusInfo(); + message = FeedUtils.strings.GetStringFromName("subscribe-feedVerified"); + win.updateStatusItem("statusText", message); + document.getElementById("editFeed").removeAttribute("disabled"); + return; + } + + // Add the feed to the databases. + FeedUtils.addFeed(feed); + + // Now add the feed to our view. If adding, the current selection will + // be a folder; if updating it will be a feed. No need to rebuild the + // entire view, that is too jarring. + let curIndex = win.mView.selection.currentIndex; + let curItem = win.mView.getItemAtIndex(curIndex); + if (curItem) + { + let parentIndex, parentItem, newItem, level; + let rows = win.mFeedContainers; + if (curItem.container) + { + // Open the container, if it exists. + let folderExists = win.selectFolder(feed.folder, + { parentIndex: curIndex }); + if (!folderExists) + { + // This means a new folder was created. + parentIndex = curIndex; + parentItem = curItem; + level = curItem.level + 1; + newItem = win.makeFolderObject(feed.folder, level); + } + else + { + // If a folder happens to exist which matches one that would + // have been created, the feed system reuses it. Get the + // current item again if reusing a previously unselected folder. + curIndex = win.mView.selection.currentIndex; + curItem = win.mView.getItemAtIndex(curIndex); + parentIndex = curIndex; + parentItem = curItem; + level = curItem.level + 1; + newItem = win.makeFeedObject(feed, feed.folder, level); + } + } + else + { + // Adding a feed. + parentIndex = win.mView.getParentIndex(curIndex); + parentItem = win.mView.getItemAtIndex(parentIndex); + level = curItem.level; + newItem = win.makeFeedObject(feed, feed.folder, level); + } + + if (!newItem.container) + win.updateFolderQuickModeInView(newItem, parentItem, false); + parentItem.children.push(newItem); + parentItem.children = win.folderItemSorter(parentItem.children); + parentItem.favicon = null; + + if (win.mActionMode == win.kSubscribeMode) + message = FeedUtils.strings.GetStringFromName( + "subscribe-feedAdded"); + if (win.mActionMode == win.kUpdateMode) + { + win.removeFeed(false); + message = FeedUtils.strings.GetStringFromName( + "subscribe-feedUpdated"); + } + if (win.mActionMode == win.kMoveMode) + message = FeedUtils.strings.GetStringFromName( + "subscribe-feedMoved"); + if (win.mActionMode == win.kCopyMode) + message = FeedUtils.strings.GetStringFromName( + "subscribe-feedCopied"); + + win.selectFeed(feed, parentIndex); + } + } + else + { + // Non success. Remove intermediate traces from the feeds database. + // But only if we're not in verify mode. + if (win.mActionMode != win.kVerifyUrlMode && + feed && feed.url && feed.server) + FeedUtils.deleteFeed(FeedUtils.rdf.GetResource(feed.url), + feed.server, + feed.server.rootFolder); + + if (aErrorCode == FeedUtils.kNewsBlogInvalidFeed) + message = FeedUtils.strings.GetStringFromName( + "subscribe-feedNotValid"); + if (aErrorCode == FeedUtils.kNewsBlogRequestFailure) + message = FeedUtils.strings.GetStringFromName( + "subscribe-networkError"); + if (aErrorCode == FeedUtils.kNewsBlogFileError) + message = FeedUtils.strings.GetStringFromName( + "subscribe-errorOpeningFile"); + if (aErrorCode == FeedUtils.kNewsBlogBadCertError) { + let host = Services.io.newURI(feed.url, null, null).host; + message = FeedUtils.strings.formatStringFromName( + "newsblog-badCertError", [host], 1); + } + if (aErrorCode == FeedUtils.kNewsBlogNoAuthError) + message = FeedUtils.strings.GetStringFromName( + "subscribe-noAuthError"); + + if (win.mActionMode != win.kUpdateMode && + win.mActionMode != win.kVerifyUrlMode) + // Re-enable the add button if subscribe failed. + document.getElementById("addFeed").removeAttribute("disabled"); + if (win.mActionMode == win.kVerifyUrlMode) + // Re-enable the update button if verify failed. + document.getElementById("editFeed").removeAttribute("disabled"); + } + + win.mActionMode = null; + win.clearStatusInfo(); + let code = feed.url.startsWith("http") ? aErrorCode : null; + win.updateStatusItem("statusText", message, code); + }, + + // This gets called after the RSS parser finishes storing a feed item to + // disk. aCurrentFeedItems is an integer corresponding to how many feed + // items have been downloaded so far. aMaxFeedItems is an integer + // corresponding to the total number of feed items to download. + onFeedItemStored: function (feed, aCurrentFeedItems, aMaxFeedItems) + { + window.focus(); + let message = FeedUtils.strings.formatStringFromName( + "subscribe-gettingFeedItems", + [aCurrentFeedItems, aMaxFeedItems], 2); + FeedSubscriptions.updateStatusItem("statusText", message); + this.onProgress(feed, aCurrentFeedItems, aMaxFeedItems); + }, + + onProgress: function(feed, aProgress, aProgressMax, aLengthComputable) + { + FeedSubscriptions.updateStatusItem("progressMeter", + (aProgress * 100) / aProgressMax); + } + }, + + // Status routines. + updateStatusItem: function(aID, aValue, aErrorCode) + { + let el = document.getElementById(aID); + if (el.getAttribute("collapsed")) + el.removeAttribute("collapsed"); + + if (aID == "progressMeter") + el.setAttribute("mode", aValue == "?" ? "undetermined" : "determined"); + + if (aID == "statusText") + el.textContent = aValue; + else + el.value = aValue; + + el = document.getElementById("validationText"); + if (aErrorCode == FeedUtils.kNewsBlogInvalidFeed) + el.removeAttribute("collapsed"); + else + el.setAttribute("collapsed", true); + + el = document.getElementById("addCertException"); + if (aErrorCode == FeedUtils.kNewsBlogBadCertError) + el.removeAttribute("collapsed"); + else + el.setAttribute("collapsed", true); + }, + + clearStatusInfo: function() + { + document.getElementById("statusText").textContent = ""; + document.getElementById("progressMeter").collapsed = true; + document.getElementById("validationText").collapsed = true; + document.getElementById("addCertException").collapsed = true; + }, + + checkValidation: function(aEvent) + { + if (aEvent.button != 0) + return; + + let validationSite = "http://validator.w3.org"; + let validationQuery = "http://validator.w3.org/feed/check.cgi?url="; + + if (this.mMainWin) + { + let tabmail = this.mMainWin.document.getElementById("tabmail"); + if (tabmail) + { + let feedLocation = document.getElementById("locationValue").value; + let url = validationQuery + encodeURIComponent(feedLocation); + + this.mMainWin.focus(); + this.mMainWin.openContentTab(url, "tab", "^" + validationSite); + FeedUtils.log.debug("checkValidation: query url - " + url); + } + } + aEvent.stopPropagation(); + }, + + addCertExceptionDialog: function() + { + let feedURL = document.getElementById("locationValue").value.trim(); + let params = { exceptionAdded : false, + location: feedURL, + prefetchCert: true }; + window.openDialog("chrome://pippki/content/exceptionDialog.xul", + "", "chrome,centerscreen,modal", params); + if (params.exceptionAdded) + this.clearStatusInfo(); + }, + + // Listener for folder pane changes. + FolderListener: { + get feedWindow() { + let subscriptionsWindow = + Services.wm.getMostRecentWindow("Mail:News-BlogSubscriptions"); + return subscriptionsWindow ? subscriptionsWindow.FeedSubscriptions : null; + }, + + get currentSelectedIndex() { + return this.feedWindow ? this.feedWindow.mView.selection.currentIndex : -1; + }, + + get currentSelectedItem() { + return this.feedWindow ? this.feedWindow.mView.currentItem : null; + }, + + folderAdded: function(aFolder) + { + if (aFolder.server.type != "rss" || + FeedUtils.isInTrash(aFolder)) + return; + + let parentFolder = aFolder.isServer ? aFolder : aFolder.parent; + FeedUtils.log.debug("folderAdded: folder:parent - " + aFolder.name + ":" + + (parentFolder ? parentFolder.filePath.path : "(null)")); + + if (!parentFolder || !this.feedWindow) + return; + + let feedWindow = this.feedWindow; + let curSelIndex = this.currentSelectedIndex; + let curSelItem = this.currentSelectedItem; + let firstVisRow = feedWindow.mView.treeBox.getFirstVisibleRow(); + let indexInView = feedWindow.mView.getItemInViewIndex(parentFolder); + let open = indexInView != null; + + if (aFolder.isServer) + { + if (indexInView != null) + // Existing account root folder in the view. + open = feedWindow.mView.getItemAtIndex(indexInView).open; + else + { + // Add the account root folder to the view. + feedWindow.mFeedContainers.push(feedWindow.makeFolderObject(parentFolder, 0)); + feedWindow.mView.mRowCount++; + feedWindow.mTree.view = feedWindow.mView; + feedWindow.mView.treeBox.scrollToRow(firstVisRow); + return; + } + } + + // Rebuild the added folder's parent item in the tree row cache. + feedWindow.selectFolder(parentFolder, { select: false, + open: open, + newFolder: parentFolder }); + + if (indexInView == null || !curSelItem) + // Folder isn't in the tree view, no need to update the view. + return; + + let parentIndex = feedWindow.mView.getParentIndex(indexInView); + if (parentIndex == feedWindow.mView.kRowIndexUndefined) + // Root folder is its own parent. + parentIndex = indexInView; + if (open) + { + // Close an open parent (or root) folder. + feedWindow.mView.toggle(parentIndex); + feedWindow.mView.toggleOpenState(parentIndex); + } + feedWindow.mView.treeBox.scrollToRow(firstVisRow); + + if (curSelItem.container) + feedWindow.selectFolder(curSelItem.folder, { open: curSelItem.open }); + else + feedWindow.selectFeed({ folder: curSelItem.parentFolder, + url: curSelItem.url }, parentIndex); + }, + + folderDeleted: function(aFolder) + { + if (aFolder.server.type != "rss" || FeedUtils.isInTrash(aFolder)) + return; + + FeedUtils.log.debug("folderDeleted: folder - " + aFolder.name); + if (!this.feedWindow) + return; + + let feedWindow = this.feedWindow; + let curSelIndex = this.currentSelectedIndex; + let indexInView = feedWindow.mView.getItemInViewIndex(aFolder); + let open = indexInView != null; + + // Delete the folder from the tree row cache. + feedWindow.selectFolder(aFolder, { select: false, open: false, remove: true }); + + if (!open || curSelIndex < 0) + // Folder isn't in the tree view, no need to update the view. + return; + + let select = + indexInView == curSelIndex || + feedWindow.mView.isIndexChildOfParentIndex(indexInView, curSelIndex); + feedWindow.mView.removeItemAtIndex(indexInView, !select); + }, + + folderRenamed: function(aOrigFolder, aNewFolder) + { + if (aNewFolder.server.type != "rss" || FeedUtils.isInTrash(aNewFolder)) + return; + + FeedUtils.log.debug("folderRenamed: old:new - " + + aOrigFolder.name + ":" + aNewFolder.name); + if (!this.feedWindow) + return; + + let feedWindow = this.feedWindow; + let curSelIndex = this.currentSelectedIndex; + let curSelItem = this.currentSelectedItem; + let firstVisRow = feedWindow.mView.treeBox.getFirstVisibleRow(); + let indexInView = feedWindow.mView.getItemInViewIndex(aOrigFolder); + let open = indexInView != null; + + // Rebuild the renamed folder's item in the tree row cache. + feedWindow.selectFolder(aOrigFolder, { select: false, + open: open, + newFolder: aNewFolder }); + + if (!open || !curSelItem) + // Folder isn't in the tree view, no need to update the view. + return; + + let select = + indexInView == curSelIndex || + feedWindow.mView.isIndexChildOfParentIndex(indexInView, curSelIndex); + let parentIndex = feedWindow.mView.getParentIndex(indexInView); + if (parentIndex == feedWindow.mView.kRowIndexUndefined) + // Root folder is its own parent. + parentIndex = indexInView; + feedWindow.mView.toggle(parentIndex); + feedWindow.mView.toggleOpenState(parentIndex); + feedWindow.mView.treeBox.scrollToRow(firstVisRow); + + if (curSelItem.container) { + if (curSelItem.folder == aOrigFolder) + feedWindow.selectFolder(aNewFolder, { open: curSelItem.open }); + else if (select) + feedWindow.mView.selection.select(indexInView); + else + feedWindow.selectFolder(curSelItem.folder, { open: curSelItem.open }); + } + else + feedWindow.selectFeed({ folder: curSelItem.parentFolder.rootFolder, + url: curSelItem.url }, parentIndex); + }, + + folderMoveCopyCompleted: function(aMove, aSrcFolder, aDestFolder) + { + if (aDestFolder.server.type != "rss") + return; + + FeedUtils.log.debug("folderMoveCopyCompleted: move:src:dest - " + + aMove + ":" + aSrcFolder.name + ":" + aDestFolder.name); + if (!this.feedWindow) + return; + + let feedWindow = this.feedWindow; + let curSelIndex = this.currentSelectedIndex; + let curSelItem = this.currentSelectedItem; + let firstVisRow = feedWindow.mView.treeBox.getFirstVisibleRow(); + let indexInView = feedWindow.mView.getItemInViewIndex(aSrcFolder); + let destIndexInView = feedWindow.mView.getItemInViewIndex(aDestFolder); + let open = indexInView != null || destIndexInView != null; + let parentIndex = feedWindow.mView.getItemInViewIndex(aDestFolder.parent || + aDestFolder); + let select = + indexInView == curSelIndex || + feedWindow.mView.isIndexChildOfParentIndex(indexInView, curSelIndex); + + if (aMove) + { + this.folderDeleted(aSrcFolder); + if (aDestFolder.getFlag(Ci.nsMsgFolderFlags.Trash)) + return; + } + + setTimeout(function() { + // State on disk needs to settle before a folder object can be rebuilt. + feedWindow.selectFolder(aDestFolder, { select: false, + open: open || select, + newFolder: aDestFolder }); + + if (!open || !curSelItem) + // Folder isn't in the tree view, no need to update the view. + return; + + feedWindow.mView.toggle(parentIndex); + feedWindow.mView.toggleOpenState(parentIndex); + feedWindow.mView.treeBox.scrollToRow(firstVisRow); + + if (curSelItem.container) { + if (curSelItem.folder == aSrcFolder || select) + feedWindow.selectFolder(aDestFolder, { open: true }); + else + feedWindow.selectFolder(curSelItem.folder, { open: curSelItem.open }); + } + else + feedWindow.selectFeed({ folder: curSelItem.parentFolder.rootFolder, + url: curSelItem.url }, null); + }, 50); + } + }, + + /* *************************************************************** */ + /* OPML Functions */ + /* *************************************************************** */ + + get brandShortName() { + let brandBundle = document.getElementById("bundle_brand"); + return brandBundle ? brandBundle.getString("brandShortName") : ""; + }, + +/** + * Export feeds as opml file Save As filepicker function. + * + * @param bool aList - if true, exporting as list; if false (default) + * exporting feeds in folder structure - used for title. + * @return nsILocalFile or null. + */ + opmlPickSaveAsFile: function(aList) + { + let accountName = this.mRSSServer.rootFolder.prettyName; + let fileName = FeedUtils.strings.formatStringFromName( + "subscribe-OPMLExportDefaultFileName", + [this.brandShortName, accountName], 2); + let title = aList ? FeedUtils.strings.formatStringFromName( + "subscribe-OPMLExportTitleList", [accountName], 1) : + FeedUtils.strings.formatStringFromName( + "subscribe-OPMLExportTitleStruct", [accountName], 1); + let fp = Cc["@mozilla.org/filepicker;1"].createInstance(Ci.nsIFilePicker); + + fp.defaultString = fileName; + fp.defaultExtension = "opml"; + if (this.opmlLastSaveAsDir && (this.opmlLastSaveAsDir instanceof Ci.nsILocalFile)) + fp.displayDirectory = this.opmlLastSaveAsDir; + + let opmlFilterText = FeedUtils.strings.GetStringFromName( + "subscribe-OPMLExportOPMLFilesFilterText"); + fp.appendFilter(opmlFilterText, "*.opml"); + fp.appendFilters(Ci.nsIFilePicker.filterAll); + fp.filterIndex = 0; + fp.init(window, title, Ci.nsIFilePicker.modeSave); + + if (fp.show() != Ci.nsIFilePicker.returnCancel && fp.file) + { + this.opmlLastSaveAsDir = fp.file.parent; + return fp.file; + } + + return null; + }, + +/** + * Import feeds opml file Open filepicker function. + * + * @return nsILocalFile or null. + */ + opmlPickOpenFile: function() + { + let title = FeedUtils.strings.GetStringFromName("subscribe-OPMLImportTitle"); + let fp = Cc["@mozilla.org/filepicker;1"].createInstance(Ci.nsIFilePicker); + + fp.defaultString = ""; + if (this.opmlLastOpenDir && (this.opmlLastOpenDir instanceof Ci.nsILocalFile)) + fp.displayDirectory = this.opmlLastOpenDir; + + let opmlFilterText = FeedUtils.strings.GetStringFromName( + "subscribe-OPMLExportOPMLFilesFilterText"); + fp.appendFilter(opmlFilterText, "*.opml"); + fp.appendFilters(Ci.nsIFilePicker.filterXML); + fp.appendFilters(Ci.nsIFilePicker.filterAll); + fp.init(window, title, Ci.nsIFilePicker.modeOpen); + + if (fp.show() != Ci.nsIFilePicker.returnCancel && fp.file) + { + this.opmlLastOpenDir = fp.file.parent; + return fp.file; + } + + return null; + }, + + exportOPML: function(aEvent) + { + // Account folder must be selected. + let item = this.mView.currentItem; + if (!item || !item.folder || !item.folder.isServer) + return; + + this.mRSSServer = item.folder.server; + let rootFolder = this.mRSSServer.rootFolder; + let exportAsList = aEvent.ctrlKey; + let SPACES2 = " "; + let SPACES4 = " "; + + if (this.mRSSServer.rootFolder.hasSubFolders) + { + let opmlDoc = document.implementation.createDocument("", "opml", null); + let opmlRoot = opmlDoc.documentElement; + opmlRoot.setAttribute("version", "1.0"); + opmlRoot.setAttribute("xmlns:fz", "urn:forumzilla:"); + + this.generatePPSpace(opmlRoot, SPACES2); + + // Make the <head> element. + let head = opmlDoc.createElement("head"); + this.generatePPSpace(head, SPACES4); + let titleText = FeedUtils.strings.formatStringFromName( + "subscribe-OPMLExportFileDialogTitle", + [this.brandShortName, rootFolder.prettyName], 2); + let title = opmlDoc.createElement("title"); + title.appendChild(opmlDoc.createTextNode(titleText)); + head.appendChild(title); + this.generatePPSpace(head, SPACES4); + let dt = opmlDoc.createElement("dateCreated"); + dt.appendChild(opmlDoc.createTextNode((new Date()).toUTCString())); + head.appendChild(dt); + this.generatePPSpace(head, SPACES2); + opmlRoot.appendChild(head); + + this.generatePPSpace(opmlRoot, SPACES2); + + // Add <outline>s to the <body>. + let body = opmlDoc.createElement("body"); + if (exportAsList) + this.generateOutlineList(rootFolder, body, SPACES4.length + 2); + else + this.generateOutlineStruct(rootFolder, body, SPACES4.length); + + this.generatePPSpace(body, SPACES2); + + if (!body.childElementCount) + // No folders/feeds. + return; + + opmlRoot.appendChild(body); + this.generatePPSpace(opmlRoot, ""); + + let serializer = new XMLSerializer(); + + if (FeedUtils.log.level <= Log4Moz.Level.Debug) + FeedUtils.log.debug("exportOPML: opmlDoc -\n" + + serializer.serializeToString(opmlDoc) + "\n"); + + // Get file to save from filepicker. + let saveAsFile = this.opmlPickSaveAsFile(exportAsList); + if (!saveAsFile) + return; + + let fos = FileUtils.openSafeFileOutputStream(saveAsFile); + serializer.serializeToStream(opmlDoc, fos, "utf-8"); + FileUtils.closeSafeFileOutputStream(fos); + + let statusReport = FeedUtils.strings.formatStringFromName( + "subscribe-OPMLExportDone", [saveAsFile.path], 1); + this.updateStatusItem("statusText", statusReport); + } + }, + + generatePPSpace: function(aNode, indentString) + { + aNode.appendChild(aNode.ownerDocument.createTextNode("\n")); + aNode.appendChild(aNode.ownerDocument.createTextNode(indentString)); + }, + + generateOutlineList: function(baseFolder, parent, indentLevel) + { + // Pretty printing. + let indentString = " ".repeat(indentLevel - 2); + + let feedOutline; + let folderEnumerator = baseFolder.subFolders; + while (folderEnumerator.hasMoreElements()) + { + let folder = folderEnumerator.getNext().QueryInterface(Ci.nsIMsgFolder); + FeedUtils.log.debug("generateOutlineList: folder - " + + folder.filePath.path); + if (!(folder instanceof Ci.nsIMsgFolder) || + folder.getFlag(Ci.nsMsgFolderFlags.Trash) || + folder.getFlag(Ci.nsMsgFolderFlags.Virtual)) + continue; + + FeedUtils.log.debug("generateOutlineList: CONTINUE folderName - " + + folder.name); + + if (folder.hasSubFolders) + { + FeedUtils.log.debug("generateOutlineList: has subfolders - " + + folder.name); + // Recurse. + this.generateOutlineList(folder, parent, indentLevel); + } + + // Add outline elements with xmlUrls. + let feeds = this.getFeedsInFolder(folder); + for (let feed of feeds) + { + FeedUtils.log.debug("generateOutlineList: folder has FEED url - " + + folder.name + " : " + feed.url); + feedOutline = this.exportOPMLOutline(feed, parent.ownerDocument); + this.generatePPSpace(parent, indentString); + parent.appendChild(feedOutline); + } + } + }, + + generateOutlineStruct: function(baseFolder, parent, indentLevel) + { + // Pretty printing. + function indentString(len) { return " ".repeat(len - 2); }; + + let folderOutline, feedOutline; + let folderEnumerator = baseFolder.subFolders; + while (folderEnumerator.hasMoreElements()) + { + let folder = folderEnumerator.getNext().QueryInterface(Ci.nsIMsgFolder); + FeedUtils.log.debug("generateOutlineStruct: folder - " + + folder.filePath.path); + if (!(folder instanceof Ci.nsIMsgFolder) || + folder.getFlag(Ci.nsMsgFolderFlags.Trash) || + folder.getFlag(Ci.nsMsgFolderFlags.Virtual)) + continue; + + FeedUtils.log.debug("generateOutlineStruct: CONTINUE folderName - " + + folder.name); + + // Make a folder outline element. + folderOutline = parent.ownerDocument.createElement("outline"); + folderOutline.setAttribute("title", folder.prettyName); + this.generatePPSpace(parent, indentString(indentLevel + 2)); + + if (folder.hasSubFolders) + { + FeedUtils.log.debug("generateOutlineStruct: has subfolders - " + + folder.name); + // Recurse. + this.generateOutlineStruct(folder, folderOutline, indentLevel + 2); + } + + let feeds = this.getFeedsInFolder(folder); + for (let feed of feeds) + { + // Add feed outline elements with xmlUrls. + FeedUtils.log.debug("generateOutlineStruct: folder has FEED url - "+ + folder.name + " : " + feed.url); + feedOutline = this.exportOPMLOutline(feed, parent.ownerDocument); + this.generatePPSpace(folderOutline, indentString(indentLevel + 4)); + folderOutline.appendChild(feedOutline); + } + + parent.appendChild(folderOutline); + } + }, + + exportOPMLOutline: function(aFeed, aDoc) + { + let outRv = aDoc.createElement("outline"); + outRv.setAttribute("type", "rss"); + outRv.setAttribute("title", aFeed.title); + outRv.setAttribute("text", aFeed.title); + outRv.setAttribute("version", "RSS"); + outRv.setAttribute("fz:quickMode", aFeed.quickMode); + outRv.setAttribute("fz:options", JSON.stringify(aFeed.options)); + outRv.setAttribute("xmlUrl", aFeed.url); + outRv.setAttribute("htmlUrl", aFeed.link); + return outRv; + }, + + importOPML: function() + { + // Account folder must be selected in subscribe dialog. + let item = this.mView ? this.mView.currentItem : null; + if (!item || !item.folder || !item.folder.isServer) + return; + + let server = item.folder.server; + // Get file to open from filepicker. + let openFile = this.opmlPickOpenFile(); + if (!openFile) + return; + + this.mActionMode = this.kImportingOPML; + this.updateButtons(null); + this.selectFolder(item.folder, { select: false, open: true }); + let statusReport = FeedUtils.strings.GetStringFromName("subscribe-loading"); + this.updateStatusItem("statusText", statusReport); + // If there were a getElementsByAttribute in html, we could go determined... + this.updateStatusItem("progressMeter", "?"); + + if (!this.importOPMLFile(openFile, server, this.importOPMLFinished)) { + this.mActionMode = null; + this.updateButtons(item); + this.clearStatusInfo(); + } + }, + +/** + * Import opml file into a feed account. Used by the Subscribe dialog and + * the Import wizard. + * + * @param nsILocalFile aFile - the opml file. + * @param nsIMsgIncomingServer aServer - the account server. + * @param func aCallback - callback function. + * + * @return bool - false if error. + */ + importOPMLFile: function(aFile, aServer, aCallback) + { + if (aServer && (aServer instanceof Ci.nsIMsgIncomingServer)) + this.mRSSServer = aServer; + + if (!aFile || !this.mRSSServer || !aCallback) + return false; + + let opmlDom, statusReport; + let stream = Cc["@mozilla.org/network/file-input-stream;1"]. + createInstance(Ci.nsIFileInputStream); + + // Read in file as raw bytes, so Expat can do the decoding for us. + try { + stream.init(aFile, FileUtils.MODE_RDONLY, FileUtils.PERMS_FILE, 0); + let parser = new DOMParser(); + opmlDom = parser.parseFromStream(stream, null, stream.available(), + "application/xml"); + } + catch(e) { + statusReport = FeedUtils.strings.GetStringFromName( + "subscribe-errorOpeningFile"); + Services.prompt.alert(window, null, statusReport); + return false; + } + finally { + stream.close(); + } + + let body = opmlDom ? opmlDom.querySelector("body") : null; + + // Return if the OPML file is invalid or empty. + if (!body || !body.childElementCount || + opmlDom.documentElement.tagName != "opml") + { + statusReport = FeedUtils.strings.formatStringFromName( + "subscribe-OPMLImportInvalidFile", [aFile.leafName], 1); + Services.prompt.alert(window, null, statusReport); + return false; + } + + this.importOPMLOutlines(body, this.mRSSServer, aCallback); + return true; + }, + + importOPMLOutlines: function(aBody, aRSSServer, aCallback) + { + let win = this; + let rssServer = aRSSServer; + let callback = aCallback; + let outline, feedFolder; + let badTag = false; + let firstFeedInFolderQuickMode = null; + let lastFolder; + let feedsAdded = 0; + let rssOutlines = 0; + let folderOutlines = 0; + + function processor(aParentNode, aParentFolder) + { + FeedUtils.log.trace("importOPMLOutlines: PROCESSOR tag:name:childs - " + + aParentNode.tagName + ":" + + aParentNode.getAttribute("text") + ":" + + aParentNode.childElementCount); + while (true) + { + if (aParentNode.tagName == "body" && !aParentNode.childElementCount) + { + // Finished. + let statusReport = win.importOPMLStatus(feedsAdded, rssOutlines); + callback(statusReport, lastFolder, win); + return; + } + + outline = aParentNode.firstElementChild; + if (outline.tagName != "outline") + { + FeedUtils.log.info("importOPMLOutlines: skipping, node is not an " + + "<outline> - <" + outline.tagName + ">"); + badTag = true; + break; + } + + let outlineName = outline.getAttribute("text") || + outline.getAttribute("title") || + outline.getAttribute("xmlUrl"); + let feedUrl, folderURI; + + if (outline.getAttribute("type") == "rss") + { + // A feed outline. + feedUrl = outline.getAttribute("xmlUrl") || outline.getAttribute("url"); + if (!feedUrl) + { + FeedUtils.log.info("importOPMLOutlines: skipping, type=rss <outline> " + + "has no url - " + outlineName); + break; + } + + rssOutlines++; + feedFolder = aParentFolder; + + if (FeedUtils.feedAlreadyExists(feedUrl, rssServer)) + { + FeedUtils.log.info("importOPMLOutlines: feed already subscribed in account " + + rssServer.prettyName + ", url - " + feedUrl); + break; + } + + if (aParentNode.tagName == "outline" && + aParentNode.getAttribute("type") != "rss") + // Parent is a folder, already created. + folderURI = feedFolder.URI; + else + { + // Parent is not a folder outline, likely the <body> in a flat list. + // Create feed's folder with feed's name and account rootFolder as + // parent of feed's folder. + // NOTE: Assume a type=rss outline must be a leaf and is not a + // direct parent of another type=rss outline; such a structure + // may lead to unintended nesting and inaccurate counts. + } + + // Create the feed. + let quickMode = outline.hasAttribute("fz:quickMode") ? + outline.getAttribute("fz:quickMode") == "true" : + rssServer.getBoolValue("quickMode"); + let options = outline.getAttribute("fz:options"); + options = options ? JSON.parse(options) : null; + + if (firstFeedInFolderQuickMode === null) + // The summary/web page pref applies to all feeds in a folder, + // though it is a property of an individual feed. This can be + // set (and is obvious) in the subscribe dialog; ensure import + // doesn't leave mismatches if mismatched in the opml file. + firstFeedInFolderQuickMode = quickMode; + else + quickMode = firstFeedInFolderQuickMode; + + let feedProperties = { feedName : outlineName, + feedLocation : feedUrl, + server : rssServer, + folderURI : folderURI, + quickMode : quickMode, + options : options }; + + FeedUtils.log.info("importOPMLOutlines: importing feed: name, url - "+ + outlineName + ", " + feedUrl); + + let feed = win.storeFeed(feedProperties); + if (outline.hasAttribute("htmlUrl")) + feed.link = outline.getAttribute("htmlUrl"); + + feed.createFolder(); + if (!feed.folder) + { + // Non success. Remove intermediate traces from the feeds database. + if (feed && feed.url && feed.server) + FeedUtils.deleteFeed(FeedUtils.rdf.GetResource(feed.url), + feed.server, + feed.server.rootFolder); + FeedUtils.log.info("importOPMLOutlines: skipping, error creating folder - '" + + feed.folderName + "' from outlineName - '" + + outlineName + "' in parent folder " + + aParentFolder.filePath.path); + badTag = true; + break; + } + + // Add the feed to the databases. + FeedUtils.addFeed(feed); + // Feed correctly added. + feedsAdded++; + lastFolder = feed.folder; + } + else + { + // A folder outline. If a folder exists in the account structure at + // the same level as in the opml structure, feeds are placed into the + // existing folder. + let defaultName = FeedUtils.strings.GetStringFromName("ImportFeedsNew"); + let folderName = FeedUtils.getSanitizedFolderName(aParentFolder, + outlineName, + defaultName, + false); + try { + feedFolder = aParentFolder.getChildNamed(folderName); + } + catch (ex) { + // Folder not found, create it. + FeedUtils.log.info("importOPMLOutlines: creating folder - '" + + folderName + "' from outlineName - '" + + outlineName + "' in parent folder " + + aParentFolder.filePath.path); + firstFeedInFolderQuickMode = null; + try { + feedFolder = aParentFolder.QueryInterface(Ci.nsIMsgLocalMailFolder). + createLocalSubfolder(folderName); + folderOutlines++; + } + catch (ex) { + // An error creating. Skip it. + FeedUtils.log.info("importOPMLOutlines: skipping, error creating folder - '" + + folderName + "' from outlineName - '" + + outlineName + "' in parent folder " + + aParentFolder.filePath.path); + let xfolder = aParentFolder.getChildNamed(folderName); + aParentFolder.propagateDelete(xfolder, true, null); + badTag = true; + break; + } + } + } + break; + } + + if (!outline.childElementCount || badTag) + { + // Remove leaf nodes that are processed or bad tags from the opml dom, + // and go back to reparse. This method lets us use setTimeout to + // prevent UI hang, in situations of both deep and shallow trees. + // A yield/generator.next() method is fine for shallow trees, but not + // the true recursion required for deeper trees; both the shallow loop + // and the recurse should give it up. + outline.remove(); + badTag = false; + outline = aBody; + feedFolder = rssServer.rootFolder; + } + + setTimeout(function() { + processor(outline, feedFolder); + }, 0); + } + + processor(aBody, rssServer.rootFolder); + }, + + importOPMLStatus: function(aFeedsAdded, aRssOutlines, aFolderOutlines) + { + let statusReport; + if (aRssOutlines > aFeedsAdded) + statusReport = FeedUtils.strings.formatStringFromName("subscribe-OPMLImportStatus", + [PluralForm.get(aFeedsAdded, + FeedUtils.strings.GetStringFromName("subscribe-OPMLImportUniqueFeeds")) + .replace("#1", aFeedsAdded), + PluralForm.get(aRssOutlines, + FeedUtils.strings.GetStringFromName("subscribe-OPMLImportFoundFeeds")) + .replace("#1", aRssOutlines)], 2); + else + statusReport = PluralForm.get(aFeedsAdded, + FeedUtils.strings.GetStringFromName("subscribe-OPMLImportFeedCount")) + .replace("#1", aFeedsAdded); + + return statusReport; + }, + + importOPMLFinished: function(aStatusReport, aLastFolder, aWin) + { + if (aLastFolder) + { + aWin.selectFolder(aLastFolder, { select: false, newFolder: aLastFolder }); + aWin.selectFolder(aLastFolder.parent); + } + aWin.mActionMode = null; + aWin.updateButtons(aWin.mView.currentItem); + aWin.clearStatusInfo(); + aWin.updateStatusItem("statusText", aStatusReport); + } + +}; diff --git a/mailnews/extensions/newsblog/content/feed-subscriptions.xul b/mailnews/extensions/newsblog/content/feed-subscriptions.xul new file mode 100644 index 000000000..d6f4ea18f --- /dev/null +++ b/mailnews/extensions/newsblog/content/feed-subscriptions.xul @@ -0,0 +1,235 @@ +<?xml version="1.0"?> +<!-- -*- Mode: Java; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + - This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> + +<?xml-stylesheet href="chrome://messenger/skin/" type="text/css"?> +<?xml-stylesheet href="chrome://messenger/skin/folderPane.css" type="text/css"?> +<?xml-stylesheet href="chrome://messenger/skin/folderMenus.css" type="text/css"?> +<?xml-stylesheet href="chrome://messenger-newsblog/skin/feed-subscriptions.css" type="text/css"?> + +<!DOCTYPE window [ + <!ENTITY % feedDTD SYSTEM "chrome://messenger-newsblog/locale/feed-subscriptions.dtd"> + %feedDTD; + <!ENTITY % certDTD SYSTEM "chrome://pippki/locale/certManager.dtd"> + %certDTD; +]> + +<window id="subscriptionsDialog" + flex="1" + title="&feedSubscriptions.label;" + windowtype="Mail:News-BlogSubscriptions" + xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul" + xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns:nc="http://home.netscape.com/NC-rdf#" + persist="width height screenX screenY sizemode" + onload="FeedSubscriptions.onLoad();" + onclose="return FeedSubscriptions.onClose();" + onkeypress="FeedSubscriptions.onKeyPress(event);" + onmousedown="FeedSubscriptions.onMouseDown(event);"> + + <script type="application/javascript" + src="chrome://messenger/content/specialTabs.js"/> + <script type="application/javascript" + src="chrome://messenger-newsblog/content/feed-subscriptions.js"/> + + <keyset id="extensionsKeys"> + <key id="key_close" + key="&cmd.close.commandKey;" + modifiers="accel" + oncommand="window.close();"/> + <key id="key_close2" + keycode="VK_ESCAPE" + oncommand="window.close();"/> + </keyset> + + <stringbundle id="bundle_newsblog" + src="chrome://messenger-newsblog/locale/newsblog.properties"/> + <stringbundle id="bundle_brand" + src="chrome://branding/locale/brand.properties"/> + + <vbox flex="1" id="contentPane"> + <hbox align="right"> + <label id="learnMore" + class="text-link" + crop="end" + value="&learnMore.label;" + href="https://support.mozilla.org/kb/how-subscribe-news-feeds-and-blogs"/> + </hbox> + + <tree id="rssSubscriptionsList" + treelines="true" + flex="1" + hidecolumnpicker="true" + onselect="FeedSubscriptions.onSelect();" + seltype="single"> + <treecols> + <treecol id="folderNameCol" + flex="2" + primary="true" + hideheader="true"/> + </treecols> + <treechildren id="subscriptionChildren" + ondragstart="FeedSubscriptions.onDragStart(event);" + ondragover="FeedSubscriptions.onDragOver(event);"/> + </tree> + + <hbox id="rssFeedInfoBox"> + <vbox flex="1"> + <grid flex="1"> + <columns> + <column/> + <column flex="1"/> + </columns> + <rows> + <row> + <hbox align="right" valign="middle"> + <label id="nameLabel" + accesskey="&feedTitle.accesskey;" + control="nameValue" + value="&feedTitle.label;"/> + </hbox> + <textbox id="nameValue" + clickSelectsAll="true"/> + </row> + <row> + <hbox align="right" valign="middle"> + <label id="locationLabel" + accesskey="&feedLocation.accesskey;" + control="locationValue" + value="&feedLocation.label;"/> + </hbox> + <hbox> + <textbox id="locationValue" + flex="1" + class="uri-element" + placeholder="&feedLocation.placeholder;" + clickSelectsAll="true" + onfocus="FeedSubscriptions.setSummaryFocus();" + onblur="FeedSubscriptions.setSummaryFocus();"/> + <hbox align="center"> + <label id="locationValidate" + collapsed="true" + class="text-link" + crop="end" + value="&locationValidate.label;" + onclick="FeedSubscriptions.checkValidation(event);"/> + </hbox> + </hbox> + </row> + <row> + <hbox align="right" valign="middle"> + <label id="feedFolderLabel" + value="&feedFolder.label;" + accesskey="&feedFolder.accesskey;" + control="selectFolder"/> + </hbox> + <hbox> + <menulist id="selectFolder" + flex="1" + class="folderMenuItem" + hidden="true"> + <menupopup id="selectFolderPopup" + class="menulist-menupopup" + type="folder" + mode="feeds" + showFileHereLabel="true" + showAccountsFileHere="true" + oncommand="FeedSubscriptions.setNewFolder(event)"/> + </menulist> + <textbox id="selectFolderValue" + flex="1" + readonly="true" + onkeypress="FeedSubscriptions.onClickSelectFolderValue(event)" + onclick="FeedSubscriptions.onClickSelectFolderValue(event)"/> + </hbox> + </row> + </rows> + </grid> + <checkbox id="quickMode" + accesskey="&quickMode.accesskey;" + label="&quickMode.label;" + oncommand="FeedSubscriptions.setSummary(this.checked)"/> + <checkbox id="autotagEnable" + accesskey="&autotagEnable.accesskey;" + label="&autotagEnable.label;" + oncommand="FeedSubscriptions.setCategoryPrefs(this)"/> + <hbox> + <checkbox id="autotagUsePrefix" + class="indent" + accesskey="&autotagUsePrefix.accesskey;" + label="&autotagUsePrefix.label;" + oncommand="FeedSubscriptions.setCategoryPrefs(this)"/> + <textbox id="autotagPrefix" + placeholder="&autoTagPrefix.placeholder;" + clickSelectsAll="true"/> + </hbox> + <separator class="thin"/> + </vbox> + </hbox> + + <hbox id="statusContainerBox" + align="center" + valign="middle"> + <vbox flex="1"> + <description id="statusText"/> + </vbox> + <spacer flex="1"/> + <label id="validationText" + collapsed="true" + class="text-link" + crop="end" + value="&validateText.label;" + onclick="FeedSubscriptions.checkValidation(event);"/> + <button id="addCertException" + collapsed="true" + label="&certmgr.addException.label;" + accesskey="&certmgr.addException.accesskey;" + oncommand="FeedSubscriptions.addCertExceptionDialog();"/> + <progressmeter id="progressMeter" + collapsed="true" + mode="determined" + value="0"/> + </hbox> + + <hbox align="end"> + <hbox class="actionButtons" flex="1"> + <button id="addFeed" + label="&button.addFeed.label;" + accesskey="&button.addFeed.accesskey;" + oncommand="FeedSubscriptions.addFeed();"/> + + <button id="editFeed" + disabled="true" + label="&button.updateFeed.label;" + accesskey="&button.updateFeed.accesskey;" + oncommand="FeedSubscriptions.editFeed();"/> + + <button id="removeFeed" + disabled="true" + label="&button.removeFeed.label;" + accesskey="&button.removeFeed.accesskey;" + oncommand="FeedSubscriptions.removeFeed(true);"/> + + <button id="importOPML" + label="&button.importOPML.label;" + accesskey="&button.importOPML.accesskey;" + oncommand="FeedSubscriptions.importOPML();"/> + + <button id="exportOPML" + label="&button.exportOPML.label;" + accesskey="&button.exportOPML.accesskey;" + tooltiptext="&button.exportOPML.tooltip;" + oncommand="FeedSubscriptions.exportOPML(event);"/> + + <spacer flex="1"/> + + <button id="close" + label="&button.close.label;" + icon="close" + oncommand="if (FeedSubscriptions.onClose()) window.close();"/> + </hbox> + </hbox> + </vbox> +</window> diff --git a/mailnews/extensions/newsblog/content/feedAccountWizard.js b/mailnews/extensions/newsblog/content/feedAccountWizard.js new file mode 100644 index 000000000..a79da073a --- /dev/null +++ b/mailnews/extensions/newsblog/content/feedAccountWizard.js @@ -0,0 +1,45 @@ +/* -*- Mode: Java; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+Components.utils.import("resource:///modules/FeedUtils.jsm");
+
+/* Feed account standalone wizard functions */
+var FeedAccountWizard = {
+ accountName: "",
+
+ accountSetupPageInit: function() {
+ this.accountSetupPageValidate();
+ },
+
+ accountSetupPageValidate: function() {
+ this.accountName = document.getElementById("prettyName").value.trim();
+ document.documentElement.canAdvance = this.accountName;
+ },
+
+ accountSetupPageUnload: function() {
+ return;
+ },
+
+ donePageInit: function() {
+ document.getElementById("account.name.text").value = this.accountName;
+ },
+
+ onCancel: function() {
+ return true;
+ },
+
+ onFinish: function() {
+ let account = FeedUtils.createRssAccount(this.accountName);
+ if ("gFolderTreeView" in window.opener.top)
+ // Opened from 3pane File->New or Appmenu New Message, or
+ // Account Central link.
+ window.opener.top.gFolderTreeView.selectFolder(account.incomingServer.rootMsgFolder);
+ else if ("selectServer" in window.opener)
+ // Opened from Account Settings.
+ window.opener.selectServer(account.incomingServer);
+
+ window.close();
+ }
+}
diff --git a/mailnews/extensions/newsblog/content/feedAccountWizard.xul b/mailnews/extensions/newsblog/content/feedAccountWizard.xul new file mode 100644 index 000000000..0535fb237 --- /dev/null +++ b/mailnews/extensions/newsblog/content/feedAccountWizard.xul @@ -0,0 +1,79 @@ +<?xml version="1.0"?>
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+ - License, v. 2.0. If a copy of the MPL was not distributed with this
+ - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+
+<?xml-stylesheet href="chrome://messenger/skin/accountWizard.css" type="text/css"?>
+
+<!DOCTYPE wizard [
+ <!ENTITY % accountDTD SYSTEM "chrome://messenger/locale/AccountWizard.dtd">
+ %accountDTD;
+ <!ENTITY % newsblogDTD SYSTEM "chrome://messenger-newsblog/locale/am-newsblog.dtd" >
+ %newsblogDTD;
+ <!ENTITY % imDTD SYSTEM "chrome://messenger/locale/imAccountWizard.dtd" >
+ %imDTD;
+]>
+
+<wizard id="FeedAccountWizard"
+ title="&feedWindowTitle.label;"
+ onwizardcancel="return FeedAccountWizard.onCancel();"
+ onwizardfinish="return FeedAccountWizard.onFinish();"
+ xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul">
+
+ <script type="application/javascript"
+ src="chrome://messenger-newsblog/content/feedAccountWizard.js"/>
+
+ <!-- Account setup page : User gets a choice to enter a name for the account -->
+ <!-- Defaults : Feed account name -> default string -->
+ <wizardpage id="accountsetuppage"
+ pageid="accountsetuppage"
+ label="&accnameTitle.label;"
+ onpageshow="return FeedAccountWizard.accountSetupPageInit();"
+ onpageadvanced="return FeedAccountWizard.accountSetupPageUnload();">
+ <vbox flex="1">
+ <description>&accnameDesc.label;</description>
+ <separator class="thin"/>
+ <hbox align="center">
+ <label class="label"
+ value="&accnameLabel.label;"
+ accesskey="&accnameLabel.accesskey;"
+ control="prettyName"/>
+ <textbox id="prettyName"
+ flex="1"
+ value="&feeds.accountName;"
+ oninput="FeedAccountWizard.accountSetupPageValidate();"/>
+ </hbox>
+ </vbox>
+ </wizardpage>
+
+ <!-- Done page : Summarizes information collected to create a feed account -->
+ <wizardpage id="done"
+ pageid="done"
+ label="&accountSummaryTitle.label;"
+ onpageshow="return FeedAccountWizard.donePageInit();">
+ <vbox flex="1">
+ <description>&accountSummaryInfo.label;</description>
+ <separator class="thin"/>
+ <grid>
+ <columns>
+ <column/>
+ <column flex="1"/>
+ </columns>
+ <rows>
+ <row id="account.name"
+ align="center">
+ <label id="account.name.label"
+ class="label"
+ flex="1"
+ value="&accnameLabel.label;"/>
+ <label id="account.name.text"
+ class="label"/>
+ </row>
+ </rows>
+ </grid>
+ <separator/>
+ <spacer flex="1"/>
+ </vbox>
+ </wizardpage>
+
+</wizard>
diff --git a/mailnews/extensions/newsblog/content/newsblogOverlay.js b/mailnews/extensions/newsblog/content/newsblogOverlay.js new file mode 100644 index 000000000..f7e08ec95 --- /dev/null +++ b/mailnews/extensions/newsblog/content/newsblogOverlay.js @@ -0,0 +1,363 @@ +/* -*- Mode: JavaScript; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +Components.utils.import("resource:///modules/gloda/mimemsg.js"); +Components.utils.import("resource:///modules/mailServices.js"); +Components.utils.import("resource://gre/modules/Services.jsm"); + +// This global is for SeaMonkey compatibility. +var gShowFeedSummary; + +var FeedMessageHandler = { + gShowSummary: true, + gToggle: false, + kSelectOverrideWebPage: 0, + kSelectOverrideSummary: 1, + kSelectFeedDefault: 2, + kOpenWebPage: 0, + kOpenSummary: 1, + kOpenToggleInMessagePane: 2, + kOpenLoadInBrowser: 3, + + /** + * How to load message on threadpane select. + */ + get onSelectPref() { + return Services.prefs.getIntPref("rss.show.summary"); + }, + + set onSelectPref(val) { + Services.prefs.setIntPref("rss.show.summary", val); + ReloadMessage(); + }, + + /** + * Load web page on threadpane select. + */ + get loadWebPageOnSelectPref() { + return Services.prefs.getIntPref("rss.message.loadWebPageOnSelect") ? true : false; + }, + + /** + * How to load message on open (enter/dbl click in threadpane, contextmenu). + */ + get onOpenPref() { + return Services.prefs.getIntPref("rss.show.content-base"); + }, + + set onOpenPref(val) { + Services.prefs.setIntPref("rss.show.content-base", val); + }, + + /** + * Determine if a message is a feed message. Prior to Tb15, a message had to + * be in an rss acount type folder. In Tb15 and later, a flag is set on the + * message itself upon initial store; the message can be moved to any folder. + * + * @param nsIMsgDBHdr aMsgHdr - the message. + * + * @return true if message is a feed, false if not. + */ + isFeedMessage: function(aMsgHdr) { + return (aMsgHdr instanceof Components.interfaces.nsIMsgDBHdr) && + ((aMsgHdr.flags & Components.interfaces.nsMsgMessageFlags.FeedMsg) || + (aMsgHdr.folder && aMsgHdr.folder.server.type == "rss")); + }, + + /** + * Determine whether to show a feed message summary or load a web page in the + * message pane. + * + * @param nsIMsgDBHdr aMsgHdr - the message. + * @param bool aToggle - true if in toggle mode, false otherwise. + * + * @return true if summary is to be displayed, false if web page. + */ + shouldShowSummary: function(aMsgHdr, aToggle) { + // Not a feed message, always show summary (the message). + if (!this.isFeedMessage(aMsgHdr)) + return true; + + // Notified of a summary reload when toggling, reset toggle and return. + if (!aToggle && this.gToggle) + return !(this.gToggle = false); + + let showSummary = true; + this.gToggle = aToggle; + + // Thunderbird 2 rss messages with 'Show article summary' not selected, + // ie message body constructed to show web page in an iframe, can't show + // a summary - notify user. + let browser = getBrowser(); + let contentDoc = browser ? browser.contentDocument : null; + let rssIframe = contentDoc ? contentDoc.getElementById("_mailrssiframe") : null; + if (rssIframe) { + if (this.gToggle || this.onSelectPref == this.kSelectOverrideSummary) + this.gToggle = false; + return false; + } + + if (aToggle) + // Toggle mode, flip value. + return gShowFeedSummary = this.gShowSummary = !this.gShowSummary; + + let wintype = document.documentElement.getAttribute("windowtype"); + let tabMail = document.getElementById("tabmail"); + let messageTab = tabMail && tabMail.currentTabInfo.mode.type == "message"; + let messageWindow = wintype == "mail:messageWindow"; + + switch (this.onSelectPref) { + case this.kSelectOverrideWebPage: + showSummary = false; + break; + case this.kSelectOverrideSummary: + showSummary = true + break; + case this.kSelectFeedDefault: + // Get quickmode per feed folder pref from feeds.rdf. If the feed + // message is not in a feed account folder (hence the folder is not in + // the feeds database), or FZ_QUICKMODE property is not found (possible + // in pre renovation urls), err on the side of showing the summary. + // For the former, toggle or global override is necessary; for the + // latter, a show summary checkbox toggle in Subscribe dialog will set + // one on the path to bliss. + let folder = aMsgHdr.folder, targetRes; + try { + targetRes = FeedUtils.getParentTargetForChildResource( + folder.URI, FeedUtils.FZ_QUICKMODE, folder.server); + } + catch (ex) { + // Not in a feed account folder or other error. + FeedUtils.log.info("FeedMessageHandler.shouldShowSummary: could not " + + "get summary pref for this folder"); + } + + showSummary = targetRes && targetRes.QueryInterface(Ci.nsIRDFLiteral). + Value == "false" ? false : true; + break; + } + + gShowFeedSummary = this.gShowSummary = showSummary; + + if (messageWindow || messageTab) { + // Message opened in either standalone window or tab, due to either + // message open pref (we are here only if the pref is 0 or 1) or + // contextmenu open. + switch (this.onOpenPref) { + case this.kOpenToggleInMessagePane: + // Opened by contextmenu, use the value derived above. + // XXX: allow a toggle via crtl? + break; + case this.kOpenWebPage: + showSummary = false; + break; + case this.kOpenSummary: + showSummary = true; + break; + } + } + + // Auto load web page in browser on select, per pref; shouldShowSummary() is + // always called first to 1)test if feed, 2)get summary pref, so do it here. + if (this.loadWebPageOnSelectPref) + setTimeout(FeedMessageHandler.loadWebPage, 20, aMsgHdr, {browser:true}); + + return showSummary; + }, + + /** + * Load a web page for feed messages. Use MsgHdrToMimeMessage() to get + * the content-base url from the message headers. We cannot rely on + * currentHeaderData; it has not yet been streamed at our entry point in + * displayMessageChanged(), and in the case of a collapsed message pane it + * is not streamed. + * + * @param nsIMsgDBHdr aMessageHdr - the message. + * @param {obj} aWhere - name value=true pair, where name is in: + * 'messagepane', 'browser', 'tab', 'window'. + */ + loadWebPage: function(aMessageHdr, aWhere) { + MsgHdrToMimeMessage(aMessageHdr, null, function(aMsgHdr, aMimeMsg) { + if (aMimeMsg && aMimeMsg.headers["content-base"] && + aMimeMsg.headers["content-base"][0]) { + let url = aMimeMsg.headers["content-base"], uri; + try { + let converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"] + .createInstance(Ci.nsIScriptableUnicodeConverter); + converter.charset = "UTF-8"; + url = converter.ConvertToUnicode(url); + uri = Services.io.newURI(url, null, null); + url = uri.spec; + } + catch (ex) { + FeedUtils.log.info("FeedMessageHandler.loadWebPage: " + + "invalid Content-Base header url - " + url); + return; + } + if (aWhere.browser) + Components.classes["@mozilla.org/uriloader/external-protocol-service;1"] + .getService(Components.interfaces.nsIExternalProtocolService) + .loadURI(uri); + else if (aWhere.messagepane) { + let loadFlag = getBrowser().webNavigation.LOAD_FLAGS_NONE; + getBrowser().webNavigation.loadURI(url, loadFlag, null, null, null); + } + else if (aWhere.tab) + openContentTab(url, "tab", "^"); + else if (aWhere.window) + openContentTab(url, "window", "^"); + } + else + FeedUtils.log.info("FeedMessageHandler.loadWebPage: could not get " + + "Content-Base header url for this message"); + }); + }, + + /** + * Display summary or load web page for feed messages. Caller should already + * know if the message is a feed message. + * + * @param nsIMsgDBHdr aMsgHdr - the message. + * @param bool aShowSummary - true if summary is to be displayed, false if + * web page. + */ + setContent: function(aMsgHdr, aShowSummary) { + if (aShowSummary) { + // Only here if toggling to summary in 3pane. + if (this.gToggle && gDBView && GetNumSelectedMessages() == 1) + ReloadMessage(); + } + else { + let browser = getBrowser(); + if (browser && browser.contentDocument && browser.contentDocument.body) + browser.contentDocument.body.hidden = true; + // If in a non rss folder, hide possible remote content bar on a web + // page load, as it doesn't apply. + if ("msgNotificationBar" in window) + gMessageNotificationBar.clearMsgNotifications(); + + this.loadWebPage(aMsgHdr, {messagepane:true}); + this.gToggle = false; + } + } +} + +function openSubscriptionsDialog(aFolder) +{ + // Check for an existing feed subscriptions window and focus it. + let subscriptionsWindow = + Services.wm.getMostRecentWindow("Mail:News-BlogSubscriptions"); + + if (subscriptionsWindow) + { + if (aFolder) + { + subscriptionsWindow.FeedSubscriptions.selectFolder(aFolder); + subscriptionsWindow.FeedSubscriptions.mView.treeBox.ensureRowIsVisible( + subscriptionsWindow.FeedSubscriptions.mView.selection.currentIndex); + } + + subscriptionsWindow.focus(); + } + else + { + window.openDialog("chrome://messenger-newsblog/content/feed-subscriptions.xul", + "", "centerscreen,chrome,dialog=no,resizable", + { folder: aFolder}); + } +} + +// Special case attempts to reply/forward/edit as new RSS articles. For +// messages stored prior to Tb15, we are here only if the message's folder's +// account server is rss and feed messages moved to other types will have their +// summaries loaded, as viewing web pages only happened in an rss account. +// The user may choose whether to load a summary or web page link by ensuring +// the current feed message is being viewed as either a summary or web page. +function openComposeWindowForRSSArticle(aMsgComposeWindow, aMsgHdr, aMessageUri, + aType, aFormat, aIdentity, aMsgWindow) +{ + // Ensure right content is handled for web pages in window/tab. + let tabmail = document.getElementById("tabmail"); + let is3pane = tabmail && tabmail.selectedTab && tabmail.selectedTab.mode ? + tabmail.selectedTab.mode.type == "folder" : false; + let showingwebpage = ("FeedMessageHandler" in window) && !is3pane && + FeedMessageHandler.onOpenPref == FeedMessageHandler.kOpenWebPage; + + if (gShowFeedSummary && !showingwebpage) + { + // The user is viewing the summary. + MailServices.compose.OpenComposeWindow(aMsgComposeWindow, aMsgHdr, aMessageUri, + aType, aFormat, aIdentity, aMsgWindow); + + } + else + { + // Set up the compose message and get the feed message's web page link. + let Cc = Components.classes; + let Ci = Components.interfaces; + let msgHdr = aMsgHdr; + let type = aType; + let msgComposeType = Ci.nsIMsgCompType; + let subject = msgHdr.mime2DecodedSubject; + let fwdPrefix = Services.prefs.getCharPref("mail.forward_subject_prefix"); + fwdPrefix = fwdPrefix ? fwdPrefix + ": " : ""; + + let params = Cc["@mozilla.org/messengercompose/composeparams;1"] + .createInstance(Ci.nsIMsgComposeParams); + + let composeFields = Cc["@mozilla.org/messengercompose/composefields;1"] + .createInstance(Ci.nsIMsgCompFields); + + if (type == msgComposeType.Reply || + type == msgComposeType.ReplyAll || + type == msgComposeType.ReplyToSender || + type == msgComposeType.ReplyToGroup || + type == msgComposeType.ReplyToSenderAndGroup) + { + subject = "Re: " + subject; + } + else if (type == msgComposeType.ForwardInline || + type == msgComposeType.ForwardAsAttachment) + { + subject = fwdPrefix + subject; + } + + params.composeFields = composeFields; + params.composeFields.subject = subject; + params.composeFields.characterSet = msgHdr.Charset; + params.composeFields.body = ""; + params.bodyIsLink = false; + params.identity = aIdentity; + + try + { + // The feed's web page url is stored in the Content-Base header. + MsgHdrToMimeMessage(msgHdr, null, function(aMsgHdr, aMimeMsg) { + if (aMimeMsg && aMimeMsg.headers["content-base"] && + aMimeMsg.headers["content-base"][0]) + { + let converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"] + .createInstance(Ci.nsIScriptableUnicodeConverter); + converter.charset = "UTF-8"; + let url = converter.ConvertToUnicode(aMimeMsg.headers["content-base"]); + params.composeFields.body = url; + params.bodyIsLink = true; + MailServices.compose.OpenComposeWindowWithParams(null, params); + } + else + // No content-base url, use the summary. + MailServices.compose.OpenComposeWindow(aMsgComposeWindow, aMsgHdr, aMessageUri, + aType, aFormat, aIdentity, aMsgWindow); + + }, false, {saneBodySize: true}); + } + catch (ex) + { + // Error getting header, use the summary. + MailServices.compose.OpenComposeWindow(aMsgComposeWindow, aMsgHdr, aMessageUri, + aType, aFormat, aIdentity, aMsgWindow); + } + } +} diff --git a/mailnews/extensions/newsblog/jar.mn b/mailnews/extensions/newsblog/jar.mn new file mode 100644 index 000000000..aa16a0100 --- /dev/null +++ b/mailnews/extensions/newsblog/jar.mn @@ -0,0 +1,16 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +newsblog.jar: +% content messenger-newsblog %content/messenger-newsblog/ + content/messenger-newsblog/newsblogOverlay.js (content/newsblogOverlay.js) + content/messenger-newsblog/Feed.js (content/Feed.js) + content/messenger-newsblog/FeedItem.js (content/FeedItem.js) + content/messenger-newsblog/feed-parser.js (content/feed-parser.js) +* content/messenger-newsblog/feed-subscriptions.js (content/feed-subscriptions.js) + content/messenger-newsblog/feed-subscriptions.xul (content/feed-subscriptions.xul) + content/messenger-newsblog/am-newsblog.js (content/am-newsblog.js) + content/messenger-newsblog/am-newsblog.xul (content/am-newsblog.xul) + content/messenger-newsblog/feedAccountWizard.js (content/feedAccountWizard.js) + content/messenger-newsblog/feedAccountWizard.xul (content/feedAccountWizard.xul) diff --git a/mailnews/extensions/newsblog/js/newsblog.js b/mailnews/extensions/newsblog/js/newsblog.js new file mode 100644 index 000000000..364038ee5 --- /dev/null +++ b/mailnews/extensions/newsblog/js/newsblog.js @@ -0,0 +1,99 @@ +/* -*- Mode: Java; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +var {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components; + +Cu.import("resource:///modules/FeedUtils.jsm"); +Cu.import("resource://gre/modules/XPCOMUtils.jsm"); + +var nsNewsBlogFeedDownloader = +{ + downloadFeed: function(aFolder, aUrlListener, aIsBiff, aMsgWindow) + { + FeedUtils.downloadFeed(aFolder, aUrlListener, aIsBiff, aMsgWindow); + }, + + subscribeToFeed: function(aUrl, aFolder, aMsgWindow) + { + FeedUtils.subscribeToFeed(aUrl, aFolder, aMsgWindow); + }, + + updateSubscriptionsDS: function(aFolder, aOrigFolder, aAction) + { + FeedUtils.updateSubscriptionsDS(aFolder, aOrigFolder, aAction); + }, + + QueryInterface: function(aIID) + { + if (aIID.equals(Ci.nsINewsBlogFeedDownloader) || + aIID.equals(Ci.nsISupports)) + return this; + + throw Cr.NS_ERROR_NO_INTERFACE; + } +} + +var nsNewsBlogAcctMgrExtension = +{ + name: "newsblog", + chromePackageName: "messenger-newsblog", + showPanel: function (server) + { + return false; + }, + QueryInterface: function(aIID) + { + if (aIID.equals(Ci.nsIMsgAccountManagerExtension) || + aIID.equals(Ci.nsISupports)) + return this; + + throw Cr.NS_ERROR_NO_INTERFACE; + } +} + +function FeedDownloader() {} + +FeedDownloader.prototype = +{ + classID: Components.ID("{5c124537-adca-4456-b2b5-641ab687d1f6}"), + _xpcom_factory: + { + createInstance: function (aOuter, aIID) + { + if (aOuter != null) + throw Cr.NS_ERROR_NO_AGGREGATION; + if (!aIID.equals(Ci.nsINewsBlogFeedDownloader) && + !aIID.equals(Ci.nsISupports)) + throw Cr.NS_ERROR_INVALID_ARG; + + // return the singleton + return nsNewsBlogFeedDownloader.QueryInterface(aIID); + } + } // factory +}; // feed downloader + +function AcctMgrExtension() {} + +AcctMgrExtension.prototype = +{ + classID: Components.ID("{E109C05F-D304-4ca5-8C44-6DE1BFAF1F74}"), + _xpcom_factory: + { + createInstance: function (aOuter, aIID) + { + if (aOuter != null) + throw Cr.NS_ERROR_NO_AGGREGATION; + if (!aIID.equals(Ci.nsIMsgAccountManagerExtension) && + !aIID.equals(Ci.nsISupports)) + throw Cr.NS_ERROR_INVALID_ARG; + + // return the singleton + return nsNewsBlogAcctMgrExtension.QueryInterface(aIID); + } + } // factory +}; // account manager extension + +var components = [FeedDownloader, AcctMgrExtension]; +var NSGetFactory = XPCOMUtils.generateNSGetFactory(components); diff --git a/mailnews/extensions/newsblog/js/newsblog.manifest b/mailnews/extensions/newsblog/js/newsblog.manifest new file mode 100644 index 000000000..5a24df5e7 --- /dev/null +++ b/mailnews/extensions/newsblog/js/newsblog.manifest @@ -0,0 +1,5 @@ +component {5c124537-adca-4456-b2b5-641ab687d1f6} newsblog.js +contract @mozilla.org/newsblog-feed-downloader;1 {5c124537-adca-4456-b2b5-641ab687d1f6} +component {E109C05F-D304-4ca5-8C44-6DE1BFAF1F74} newsblog.js +contract @mozilla.org/accountmanager/extension;1?name=newsblog {E109C05F-D304-4ca5-8C44-6DE1BFAF1F74} +category mailnews-accountmanager-extensions newsblog @mozilla.org/accountmanager/extension;1?name=newsblog diff --git a/mailnews/extensions/newsblog/moz.build b/mailnews/extensions/newsblog/moz.build new file mode 100644 index 000000000..367f60574 --- /dev/null +++ b/mailnews/extensions/newsblog/moz.build @@ -0,0 +1,18 @@ +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +EXTRA_COMPONENTS += [ + 'js/newsblog.js', + 'js/newsblog.manifest', +] + +EXTRA_JS_MODULES += [ + 'content/FeedUtils.jsm', +] +JAR_MANIFESTS += ['jar.mn'] + +FINAL_TARGET_FILES.isp += [ + 'rss.rdf', +] diff --git a/mailnews/extensions/newsblog/rss.rdf b/mailnews/extensions/newsblog/rss.rdf new file mode 100644 index 000000000..c7223c01b --- /dev/null +++ b/mailnews/extensions/newsblog/rss.rdf @@ -0,0 +1,43 @@ +<?xml version="1.0"?> +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> + +<!DOCTYPE RDF SYSTEM "chrome://messenger-newsblog/locale/am-newsblog.dtd"> +<RDF:RDF + xmlns:NC="http://home.netscape.com/NC-rdf#" + xmlns:RDF="http://www.w3.org/1999/02/22-rdf-syntax-ns#"> + <RDF:Description about="NC:ispinfo"> + <NC:providers> + <NC:nsIMsgAccount about="newsblog"> + + <!-- server info --> + <NC:incomingServer> + <NC:nsIMsgIncomingServer> + <NC:hostName>Feeds</NC:hostName> + <NC:type>rss</NC:type> + <NC:biffMinutes>100</NC:biffMinutes> + <NC:username>nobody</NC:username> + </NC:nsIMsgIncomingServer> + </NC:incomingServer> + + <!-- identity defaults --> + <NC:identity> + <NC:nsIMsgIdentity> + </NC:nsIMsgIdentity> + </NC:identity> + + <NC:wizardAutoGenerateUniqueHostname>true</NC:wizardAutoGenerateUniqueHostname> + <NC:wizardHideIncoming>true</NC:wizardHideIncoming> + <NC:wizardAccountName>&feeds.accountName;</NC:wizardAccountName> + <NC:wizardSkipPanels>identitypage,incomingpage,outgoingpage</NC:wizardSkipPanels> + <NC:wizardShortName>&feeds.wizardShortName;</NC:wizardShortName> + <NC:wizardLongName>&feeds.wizardLongName;</NC:wizardLongName> + <NC:wizardLongNameAccesskey>&feeds.wizardLongName.accesskey;</NC:wizardLongNameAccesskey> + <NC:wizardShow>true</NC:wizardShow> + <NC:emailProviderName>RSS</NC:emailProviderName> + <NC:showServerDetailsOnWizardSummary>false</NC:showServerDetailsOnWizardSummary> + </NC:nsIMsgAccount> + </NC:providers> + </RDF:Description> +</RDF:RDF> diff --git a/mailnews/extensions/offline-startup/js/offlineStartup.js b/mailnews/extensions/offline-startup/js/offlineStartup.js new file mode 100644 index 000000000..56584465d --- /dev/null +++ b/mailnews/extensions/offline-startup/js/offlineStartup.js @@ -0,0 +1,170 @@ +/* -*- Mode: Java; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +Components.utils.import("resource://gre/modules/Services.jsm"); +Components.utils.import("resource://gre/modules/XPCOMUtils.jsm"); + +var kDebug = false; +var kOfflineStartupPref = "offline.startup_state"; +var kRememberLastState = 0; +var kAskForOnlineState = 1; +var kAlwaysOnline = 2; +var kAlwaysOffline = 3; +var kAutomatic = 4; +var gStartingUp = true; +var gOfflineStartupMode; //0 = remember last state, 1 = ask me, 2 == online, 3 == offline, 4 = automatic + + +//////////////////////////////////////////////////////////////////////// +// +// nsOfflineStartup : nsIObserver +// +// Check if the user has set the pref to be prompted for +// online/offline startup mode. If so, prompt the user. Also, +// check if the user wants to remember their offline state +// the next time they start up. +// If the user shutdown offline, and is now starting up in online +// mode, we will set the boolean pref "mailnews.playback_offline" to true. +// +//////////////////////////////////////////////////////////////////////// + +var nsOfflineStartup = +{ + onProfileStartup: function() + { + debug("onProfileStartup"); + + if (gStartingUp) + { + gStartingUp = false; + // if checked, the "work offline" checkbox overrides + if (Services.io.offline && !Services.io.manageOfflineStatus) + { + debug("already offline!"); + return; + } + } + + var manageOfflineStatus = Services.prefs.getBoolPref("offline.autoDetect"); + gOfflineStartupMode = Services.prefs.getIntPref(kOfflineStartupPref); + let wasOffline = !Services.prefs.getBoolPref("network.online"); + + if (gOfflineStartupMode == kAutomatic) + { + // Offline state should be managed automatically + // so do nothing specific at startup. + } + else if (gOfflineStartupMode == kAlwaysOffline) + { + Services.io.manageOfflineStatus = false; + Services.io.offline = true; + } + else if (gOfflineStartupMode == kAlwaysOnline) + { + Services.io.manageOfflineStatus = manageOfflineStatus; + if (wasOffline) + Services.prefs.setBoolPref("mailnews.playback_offline", true); + // If we're managing the offline status, don't force online here... it may + // be the network really is offline. + if (!manageOfflineStatus) + Services.io.offline = false; + } + else if (gOfflineStartupMode == kRememberLastState) + { + Services.io.manageOfflineStatus = manageOfflineStatus && !wasOffline; + // If we are meant to be online, and managing the offline status + // then don't force it - it may be the network really is offline. + if (!manageOfflineStatus || wasOffline) + Services.io.offline = wasOffline; + } + else if (gOfflineStartupMode == kAskForOnlineState) + { + var bundle = Services.strings.createBundle("chrome://messenger/locale/offlineStartup.properties"); + var title = bundle.GetStringFromName("title"); + var desc = bundle.GetStringFromName("desc"); + var button0Text = bundle.GetStringFromName("workOnline"); + var button1Text = bundle.GetStringFromName("workOffline"); + var checkVal = {value:0}; + + var result = Services.prompt.confirmEx(null, title, desc, + (Services.prompt.BUTTON_POS_0 * Services.prompt.BUTTON_TITLE_IS_STRING) + + (Services.prompt.BUTTON_POS_1 * Services.prompt.BUTTON_TITLE_IS_STRING), + button0Text, button1Text, null, null, checkVal); + debug ("result = " + result + "\n"); + Services.io.manageOfflineStatus = manageOfflineStatus && result != 1; + Services.io.offline = result == 1; + if (result != 1 && wasOffline) + Services.prefs.setBoolPref("mailnews.playback_offline", true); + } + }, + + observe: function(aSubject, aTopic, aData) + { + debug("observe: " + aTopic); + + if (aTopic == "profile-change-net-teardown") + { + debug("remembering offline state"); + Services.prefs.setBoolPref("network.online", !Services.io.offline); + } + else if (aTopic == "app-startup") + { + Services.obs.addObserver(this, "profile-after-change", false); + Services.obs.addObserver(this, "profile-change-net-teardown", false); + } + else if (aTopic == "profile-after-change") + { + this.onProfileStartup(); + } + }, + + + QueryInterface: function(aIID) + { + if (aIID.equals(Components.interfaces.nsIObserver) || + aIID.equals(Components.interfaces.nsISupports)) + return this; + + throw Components.results.NS_ERROR_NO_INTERFACE; + } +} + +function nsOfflineStartupModule() +{ +} + +nsOfflineStartupModule.prototype = +{ + classID: Components.ID("3028a3c8-2165-42a4-b878-398da5d32736"), + _xpcom_factory: + { + createInstance: function(aOuter, aIID) + { + if (aOuter != null) + throw Components.results.NS_ERROR_NO_AGGREGATION; + + // return the singleton + return nsOfflineStartup.QueryInterface(aIID); + }, + + lockFactory: function(aLock) + { + // quieten warnings + } + } +}; + +//////////////////////////////////////////////////////////////////////// +// +// Debug helper +// +//////////////////////////////////////////////////////////////////////// +if (!kDebug) + debug = function(m) {}; +else + debug = function(m) {dump("\t *** nsOfflineStartup: " + m + "\n");}; + +var components = [nsOfflineStartupModule]; +var NSGetFactory = XPCOMUtils.generateNSGetFactory(components); diff --git a/mailnews/extensions/offline-startup/js/offlineStartup.manifest b/mailnews/extensions/offline-startup/js/offlineStartup.manifest new file mode 100644 index 000000000..61df84f76 --- /dev/null +++ b/mailnews/extensions/offline-startup/js/offlineStartup.manifest @@ -0,0 +1,3 @@ +component {3028a3c8-2165-42a4-b878-398da5d32736} offlineStartup.js +contract @mozilla.org/offline-startup;1 {3028a3c8-2165-42a4-b878-398da5d32736} +category app-startup Offline-startup @mozilla.org/offline-startup;1 diff --git a/mailnews/extensions/offline-startup/moz.build b/mailnews/extensions/offline-startup/moz.build new file mode 100644 index 000000000..a8a406295 --- /dev/null +++ b/mailnews/extensions/offline-startup/moz.build @@ -0,0 +1,10 @@ +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +EXTRA_COMPONENTS += [ + 'js/offlineStartup.js', + 'js/offlineStartup.manifest', +] + diff --git a/mailnews/extensions/smime/content/am-smime.js b/mailnews/extensions/smime/content/am-smime.js new file mode 100644 index 000000000..4a90d0cd7 --- /dev/null +++ b/mailnews/extensions/smime/content/am-smime.js @@ -0,0 +1,478 @@ +/* -*- Mode: Java; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +Components.utils.import("resource://gre/modules/Services.jsm"); + +var nsIX509CertDB = Components.interfaces.nsIX509CertDB; +var nsX509CertDBContractID = "@mozilla.org/security/x509certdb;1"; +var nsIX509Cert = Components.interfaces.nsIX509Cert; + +var email_recipient_cert_usage = 5; +var email_signing_cert_usage = 4; + +var gIdentity; +var gPref = null; +var gEncryptionCertName = null; +var gHiddenEncryptionPolicy = null; +var gEncryptionChoices = null; +var gSignCertName = null; +var gSignMessages = null; +var gEncryptAlways = null; +var gNeverEncrypt = null; +var gBundle = null; +var gBrandBundle; +var gSmimePrefbranch; +var gEncryptionChoicesLocked; +var gSigningChoicesLocked; +var kEncryptionCertPref = "identity.encryption_cert_name"; +var kSigningCertPref = "identity.signing_cert_name"; + +function onInit() +{ + smimeInitializeFields(); +} + +function smimeInitializeFields() +{ + // initialize all of our elements based on the current identity values.... + gEncryptionCertName = document.getElementById(kEncryptionCertPref); + gHiddenEncryptionPolicy = document.getElementById("identity.encryptionpolicy"); + gEncryptionChoices = document.getElementById("encryptionChoices"); + gSignCertName = document.getElementById(kSigningCertPref); + gSignMessages = document.getElementById("identity.sign_mail"); + gEncryptAlways = document.getElementById("encrypt_mail_always"); + gNeverEncrypt = document.getElementById("encrypt_mail_never"); + gBundle = document.getElementById("bundle_smime"); + gBrandBundle = document.getElementById("bundle_brand"); + + gEncryptionChoicesLocked = false; + gSigningChoicesLocked = false; + + if (!gIdentity) { + // The user is going to create a new identity. + // Set everything to default values. + // Do not take over the values from gAccount.defaultIdentity + // as the new identity is going to have a different mail address. + + gEncryptionCertName.value = ""; + gEncryptionCertName.nickname = ""; + gEncryptionCertName.dbKey = ""; + gSignCertName.value = ""; + gSignCertName.nickname = ""; + gSignCertName.dbKey = ""; + + gEncryptAlways.setAttribute("disabled", true); + gNeverEncrypt.setAttribute("disabled", true); + gSignMessages.setAttribute("disabled", true); + + gSignMessages.checked = false; + gEncryptionChoices.value = 0; + } + else { + var certdb = Components.classes[nsX509CertDBContractID].getService(nsIX509CertDB); + var x509cert = null; + + gEncryptionCertName.value = gIdentity.getUnicharAttribute("encryption_cert_name"); + gEncryptionCertName.dbKey = gIdentity.getCharAttribute("encryption_cert_dbkey"); + // If we succeed in looking up the certificate by the dbkey pref, then + // append the serial number " [...]" to the display value, and remember the + // nickname in a separate property. + try { + if (certdb && gEncryptionCertName.dbKey && + (x509cert = certdb.findCertByDBKey(gEncryptionCertName.dbKey))) { + gEncryptionCertName.value = x509cert.nickname + " [" + x509cert.serialNumber + "]"; + gEncryptionCertName.nickname = x509cert.nickname; + } + } catch(e) {} + + gEncryptionChoices.value = gIdentity.getIntAttribute("encryptionpolicy"); + + if (!gEncryptionCertName.value) { + gEncryptAlways.setAttribute("disabled", true); + gNeverEncrypt.setAttribute("disabled", true); + } + else { + enableEncryptionControls(true); + } + + gSignCertName.value = gIdentity.getUnicharAttribute("signing_cert_name"); + gSignCertName.dbKey = gIdentity.getCharAttribute("signing_cert_dbkey"); + x509cert = null; + // same procedure as with gEncryptionCertName (see above) + try { + if (certdb && gSignCertName.dbKey && + (x509cert = certdb.findCertByDBKey(gSignCertName.dbKey))) { + gSignCertName.value = x509cert.nickname + " [" + x509cert.serialNumber + "]"; + gSignCertName.nickname = x509cert.nickname; + } + } catch(e) {} + + gSignMessages.checked = gIdentity.getBoolAttribute("sign_mail"); + if (!gSignCertName.value) + { + gSignMessages.setAttribute("disabled", true); + } + else { + enableSigningControls(true); + } + } + + // Always start with enabling signing and encryption cert select buttons. + // This will keep the visibility of buttons in a sane state as user + // jumps from security panel of one account to another. + enableCertSelectButtons(); + + // Disable all locked elements on the panel + if (gIdentity) + onLockPreference(); +} + +function onPreInit(account, accountValues) +{ + gIdentity = account.defaultIdentity; +} + +function onSave() +{ + smimeSave(); +} + +function smimeSave() +{ + // find out which radio for the encryption radio group is selected and set that on our hidden encryptionChoice pref.... + var newValue = gEncryptionChoices.value; + gHiddenEncryptionPolicy.setAttribute('value', newValue); + gIdentity.setIntAttribute("encryptionpolicy", newValue); + gIdentity.setUnicharAttribute("encryption_cert_name", + gEncryptionCertName.nickname || gEncryptionCertName.value); + gIdentity.setCharAttribute("encryption_cert_dbkey", gEncryptionCertName.dbKey); + + gIdentity.setBoolAttribute("sign_mail", gSignMessages.checked); + gIdentity.setUnicharAttribute("signing_cert_name", + gSignCertName.nickname || gSignCertName.value); + gIdentity.setCharAttribute("signing_cert_dbkey", gSignCertName.dbKey); +} + +function smimeOnAcceptEditor() +{ + try { + if (!onOk()) + return false; + } + catch (ex) {} + + smimeSave(); + + return true; +} + +function onLockPreference() +{ + var initPrefString = "mail.identity"; + var finalPrefString; + + var allPrefElements = [ + { prefstring:"signingCertSelectButton", id:"signingCertSelectButton"}, + { prefstring:"encryptionCertSelectButton", id:"encryptionCertSelectButton"}, + { prefstring:"sign_mail", id:"identity.sign_mail"}, + { prefstring:"encryptionpolicy", id:"encryptionChoices"} + ]; + + finalPrefString = initPrefString + "." + gIdentity.key + "."; + gSmimePrefbranch = Services.prefs.getBranch(finalPrefString); + + disableIfLocked( allPrefElements ); +} + + +// Does the work of disabling an element given the array which contains xul id/prefstring pairs. +// Also saves the id/locked state in an array so that other areas of the code can avoid +// stomping on the disabled state indiscriminately. +function disableIfLocked( prefstrArray ) +{ + var i; + for (i=0; i<prefstrArray.length; i++) { + var id = prefstrArray[i].id; + var element = document.getElementById(id); + if (gSmimePrefbranch.prefIsLocked(prefstrArray[i].prefstring)) { + // If encryption choices radio group is locked, make sure the individual + // choices in the group are locked. Set a global (gEncryptionChoicesLocked) + // indicating the status so that locking can be maintained further. + if (id == "encryptionChoices") { + document.getElementById("encrypt_mail_never").setAttribute("disabled", "true"); + document.getElementById("encrypt_mail_always").setAttribute("disabled", "true"); + gEncryptionChoicesLocked = true; + } + // If option to sign mail is locked (with true/false set in config file), disable + // the corresponding checkbox and set a global (gSigningChoicesLocked) in order to + // honor the locking as user changes other elements on the panel. + if (id == "identity.sign_mail") { + document.getElementById("identity.sign_mail").setAttribute("disabled", "true"); + gSigningChoicesLocked = true; + } + else { + element.setAttribute("disabled", "true"); + if (id == "signingCertSelectButton") { + document.getElementById("signingCertClearButton").setAttribute("disabled", "true"); + } + else if (id == "encryptionCertSelectButton") { + document.getElementById("encryptionCertClearButton").setAttribute("disabled", "true"); + } + } + } + } +} + +function alertUser(message) +{ + Services.prompt.alert(window, + gBrandBundle.getString("brandShortName"), + message); +} + +function askUser(message) +{ + let button = Services.prompt.confirmEx( + window, + gBrandBundle.getString("brandShortName"), + message, + Services.prompt.STD_YES_NO_BUTTONS, + null, + null, + null, + null, + {}); + // confirmEx returns button index: + return (button == 0); +} + +function checkOtherCert(cert, pref, usage, msgNeedCertWantSame, msgWantSame, msgNeedCertWantToSelect, enabler) +{ + var otherCertInfo = document.getElementById(pref); + if (!otherCertInfo) + return; + + if (otherCertInfo.dbKey == cert.dbKey) + // all is fine, same cert is now selected for both purposes + return; + + var certdb = Components.classes[nsX509CertDBContractID].getService(nsIX509CertDB); + if (!certdb) + return; + + if (email_recipient_cert_usage == usage) { + matchingOtherCert = certdb.findEmailEncryptionCert(cert.nickname); + } + else if (email_signing_cert_usage == usage) { + matchingOtherCert = certdb.findEmailSigningCert(cert.nickname); + } + else + return; + + var userWantsSameCert = false; + + if (!otherCertInfo.value.length) { + if (matchingOtherCert && (matchingOtherCert.dbKey == cert.dbKey)) { + userWantsSameCert = askUser(gBundle.getString(msgNeedCertWantSame)); + } + else { + if (askUser(gBundle.getString(msgNeedCertWantToSelect))) { + smimeSelectCert(pref); + } + } + } + else { + if (matchingOtherCert && (matchingOtherCert.dbKey == cert.dbKey)) { + userWantsSameCert = askUser(gBundle.getString(msgWantSame)); + } + } + + if (userWantsSameCert) { + otherCertInfo.value = cert.nickname + " [" + cert.serialNumber + "]"; + otherCertInfo.nickname = cert.nickname; + otherCertInfo.dbKey = cert.dbKey; + enabler(true); + } +} + +function smimeSelectCert(smime_cert) +{ + var certInfo = document.getElementById(smime_cert); + if (!certInfo) + return; + + var picker = Components.classes["@mozilla.org/user_cert_picker;1"] + .createInstance(Components.interfaces.nsIUserCertPicker); + var canceled = new Object; + var x509cert = 0; + var certUsage; + var selectEncryptionCert; + + if (smime_cert == kEncryptionCertPref) { + selectEncryptionCert = true; + certUsage = email_recipient_cert_usage; + } else if (smime_cert == kSigningCertPref) { + selectEncryptionCert = false; + certUsage = email_signing_cert_usage; + } + + try { + x509cert = picker.pickByUsage(window, + certInfo.value, + certUsage, // this is from enum SECCertUsage + false, true, + gIdentity.email, + canceled); + } catch(e) { + canceled.value = false; + x509cert = null; + } + + if (!canceled.value) { + if (!x509cert) { + if (gIdentity.email) { + alertUser(gBundle.getFormattedString(selectEncryptionCert ? + "NoEncryptionCertForThisAddress" : + "NoSigningCertForThisAddress", + [ gIdentity.email ])); + } else { + alertUser(gBundle.getString(selectEncryptionCert ? + "NoEncryptionCert" : "NoSigningCert")); + } + } + else { + certInfo.removeAttribute("disabled"); + certInfo.value = x509cert.nickname + " [" + x509cert.serialNumber + "]"; + certInfo.nickname = x509cert.nickname; + certInfo.dbKey = x509cert.dbKey; + + if (selectEncryptionCert) { + enableEncryptionControls(true); + + checkOtherCert(x509cert, + kSigningCertPref, email_signing_cert_usage, + "signing_needCertWantSame", + "signing_wantSame", + "signing_needCertWantToSelect", + enableSigningControls); + } else { + enableSigningControls(true); + + checkOtherCert(x509cert, + kEncryptionCertPref, email_recipient_cert_usage, + "encryption_needCertWantSame", + "encryption_wantSame", + "encryption_needCertWantToSelect", + enableEncryptionControls); + } + } + } + + enableCertSelectButtons(); +} + +function enableEncryptionControls(do_enable) +{ + if (gEncryptionChoicesLocked) + return; + + if (do_enable) { + gEncryptAlways.removeAttribute("disabled"); + gNeverEncrypt.removeAttribute("disabled"); + gEncryptionCertName.removeAttribute("disabled"); + } + else { + gEncryptAlways.setAttribute("disabled", "true"); + gNeverEncrypt.setAttribute("disabled", "true"); + gEncryptionCertName.setAttribute("disabled", "true"); + gEncryptionChoices.value = 0; + } +} + +function enableSigningControls(do_enable) +{ + if (gSigningChoicesLocked) + return; + + if (do_enable) { + gSignMessages.removeAttribute("disabled"); + gSignCertName.removeAttribute("disabled"); + } + else { + gSignMessages.setAttribute("disabled", "true"); + gSignCertName.setAttribute("disabled", "true"); + gSignMessages.checked = false; + } +} + +function enableCertSelectButtons() +{ + document.getElementById("signingCertSelectButton").removeAttribute("disabled"); + + if (document.getElementById('identity.signing_cert_name').value.length) + document.getElementById("signingCertClearButton").removeAttribute("disabled"); + else + document.getElementById("signingCertClearButton").setAttribute("disabled", "true"); + + document.getElementById("encryptionCertSelectButton").removeAttribute("disabled"); + + if (document.getElementById('identity.encryption_cert_name').value.length) + document.getElementById("encryptionCertClearButton").removeAttribute("disabled"); + else + document.getElementById("encryptionCertClearButton").setAttribute("disabled", "true"); +} + +function smimeClearCert(smime_cert) +{ + var certInfo = document.getElementById(smime_cert); + if (!certInfo) + return; + + certInfo.setAttribute("disabled", "true"); + certInfo.value = ""; + certInfo.nickname = ""; + certInfo.dbKey = ""; + + if (smime_cert == kEncryptionCertPref) { + enableEncryptionControls(false); + } else if (smime_cert == kSigningCertPref) { + enableSigningControls(false); + } + + enableCertSelectButtons(); +} + +function openCertManager() +{ + // Check for an existing certManager window and focus it; it's not + // application modal. + let lastCertManager = Services.wm.getMostRecentWindow("mozilla:certmanager"); + if (lastCertManager) + lastCertManager.focus(); + else + window.openDialog("chrome://pippki/content/certManager.xul", "", + "centerscreen,resizable=yes,dialog=no"); +} + +function openDeviceManager() +{ + // Check for an existing deviceManager window and focus it; it's not + // application modal. + let lastCertManager = Services.wm.getMostRecentWindow("mozilla:devicemanager"); + if (lastCertManager) + lastCertManager.focus(); + else + window.openDialog("chrome://pippki/content/device_manager.xul", "", + "centerscreen,resizable=yes,dialog=no"); +} + +function smimeOnLoadEditor() +{ + smimeInitializeFields(); + + document.documentElement.setAttribute("ondialogaccept", + "return smimeOnAcceptEditor();"); +} + diff --git a/mailnews/extensions/smime/content/am-smime.xul b/mailnews/extensions/smime/content/am-smime.xul new file mode 100644 index 000000000..bb46bb49d --- /dev/null +++ b/mailnews/extensions/smime/content/am-smime.xul @@ -0,0 +1,26 @@ +<?xml version="1.0"?> +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> + +<?xml-stylesheet href="chrome://messenger/skin/accountManage.css" type="text/css"?> + +<?xul-overlay href="chrome://messenger/content/am-smimeOverlay.xul"?> + +<!DOCTYPE page SYSTEM "chrome://messenger/locale/am-smime.dtd"> + +<page xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul" + class="color-dialog" + onload="parent.onPanelLoaded('am-smime.xul');" + ondialogaccept="smimeOnAcceptEditor();"> + + <vbox flex="1" style="overflow: auto;"> + <script type="application/javascript" src="chrome://messenger/content/AccountManager.js"/> + <script type="application/javascript" src="chrome://messenger/content/am-smime.js"/> + + <dialogheader title="&securityTitle.label;"/> + + <vbox flex="1" id="smimeEditing"/> + </vbox> + +</page> diff --git a/mailnews/extensions/smime/content/am-smimeIdentityEditOverlay.xul b/mailnews/extensions/smime/content/am-smimeIdentityEditOverlay.xul new file mode 100644 index 000000000..2ff5c2b7d --- /dev/null +++ b/mailnews/extensions/smime/content/am-smimeIdentityEditOverlay.xul @@ -0,0 +1,39 @@ +<?xml version="1.0"?> +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> + +<?xml-stylesheet href="chrome://messenger/skin/accountManage.css" + type="text/css"?> + +<?xul-overlay href="chrome://messenger/content/am-smimeOverlay.xul"?> + +<!DOCTYPE overlay SYSTEM "chrome://messenger/locale/am-smime.dtd"> + +<!-- + This is the overlay that adds the SMIME configurator + to the identity editor of the account manager +--> +<overlay id="smimeAmIdEditOverlay" + xmlns:html="http://www.w3.org/1999/xhtml" + xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul"> + + <script type="application/javascript" + src="chrome://messenger/content/AccountManager.js"/> + <script type="application/javascript" + src="chrome://messenger/content/am-smime.js"/> + + <tabs id="identitySettings"> + <tab label="&securityTab.label;"/> + </tabs> + + <tabpanels id="identityTabsPanels"> + <vbox flex="1" name="smimeEditingContent" id="smimeEditing"/> + </tabpanels> + + <script type="application/javascript"> + <![CDATA[ + window.addEventListener("load", smimeOnLoadEditor, false); + ]]> + </script> +</overlay> diff --git a/mailnews/extensions/smime/content/am-smimeOverlay.xul b/mailnews/extensions/smime/content/am-smimeOverlay.xul new file mode 100644 index 000000000..eb76b4b2c --- /dev/null +++ b/mailnews/extensions/smime/content/am-smimeOverlay.xul @@ -0,0 +1,102 @@ +<?xml version="1.0"?> +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> + +<?xml-stylesheet href="chrome://messenger/skin/accountManage.css" + type="text/css"?> + +<!DOCTYPE overlay SYSTEM "chrome://messenger/locale/am-smime.dtd"> + +<overlay xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul"> + + <vbox id="smimeEditing"> + + <stringbundleset> + <stringbundle id="bundle_smime" src="chrome://messenger/locale/am-smime.properties"/> + <stringbundle id="bundle_brand" src="chrome://branding/locale/brand.properties"/> + </stringbundleset> + + <label hidden="true" wsm_persist="true" id="identity.encryptionpolicy"/> + + <description>&securityHeading.label;</description> + + <groupbox id="signing.titlebox"> + <caption label="&signingGroupTitle.label;"/> + + <label value="&signingCert.message;" control="identity.signing_cert_name" + prefstring="mail.identity.%identitykey%.encryptionpolicy"/> + + <hbox align="center"> + <textbox id="identity.signing_cert_name" wsm_persist="true" flex="1" + prefstring="mail.identity.%identitykey%.signing_cert_name" + readonly="true" disabled="true"/> + + <button id="signingCertSelectButton" + label="&digitalSign.certificate.button;" + accesskey="&digitalSign.certificate.accesskey;" + oncommand="smimeSelectCert('identity.signing_cert_name')"/> + + <button id="signingCertClearButton" + label="&digitalSign.certificate_clear.button;" + accesskey="&digitalSign.certificate_clear.accesskey;" + oncommand="smimeClearCert('identity.signing_cert_name')"/> + </hbox> + + <separator class="thin"/> + + <checkbox id="identity.sign_mail" wsm_persist="true" + prefstring="mail.identity.%identitykey%.sign_mail" + label="&signMessage.label;" accesskey="&signMessage.accesskey;"/> + </groupbox> + + <groupbox id="encryption.titlebox"> + <caption label="&encryptionGroupTitle.label;"/> + + <label value="&encryptionCert.message;" + control="identity.encryption_cert_name"/> + + <hbox align="center"> + <textbox id="identity.encryption_cert_name" wsm_persist="true" flex="1" + prefstring="mail.identity.%identitykey%.encryption_cert_name" + readonly="true" disabled="true"/> + + <button id="encryptionCertSelectButton" + label="&encryption.certificate.button;" + accesskey="&encryption.certificate.accesskey;" + oncommand="smimeSelectCert('identity.encryption_cert_name')"/> + + <button id="encryptionCertClearButton" + label="&encryption.certificate_clear.button;" + accesskey="&encryption.certificate_clear.accesskey;" + oncommand="smimeClearCert('identity.encryption_cert_name')"/> + </hbox> + + <separator class="thin"/> + + <label value="&encryptionChoiceLabel.label;" control="encryptionChoices"/> + + <radiogroup id="encryptionChoices"> + <radio id="encrypt_mail_never" wsm_persist="true" value="0" + label="&neverEncrypt.label;" + accesskey="&neverEncrypt.accesskey;"/> + + <radio id="encrypt_mail_always" wsm_persist="true" value="2" + label="&alwaysEncryptMessage.label;" + accesskey="&alwaysEncryptMessage.accesskey;"/> + </radiogroup> + </groupbox> + + <!-- Certificate manager --> + <groupbox id="smimeCertificateManager" orient="horizontal"> + <caption label="&certificates.label;"/> + <button id="openCertManagerButton" oncommand="openCertManager();" + label="&manageCerts2.label;" accesskey="&manageCerts2.accesskey;" + prefstring="security.disable_button.openCertManager"/> + <button id="openDeviceManagerButton" oncommand="openDeviceManager();" + label="&manageDevices.label;" accesskey="&manageDevices.accesskey;" + prefstring="security.disable_button.openDeviceManager"/> + </groupbox> + </vbox> +</overlay> diff --git a/mailnews/extensions/smime/content/certFetchingStatus.js b/mailnews/extensions/smime/content/certFetchingStatus.js new file mode 100644 index 000000000..8848ff9b6 --- /dev/null +++ b/mailnews/extensions/smime/content/certFetchingStatus.js @@ -0,0 +1,265 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* We expect the following arguments: + - pref name of LDAP directory to fetch from + - array with email addresses + + Display modal dialog with message and stop button. + In onload, kick off binding to LDAP. + When bound, kick off the searches. + On finding certificates, import into permanent cert database. + When all searches are finished, close the dialog. +*/ + +Components.utils.import("resource://gre/modules/Services.jsm"); + +var nsIX509CertDB = Components.interfaces.nsIX509CertDB; +var nsX509CertDB = "@mozilla.org/security/x509certdb;1"; +var CertAttribute = "usercertificate;binary"; + +var gEmailAddresses; +var gDirectoryPref; +var gLdapServerURL; +var gLdapConnection; +var gCertDB; +var gLdapOperation; +var gLogin; + +function onLoad() +{ + gDirectoryPref = window.arguments[0]; + gEmailAddresses = window.arguments[1]; + + if (!gEmailAddresses.length) + { + window.close(); + return; + } + + setTimeout(search, 1); +} + +function search() +{ + // get the login to authenticate as, if there is one + try { + gLogin = Services.prefs.getComplexValue(gDirectoryPref + ".auth.dn", Components.interfaces.nsISupportsString).data; + } catch (ex) { + // if we don't have this pref, no big deal + } + + try { + let url = Services.prefs.getCharPref(gDirectoryPref + ".uri"); + + gLdapServerURL = Services.io + .newURI(url, null, null).QueryInterface(Components.interfaces.nsILDAPURL); + + gLdapConnection = Components.classes["@mozilla.org/network/ldap-connection;1"] + .createInstance().QueryInterface(Components.interfaces.nsILDAPConnection); + + gLdapConnection.init(gLdapServerURL, gLogin, new boundListener(), + null, Components.interfaces.nsILDAPConnection.VERSION3); + + } catch (ex) { + dump(ex); + dump(" exception creating ldap connection\n"); + window.close(); + } +} + +function stopFetching() +{ + if (gLdapOperation) { + try { + gLdapOperation.abandon(); + } + catch (e) { + } + } + return true; +} + +function importCert(ber_value) +{ + if (!gCertDB) { + gCertDB = Components.classes[nsX509CertDB].getService(nsIX509CertDB); + } + + var cert_length = new Object(); + var cert_bytes = ber_value.get(cert_length); + + if (cert_bytes) { + gCertDB.importEmailCertificate(cert_bytes, cert_length.value, null); + } +} + +function getLDAPOperation() +{ + gLdapOperation = Components.classes["@mozilla.org/network/ldap-operation;1"] + .createInstance().QueryInterface(Components.interfaces.nsILDAPOperation); + + gLdapOperation.init(gLdapConnection, + new ldapMessageListener(), + null); +} + +function getPassword() +{ + // we only need a password if we are using credentials + if (gLogin) + { + let authPrompter = Services.ww.getNewAuthPrompter(window.QueryInterface(Components.interfaces.nsIDOMWindow)); + let strBundle = document.getElementById('bundle_ldap'); + let password = { value: "" }; + + // nsLDAPAutocompleteSession uses asciiHost instead of host for the prompt text, I think we should be + // consistent. + if (authPrompter.promptPassword(strBundle.getString("authPromptTitle"), + strBundle.getFormattedString("authPromptText", [gLdapServerURL.asciiHost]), + gLdapServerURL.spec, + authPrompter.SAVE_PASSWORD_PERMANENTLY, + password)) + return password.value; + } + + return null; +} + +function kickOffBind() +{ + try { + getLDAPOperation(); + gLdapOperation.simpleBind(getPassword()); + } + catch (e) { + window.close(); + } +} + +function kickOffSearch() +{ + try { + var prefix1 = ""; + var suffix1 = ""; + + var urlFilter = gLdapServerURL.filter; + + if (urlFilter != null && urlFilter.length > 0 && urlFilter != "(objectclass=*)") { + if (urlFilter.startsWith('(')) { + prefix1 = "(&" + urlFilter; + } + else { + prefix1 = "(&(" + urlFilter + ")"; + } + suffix1 = ")"; + } + + var prefix2 = ""; + var suffix2 = ""; + + if (gEmailAddresses.length > 1) { + prefix2 = "(|"; + suffix2 = ")"; + } + + var mailFilter = ""; + + for (var i = 0; i < gEmailAddresses.length; ++i) { + mailFilter += "(mail=" + gEmailAddresses[i] + ")"; + } + + var filter = prefix1 + prefix2 + mailFilter + suffix2 + suffix1; + + var wanted_attributes = CertAttribute; + + // Max search results => + // Double number of email addresses, because each person might have + // multiple certificates listed. We expect at most two certificates, + // one for signing, one for encrypting. + // Maybe that number should be larger, to allow for deployments, + // where even more certs can be stored per user??? + + var maxEntriesWanted = gEmailAddresses.length * 2; + + getLDAPOperation(); + gLdapOperation.searchExt(gLdapServerURL.dn, gLdapServerURL.scope, + filter, wanted_attributes, 0, maxEntriesWanted); + } + catch (e) { + window.close(); + } +} + + +function boundListener() { +} + +boundListener.prototype.QueryInterface = + function(iid) { + if (iid.equals(Components.interfaces.nsISupports) || + iid.equals(Components.interfaces.nsILDAPMessageListener)) + return this; + + throw Components.results.NS_ERROR_NO_INTERFACE; + } + +boundListener.prototype.onLDAPMessage = + function(aMessage) { + } + +boundListener.prototype.onLDAPInit = + function(aConn, aStatus) { + kickOffBind(); + } + + +function ldapMessageListener() { +} + +ldapMessageListener.prototype.QueryInterface = + function(iid) { + if (iid.equals(Components.interfaces.nsISupports) || + iid.equals(Components.interfaces.nsILDAPMessageListener)) + return this; + + throw Components.results.NS_ERROR_NO_INTERFACE; + } + +ldapMessageListener.prototype.onLDAPMessage = + function(aMessage) { + if (Components.interfaces.nsILDAPMessage.RES_SEARCH_RESULT == aMessage.type) { + window.close(); + return; + } + + if (Components.interfaces.nsILDAPMessage.RES_BIND == aMessage.type) { + if (Components.interfaces.nsILDAPErrors.SUCCESS != aMessage.errorCode) { + window.close(); + } + else { + kickOffSearch(); + } + return; + } + + if (Components.interfaces.nsILDAPMessage.RES_SEARCH_ENTRY == aMessage.type) { + var outSize = new Object(); + try { + var outBinValues = aMessage.getBinaryValues(CertAttribute, outSize); + + var i; + for (i=0; i < outSize.value; ++i) { + importCert(outBinValues[i]); + } + } + catch (e) { + } + return; + } + } + +ldapMessageListener.prototype.onLDAPInit = + function(aConn, aStatus) { + } diff --git a/mailnews/extensions/smime/content/certFetchingStatus.xul b/mailnews/extensions/smime/content/certFetchingStatus.xul new file mode 100644 index 000000000..29b824fc9 --- /dev/null +++ b/mailnews/extensions/smime/content/certFetchingStatus.xul @@ -0,0 +1,24 @@ +<?xml version="1.0"?> +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> + +<?xml-stylesheet href="chrome://global/skin/" type="text/css"?> +<?xml-stylesheet href="chrome://messenger/skin/smime/certFetchingStatus.css" type="text/css"?> + +<!DOCTYPE dialog SYSTEM "chrome://messenger-smime/locale/certFetchingStatus.dtd"> + +<dialog id="certFetchingStatus" title="&title.label;" + xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul" + style="width: 50em;" + buttons="cancel" + buttonlabelcancel="&stop.label;" + ondialogcancel="return stopFetching();" + onload="onLoad();"> + + <stringbundle id="bundle_ldap" src="chrome://mozldap/locale/ldap.properties"/> +<script type="application/javascript" src="chrome://messenger-smime/content/certFetchingStatus.js"/> + + <description>&info.message;</description> + +</dialog> diff --git a/mailnews/extensions/smime/content/certpicker.js b/mailnews/extensions/smime/content/certpicker.js new file mode 100644 index 000000000..19554066f --- /dev/null +++ b/mailnews/extensions/smime/content/certpicker.js @@ -0,0 +1,73 @@ +/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +"use strict"; + +const nsIDialogParamBlock = Components.interfaces.nsIDialogParamBlock; + +var dialogParams; +var itemCount = 0; + +function onLoad() +{ + dialogParams = window.arguments[0].QueryInterface(nsIDialogParamBlock); + + var selectElement = document.getElementById("nicknames"); + itemCount = dialogParams.GetInt(0); + + var selIndex = dialogParams.GetInt(1); + if (selIndex < 0) { + selIndex = 0; + } + + for (let i = 0; i < itemCount; i++) { + let menuItemNode = document.createElement("menuitem"); + let nick = dialogParams.GetString(i); + menuItemNode.setAttribute("value", i); + menuItemNode.setAttribute("label", nick); // This is displayed. + selectElement.firstChild.appendChild(menuItemNode); + + if (selIndex == i) { + selectElement.selectedItem = menuItemNode; + } + } + + dialogParams.SetInt(0, 0); // Set cancel return value. + setDetails(); +} + +function setDetails() +{ + let selItem = document.getElementById("nicknames").value; + if (selItem.length == 0) { + return; + } + + let index = parseInt(selItem); + let details = dialogParams.GetString(index + itemCount); + document.getElementById("details").value = details; +} + +function onCertSelected() +{ + setDetails(); +} + +function doOK() +{ + // Signal that the user accepted. + dialogParams.SetInt(0, 1); + + // Signal the index of the selected cert in the list of cert nicknames + // provided. + let index = parseInt(document.getElementById("nicknames").value); + dialogParams.SetInt(1, index); + return true; +} + +function doCancel() +{ + dialogParams.SetInt(0, 0); // Signal that the user cancelled. + return true; +} diff --git a/mailnews/extensions/smime/content/certpicker.xul b/mailnews/extensions/smime/content/certpicker.xul new file mode 100644 index 000000000..2c4cd3b22 --- /dev/null +++ b/mailnews/extensions/smime/content/certpicker.xul @@ -0,0 +1,38 @@ +<?xml version="1.0"?> +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> + +<?xml-stylesheet href="chrome://global/skin/" type="text/css"?> + +<!DOCTYPE dialog [ +<!ENTITY % amSMIMEDTD SYSTEM "chrome://messenger/locale/am-smime.dtd" > +%amSMIMEDTD; +]> + +<dialog id="certPicker" title="&certPicker.title;" + xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul" + style="width: 50em;" + buttons="accept,cancel" + ondialogaccept="return doOK();" + ondialogcancel="return doCancel();" + onload="onLoad();"> + +<script type="application/javascript" + src="chrome://messenger/content/certpicker.js"/> + + <hbox align="center"> + <broadcaster id="certSelected" oncommand="onCertSelected();"/> + <label id="pickerInfo" value="&certPicker.info;"/> + <!-- The items in this menulist must never be sorted, + but remain in the order filled by the application + --> + <menulist id="nicknames" observes="certSelected"> + <menupopup/> + </menulist> + </hbox> + <separator class="thin"/> + <label value="&certPicker.detailsLabel;"/> + <textbox readonly="true" id="details" multiline="true" + style="height: 12em;" flex="1"/> +</dialog> diff --git a/mailnews/extensions/smime/content/msgCompSMIMEOverlay.js b/mailnews/extensions/smime/content/msgCompSMIMEOverlay.js new file mode 100644 index 000000000..582a073ea --- /dev/null +++ b/mailnews/extensions/smime/content/msgCompSMIMEOverlay.js @@ -0,0 +1,357 @@ +/* -*- Mode: Java; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +Components.utils.import("resource://gre/modules/Services.jsm"); + +// Account encryption policy values: +// const kEncryptionPolicy_Never = 0; +// 'IfPossible' was used by ns4. +// const kEncryptionPolicy_IfPossible = 1; +var kEncryptionPolicy_Always = 2; + +var gEncryptedURIService = + Components.classes["@mozilla.org/messenger-smime/smime-encrypted-uris-service;1"] + .getService(Components.interfaces.nsIEncryptedSMIMEURIsService); + +var gNextSecurityButtonCommand = ""; +var gSMFields = null; +var gEncryptOptionChanged; +var gSignOptionChanged; + +function onComposerLoad() +{ + // Are we already set up ? Or are the required fields missing ? + if (gSMFields || !gMsgCompose || !gMsgCompose.compFields) + return; + + gMsgCompose.compFields.securityInfo = null; + + gSMFields = Components.classes["@mozilla.org/messenger-smime/composefields;1"] + .createInstance(Components.interfaces.nsIMsgSMIMECompFields); + if (!gSMFields) + return; + + gMsgCompose.compFields.securityInfo = gSMFields; + + // Set up the intial security state. + gSMFields.requireEncryptMessage = + gCurrentIdentity.getIntAttribute("encryptionpolicy") == kEncryptionPolicy_Always; + if (!gSMFields.requireEncryptMessage && + gEncryptedURIService && + gEncryptedURIService.isEncrypted(gMsgCompose.originalMsgURI)) + { + // Override encryption setting if original is known as encrypted. + gSMFields.requireEncryptMessage = true; + } + if (gSMFields.requireEncryptMessage) + setEncryptionUI(); + else + setNoEncryptionUI(); + + gSMFields.signMessage = gCurrentIdentity.getBoolAttribute("sign_mail"); + if (gSMFields.signMessage) + setSignatureUI(); + else + setNoSignatureUI(); +} + +addEventListener("load", smimeComposeOnLoad, {capture: false, once: true}); + +// this function gets called multiple times +function smimeComposeOnLoad() +{ + onComposerLoad(); + + top.controllers.appendController(SecurityController); + + addEventListener("compose-from-changed", onComposerFromChanged, true); + addEventListener("compose-send-message", onComposerSendMessage, true); + + addEventListener("unload", smimeComposeOnUnload, {capture: false, once: true}); +} + +function smimeComposeOnUnload() +{ + removeEventListener("compose-from-changed", onComposerFromChanged, true); + removeEventListener("compose-send-message", onComposerSendMessage, true); + + top.controllers.removeController(SecurityController); +} + +function showNeedSetupInfo() +{ + let compSmimeBundle = document.getElementById("bundle_comp_smime"); + let brandBundle = document.getElementById("bundle_brand"); + if (!compSmimeBundle || !brandBundle) + return; + + let buttonPressed = Services.prompt.confirmEx(window, + brandBundle.getString("brandShortName"), + compSmimeBundle.getString("NeedSetup"), + Services.prompt.STD_YES_NO_BUTTONS, 0, 0, 0, null, {}); + if (buttonPressed == 0) + openHelp("sign-encrypt", "chrome://communicator/locale/help/suitehelp.rdf"); +} + +function toggleEncryptMessage() +{ + if (!gSMFields) + return; + + gSMFields.requireEncryptMessage = !gSMFields.requireEncryptMessage; + + if (gSMFields.requireEncryptMessage) + { + // Make sure we have a cert. + if (!gCurrentIdentity.getUnicharAttribute("encryption_cert_name")) + { + gSMFields.requireEncryptMessage = false; + showNeedSetupInfo(); + return; + } + + setEncryptionUI(); + } + else + { + setNoEncryptionUI(); + } + + gEncryptOptionChanged = true; +} + +function toggleSignMessage() +{ + if (!gSMFields) + return; + + gSMFields.signMessage = !gSMFields.signMessage; + + if (gSMFields.signMessage) // make sure we have a cert name... + { + if (!gCurrentIdentity.getUnicharAttribute("signing_cert_name")) + { + gSMFields.signMessage = false; + showNeedSetupInfo(); + return; + } + + setSignatureUI(); + } + else + { + setNoSignatureUI(); + } + + gSignOptionChanged = true; +} + +function setSecuritySettings(menu_id) +{ + if (!gSMFields) + return; + + document.getElementById("menu_securityEncryptRequire" + menu_id) + .setAttribute("checked", gSMFields.requireEncryptMessage); + document.getElementById("menu_securitySign" + menu_id) + .setAttribute("checked", gSMFields.signMessage); +} + +function setNextCommand(what) +{ + gNextSecurityButtonCommand = what; +} + +function doSecurityButton() +{ + var what = gNextSecurityButtonCommand; + gNextSecurityButtonCommand = ""; + + switch (what) + { + case "encryptMessage": + toggleEncryptMessage(); + break; + + case "signMessage": + toggleSignMessage(); + break; + + case "show": + default: + showMessageComposeSecurityStatus(); + } +} + +function setNoSignatureUI() +{ + top.document.getElementById("securityStatus").removeAttribute("signing"); + top.document.getElementById("signing-status").collapsed = true; +} + +function setSignatureUI() +{ + top.document.getElementById("securityStatus").setAttribute("signing", "ok"); + top.document.getElementById("signing-status").collapsed = false; +} + +function setNoEncryptionUI() +{ + top.document.getElementById("securityStatus").removeAttribute("crypto"); + top.document.getElementById("encryption-status").collapsed = true; +} + +function setEncryptionUI() +{ + top.document.getElementById("securityStatus").setAttribute("crypto", "ok"); + top.document.getElementById("encryption-status").collapsed = false; +} + +function showMessageComposeSecurityStatus() +{ + Recipients2CompFields(gMsgCompose.compFields); + + window.openDialog( + "chrome://messenger-smime/content/msgCompSecurityInfo.xul", + "", + "chrome,modal,resizable,centerscreen", + { + compFields : gMsgCompose.compFields, + subject : GetMsgSubjectElement().value, + smFields : gSMFields, + isSigningCertAvailable : + gCurrentIdentity.getUnicharAttribute("signing_cert_name") != "", + isEncryptionCertAvailable : + gCurrentIdentity.getUnicharAttribute("encryption_cert_name") != "", + currentIdentity : gCurrentIdentity + } + ); +} + +var SecurityController = +{ + supportsCommand: function(command) + { + switch (command) + { + case "cmd_viewSecurityStatus": + return true; + + default: + return false; + } + }, + + isCommandEnabled: function(command) + { + switch (command) + { + case "cmd_viewSecurityStatus": + return true; + + default: + return false; + } + } +}; + +function onComposerSendMessage() +{ + let missingCount = new Object(); + let emailAddresses = new Object(); + + try + { + if (!gMsgCompose.compFields.securityInfo.requireEncryptMessage) + return; + + Components.classes["@mozilla.org/messenger-smime/smimejshelper;1"] + .createInstance(Components.interfaces.nsISMimeJSHelper) + .getNoCertAddresses(gMsgCompose.compFields, + missingCount, + emailAddresses); + } + catch (e) + { + return; + } + + if (missingCount.value > 0) + { + // The rules here: If the current identity has a directoryServer set, then + // use that, otherwise, try the global preference instead. + + let autocompleteDirectory; + + // Does the current identity override the global preference? + if (gCurrentIdentity.overrideGlobalPref) + { + autocompleteDirectory = gCurrentIdentity.directoryServer; + } + else + { + // Try the global one + if (Services.prefs.getBoolPref("ldap_2.autoComplete.useDirectory")) + autocompleteDirectory = + Services.prefs.getCharPref("ldap_2.autoComplete.directoryServer"); + } + + if (autocompleteDirectory) + window.openDialog("chrome://messenger-smime/content/certFetchingStatus.xul", + "", + "chrome,modal,resizable,centerscreen", + autocompleteDirectory, + emailAddresses.value); + } +} + +function onComposerFromChanged() +{ + if (!gSMFields) + return; + + var encryptionPolicy = gCurrentIdentity.getIntAttribute("encryptionpolicy"); + var useEncryption = false; + + if (!gEncryptOptionChanged) + { + // Encryption wasn't manually checked. + // Set up the encryption policy from the setting of the new identity. + + // 0 == never, 1 == if possible (ns4), 2 == always encrypt. + useEncryption = (encryptionPolicy == kEncryptionPolicy_Always); + } + else + { + useEncryption = !!gCurrentIdentity.getUnicharAttribute("encryption_cert_name"); + } + + gSMFields.requireEncryptMessage = useEncryption; + if (useEncryption) + setEncryptionUI(); + else + setNoEncryptionUI(); + + // - If signing is disabled, we will not turn it on automatically. + // - If signing is enabled, but the new account defaults to not sign, we will turn signing off. + var signMessage = gCurrentIdentity.getBoolAttribute("sign_mail"); + var useSigning = false; + + if (!gSignOptionChanged) + { + // Signing wasn't manually checked. + // Set up the signing policy from the setting of the new identity. + useSigning = signMessage; + } + else + { + useSigning = !!gCurrentIdentity.getUnicharAttribute("signing_cert_name"); + } + gSMFields.signMessage = useSigning; + if (useSigning) + setSignatureUI(); + else + setNoSignatureUI(); +} diff --git a/mailnews/extensions/smime/content/msgCompSMIMEOverlay.xul b/mailnews/extensions/smime/content/msgCompSMIMEOverlay.xul new file mode 100644 index 000000000..ec6495e20 --- /dev/null +++ b/mailnews/extensions/smime/content/msgCompSMIMEOverlay.xul @@ -0,0 +1,85 @@ +<?xml version="1.0"?> +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> + + +<?xml-stylesheet href="chrome://messenger/skin/smime/msgCompSMIMEOverlay.css" type="text/css"?> + +<!DOCTYPE overlay SYSTEM "chrome://messenger-smime/locale/msgCompSMIMEOverlay.dtd"> + +<overlay xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul"> + + <script type="application/javascript" src="chrome://messenger-smime/content/msgCompSMIMEOverlay.js"/> + + <window id="msgcomposeWindow"> + <broadcaster id="securityStatus" crypto="" signing=""/> + <observes element="securityStatus" attribute="crypto" /> + <observes element="securityStatus" attribute="signing" /> + <stringbundle id="bundle_comp_smime" src="chrome://messenger-smime/locale/msgCompSMIMEOverlay.properties"/> + <stringbundle id="bundle_brand" src="chrome://branding/locale/brand.properties"/> + </window> + + <menupopup id="optionsMenuPopup" + onpopupshowing="setSecuritySettings(1);"> + <menuseparator id="smimeOptionsSeparator"/> + + <menuitem id="menu_securityEncryptRequire1" + type="checkbox" + label="&menu_securityEncryptRequire.label;" + accesskey="&menu_securityEncryptRequire.accesskey;" + oncommand="toggleEncryptMessage();"/> + <menuitem id="menu_securitySign1" + type="checkbox" + label="&menu_securitySign.label;" + accesskey="&menu_securitySign.accesskey;" + oncommand="toggleSignMessage();"/> + </menupopup> + + <toolbarpalette id="MsgComposeToolbarPalette"> + <toolbarbutton id="button-security" + type="menu-button" + class="toolbarbutton-1" + label="&securityButton.label;" + tooltiptext="&securityButton.tooltip;" + oncommand="doSecurityButton();"> + <menupopup onpopupshowing="setSecuritySettings(2);"> + <menuitem id="menu_securityEncryptRequire2" + type="checkbox" + label="&menu_securityEncryptRequire.label;" + accesskey="&menu_securityEncryptRequire.accesskey;" + oncommand="setNextCommand('encryptMessage');"/> + <menuitem id="menu_securitySign2" + type="checkbox" + label="&menu_securitySign.label;" + accesskey="&menu_securitySign.accesskey;" + oncommand="setNextCommand('signMessage');"/> + <menuseparator id="smimeToolbarButtonSeparator"/> + <menuitem id="menu_securityStatus2" + label="&menu_securityStatus.label;" + accesskey="&menu_securityStatus.accesskey;" + oncommand="setNextCommand('show');"/> + </menupopup> + </toolbarbutton> + </toolbarpalette> + + <statusbar id="status-bar"> + <statusbarpanel insertbefore="offline-status" class="statusbarpanel-iconic" collapsed="true" + id="signing-status" oncommand="showMessageComposeSecurityStatus();"/> + <statusbarpanel insertbefore="offline-status" class="statusbarpanel-iconic" collapsed="true" + id="encryption-status" oncommand="showMessageComposeSecurityStatus();"/> + </statusbar> + + <commandset id="composeCommands"> + <command id="cmd_viewSecurityStatus" oncommand="showMessageComposeSecurityStatus();"/> + </commandset> + + <menupopup id="menu_View_Popup"> + <menuseparator id="viewMenuBeforeSecurityStatusSeparator"/> + <menuitem id="menu_viewSecurityStatus" + label="&menu_viewSecurityStatus.label;" + accesskey="&menu_viewSecurityStatus.accesskey;" + command="cmd_viewSecurityStatus"/> + </menupopup> + +</overlay> diff --git a/mailnews/extensions/smime/content/msgCompSecurityInfo.js b/mailnews/extensions/smime/content/msgCompSecurityInfo.js new file mode 100644 index 000000000..5a2a7432f --- /dev/null +++ b/mailnews/extensions/smime/content/msgCompSecurityInfo.js @@ -0,0 +1,244 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +Components.utils.import("resource://gre/modules/Services.jsm"); + +var gListBox; +var gViewButton; +var gBundle; + +var gEmailAddresses; +var gCertStatusSummaries; +var gCertIssuedInfos; +var gCertExpiresInfos; +var gCerts; +var gCount; + +var gSMimeContractID = "@mozilla.org/messenger-smime/smimejshelper;1"; +var gISMimeJSHelper = Components.interfaces.nsISMimeJSHelper; +var gIX509Cert = Components.interfaces.nsIX509Cert; +var nsICertificateDialogs = Components.interfaces.nsICertificateDialogs; +var nsCertificateDialogs = "@mozilla.org/nsCertificateDialogs;1" + +function getStatusExplanation(value) +{ + switch (value) + { + case gIX509Cert.VERIFIED_OK: + return gBundle.getString("StatusValid"); + + case gIX509Cert.NOT_VERIFIED_UNKNOWN: + case gIX509Cert.INVALID_CA: + case gIX509Cert.USAGE_NOT_ALLOWED: + return gBundle.getString("StatusInvalid"); + + case gIX509Cert.CERT_REVOKED: + return gBundle.getString("StatusRevoked"); + + case gIX509Cert.CERT_EXPIRED: + return gBundle.getString("StatusExpired"); + + case gIX509Cert.CERT_NOT_TRUSTED: + case gIX509Cert.ISSUER_NOT_TRUSTED: + case gIX509Cert.ISSUER_UNKNOWN: + return gBundle.getString("StatusUntrusted"); + } + + return ""; +} + +function onLoad() +{ + var params = window.arguments[0]; + if (!params) + return; + + var helper = Components.classes[gSMimeContractID].createInstance(gISMimeJSHelper); + + if (!helper) + return; + + gListBox = document.getElementById("infolist"); + gViewButton = document.getElementById("viewCertButton"); + gBundle = document.getElementById("bundle_smime_comp_info"); + + gEmailAddresses = new Object(); + gCertStatusSummaries = new Object(); + gCertIssuedInfos = new Object(); + gCertExpiresInfos = new Object(); + gCerts = new Object(); + gCount = new Object(); + var canEncrypt = new Object(); + + var allow_ldap_cert_fetching = false; + + try { + if (params.compFields.securityInfo.requireEncryptMessage) { + allow_ldap_cert_fetching = true; + } + } + catch (e) + { + } + + while (true) + { + try + { + helper.getRecipientCertsInfo( + params.compFields, + gCount, + gEmailAddresses, + gCertStatusSummaries, + gCertIssuedInfos, + gCertExpiresInfos, + gCerts, + canEncrypt); + } + catch (e) + { + dump(e); + return; + } + + if (!allow_ldap_cert_fetching) + break; + + allow_ldap_cert_fetching = false; + + var missing = new Array(); + + for (var j = gCount.value - 1; j >= 0; --j) + { + if (!gCerts.value[j]) + { + missing[missing.length] = gEmailAddresses.value[j]; + } + } + + if (missing.length > 0) + { + var autocompleteLdap = Services.prefs + .getBoolPref("ldap_2.autoComplete.useDirectory"); + + if (autocompleteLdap) + { + var autocompleteDirectory = null; + if (params.currentIdentity.overrideGlobalPref) { + autocompleteDirectory = params.currentIdentity.directoryServer; + } else { + autocompleteDirectory = Services.prefs + .getCharPref("ldap_2.autoComplete.directoryServer"); + } + + if (autocompleteDirectory) + { + window.openDialog('chrome://messenger-smime/content/certFetchingStatus.xul', + '', + 'chrome,resizable=1,modal=1,dialog=1', + autocompleteDirectory, + missing + ); + } + } + } + } + + if (gBundle) + { + var yes_string = gBundle.getString("StatusYes"); + var no_string = gBundle.getString("StatusNo"); + var not_possible_string = gBundle.getString("StatusNotPossible"); + + var signed_element = document.getElementById("signed"); + var encrypted_element = document.getElementById("encrypted"); + + if (params.smFields.requireEncryptMessage) + { + if (params.isEncryptionCertAvailable && canEncrypt.value) + { + encrypted_element.value = yes_string; + } + else + { + encrypted_element.value = not_possible_string; + } + } + else + { + encrypted_element.value = no_string; + } + + if (params.smFields.signMessage) + { + if (params.isSigningCertAvailable) + { + signed_element.value = yes_string; + } + else + { + signed_element.value = not_possible_string; + } + } + else + { + signed_element.value = no_string; + } + } + + var imax = gCount.value; + + for (var i = 0; i < imax; ++i) + { + var listitem = document.createElement("listitem"); + + listitem.appendChild(createCell(gEmailAddresses.value[i])); + + if (!gCerts.value[i]) + { + listitem.appendChild(createCell(gBundle.getString("StatusNotFound"))); + } + else + { + listitem.appendChild(createCell(getStatusExplanation(gCertStatusSummaries.value[i]))); + listitem.appendChild(createCell(gCertIssuedInfos.value[i])); + listitem.appendChild(createCell(gCertExpiresInfos.value[i])); + } + + gListBox.appendChild(listitem); + } +} + +function onSelectionChange(event) +{ + gViewButton.disabled = !(gListBox.selectedItems.length == 1 && + certForRow(gListBox.selectedIndex)); +} + +function viewCertHelper(parent, cert) { + var cd = Components.classes[nsCertificateDialogs].getService(nsICertificateDialogs); + cd.viewCert(parent, cert); +} + +function certForRow(aRowIndex) { + return gCerts.value[aRowIndex]; +} + +function viewSelectedCert() +{ + if (!gViewButton.disabled) + viewCertHelper(window, certForRow(gListBox.selectedIndex)); +} + +function doHelpButton() +{ + openHelp('compose_security', 'chrome://communicator/locale/help/suitehelp.rdf'); +} + +function createCell(label) +{ + var cell = document.createElement("listcell"); + cell.setAttribute("label", label) + return cell; +} diff --git a/mailnews/extensions/smime/content/msgCompSecurityInfo.xul b/mailnews/extensions/smime/content/msgCompSecurityInfo.xul new file mode 100644 index 000000000..c8769d621 --- /dev/null +++ b/mailnews/extensions/smime/content/msgCompSecurityInfo.xul @@ -0,0 +1,68 @@ +<?xml version="1.0"?> +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> + +<?xml-stylesheet href="chrome://global/skin/" type="text/css"?> +<?xml-stylesheet href="chrome://messenger/skin/smime/msgCompSecurityInfo.css" type="text/css"?> + +<!DOCTYPE dialog SYSTEM "chrome://messenger-smime/locale/msgCompSecurityInfo.dtd"> + +<dialog id="msgCompSecurityInfo" title="&title.label;" + xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul" + style="width: 50em;" + persist="width height" + buttons="accept" + onload="onLoad();"> + + <script type="application/javascript" src="chrome://messenger-smime/content/msgCompSecurityInfo.js"/> + + <stringbundle id="bundle_smime_comp_info" src="chrome://messenger-smime/locale/msgCompSecurityInfo.properties"/> + + <description>&subject.plaintextWarning;</description> + <separator class="thin"/> + <description>&status.heading;</description> + <grid> + <columns> + <column/> + <column/> + </columns> + <rows> + <row> + <label value="&status.signed;"/> + <label id="signed"/> + </row> + <row> + <label value="&status.encrypted;"/> + <label id="encrypted"/> + </row> + </rows> + </grid> + + <separator class="thin"/> + <label value="&status.certificates;" control="infolist"/> + + <listbox id="infolist" flex="1" + onselect="onSelectionChange(event);"> + <listcols> + <listcol flex="3" width="0"/> + <splitter class="tree-splitter"/> + <listcol flex="1" width="0"/> + <splitter class="tree-splitter"/> + <listcol flex="2" width="0"/> + <splitter class="tree-splitter"/> + <listcol flex="2" width="0"/> + </listcols> + <listhead> + <listheader label="&tree.recipient;"/> + <listheader label="&tree.status;"/> + <listheader label="&tree.issuedDate;"/> + <listheader label="&tree.expiresDate;"/> + </listhead> + </listbox> + <hbox pack="start"> + <button id="viewCertButton" disabled="true" + label="&view.label;" accesskey="&view.accesskey;" + oncommand="viewSelectedCert();"/> + </hbox> +</dialog> diff --git a/mailnews/extensions/smime/content/msgHdrViewSMIMEOverlay.js b/mailnews/extensions/smime/content/msgHdrViewSMIMEOverlay.js new file mode 100644 index 000000000..2d9469d6c --- /dev/null +++ b/mailnews/extensions/smime/content/msgHdrViewSMIMEOverlay.js @@ -0,0 +1,264 @@ +/* -*- Mode: Java; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +var gSignedUINode = null; +var gEncryptedUINode = null; +var gSMIMEContainer = null; +var gStatusBar = null; +var gSignedStatusPanel = null; +var gEncryptedStatusPanel = null; + +var gEncryptedURIService = null; +var gMyLastEncryptedURI = null; + +var gSMIMEBundle = null; +// var gBrandBundle; -- defined in mailWindow.js + +// manipulates some globals from msgReadSMIMEOverlay.js + +var nsICMSMessageErrors = Components.interfaces.nsICMSMessageErrors; + +/// Get the necko URL for the message URI. +function neckoURLForMessageURI(aMessageURI) +{ + let msgSvc = Components.classes["@mozilla.org/messenger;1"] + .createInstance(Components.interfaces.nsIMessenger) + .messageServiceFromURI(aMessageURI); + let neckoURI = {}; + msgSvc.GetUrlForUri(aMessageURI, neckoURI, null); + return neckoURI.value.spec; +} + +var smimeHeaderSink = +{ + maxWantedNesting: function() + { + return 1; + }, + + signedStatus: function(aNestingLevel, aSignatureStatus, aSignerCert) + { + if (aNestingLevel > 1) { + // we are not interested + return; + } + + gSignatureStatus = aSignatureStatus; + gSignerCert = aSignerCert; + + gSMIMEContainer.collapsed = false; + gSignedUINode.collapsed = false; + gSignedStatusPanel.collapsed = false; + + switch (aSignatureStatus) { + case nsICMSMessageErrors.SUCCESS: + gSignedUINode.setAttribute("signed", "ok"); + gStatusBar.setAttribute("signed", "ok"); + break; + + case nsICMSMessageErrors.VERIFY_NOT_YET_ATTEMPTED: + gSignedUINode.setAttribute("signed", "unknown"); + gStatusBar.setAttribute("signed", "unknown"); + break; + + case nsICMSMessageErrors.VERIFY_CERT_WITHOUT_ADDRESS: + case nsICMSMessageErrors.VERIFY_HEADER_MISMATCH: + gSignedUINode.setAttribute("signed", "mismatch"); + gStatusBar.setAttribute("signed", "mismatch"); + break; + + default: + gSignedUINode.setAttribute("signed", "notok"); + gStatusBar.setAttribute("signed", "notok"); + break; + } + }, + + encryptionStatus: function(aNestingLevel, aEncryptionStatus, aRecipientCert) + { + if (aNestingLevel > 1) { + // we are not interested + return; + } + + gEncryptionStatus = aEncryptionStatus; + gEncryptionCert = aRecipientCert; + + gSMIMEContainer.collapsed = false; + gEncryptedUINode.collapsed = false; + gEncryptedStatusPanel.collapsed = false; + + if (nsICMSMessageErrors.SUCCESS == aEncryptionStatus) + { + gEncryptedUINode.setAttribute("encrypted", "ok"); + gStatusBar.setAttribute("encrypted", "ok"); + } + else + { + gEncryptedUINode.setAttribute("encrypted", "notok"); + gStatusBar.setAttribute("encrypted", "notok"); + } + + if (gEncryptedURIService) + { + // Remember the message URI and the corresponding necko URI. + gMyLastEncryptedURI = GetLoadedMessage(); + gEncryptedURIService.rememberEncrypted(gMyLastEncryptedURI); + gEncryptedURIService.rememberEncrypted( + neckoURLForMessageURI(gMyLastEncryptedURI)); + } + + switch (aEncryptionStatus) + { + case nsICMSMessageErrors.SUCCESS: + case nsICMSMessageErrors.ENCRYPT_INCOMPLETE: + break; + default: + var brand = gBrandBundle.getString("brandShortName"); + var title = gSMIMEBundle.getString("CantDecryptTitle").replace(/%brand%/g, brand); + var body = gSMIMEBundle.getString("CantDecryptBody").replace(/%brand%/g, brand); + + // insert our message + msgWindow.displayHTMLInMessagePane(title, + "<html>\n" + + "<body bgcolor=\"#fafaee\">\n" + + "<center><br><br><br>\n" + + "<table>\n" + + "<tr><td>\n" + + "<center><strong><font size=\"+3\">\n" + + title+"</font></center><br>\n" + + body+"\n" + + "</td></tr></table></center></body></html>", false); + } + }, + + QueryInterface : function(iid) + { + if (iid.equals(Components.interfaces.nsIMsgSMIMEHeaderSink) || iid.equals(Components.interfaces.nsISupports)) + return this; + throw Components.results.NS_NOINTERFACE; + } +}; + +function forgetEncryptedURI() +{ + if (gMyLastEncryptedURI && gEncryptedURIService) + { + gEncryptedURIService.forgetEncrypted(gMyLastEncryptedURI); + gEncryptedURIService.forgetEncrypted( + neckoURLForMessageURI(gMyLastEncryptedURI)); + gMyLastEncryptedURI = null; + } +} + +function onSMIMEStartHeaders() +{ + gEncryptionStatus = -1; + gSignatureStatus = -1; + + gSignerCert = null; + gEncryptionCert = null; + + gSMIMEContainer.collapsed = true; + + gSignedUINode.collapsed = true; + gSignedUINode.removeAttribute("signed"); + gSignedStatusPanel.collapsed = true; + gStatusBar.removeAttribute("signed"); + + gEncryptedUINode.collapsed = true; + gEncryptedUINode.removeAttribute("encrypted"); + gEncryptedStatusPanel.collapsed = true; + gStatusBar.removeAttribute("encrypted"); + + forgetEncryptedURI(); +} + +function onSMIMEEndHeaders() +{} + +function onSmartCardChange() +{ + // only reload encrypted windows + if (gMyLastEncryptedURI && gEncryptionStatus != -1) + ReloadMessage(); +} + +function msgHdrViewSMIMEOnLoad(event) +{ + window.crypto.enableSmartCardEvents = true; + document.addEventListener("smartcard-insert", onSmartCardChange, false); + document.addEventListener("smartcard-remove", onSmartCardChange, false); + if (!gSMIMEBundle) + gSMIMEBundle = document.getElementById("bundle_read_smime"); + + // we want to register our security header sink as an opaque nsISupports + // on the msgHdrSink used by mail..... + msgWindow.msgHeaderSink.securityInfo = smimeHeaderSink; + + gSignedUINode = document.getElementById('signedHdrIcon'); + gEncryptedUINode = document.getElementById('encryptedHdrIcon'); + gSMIMEContainer = document.getElementById('smimeBox'); + gStatusBar = document.getElementById('status-bar'); + gSignedStatusPanel = document.getElementById('signed-status'); + gEncryptedStatusPanel = document.getElementById('encrypted-status'); + + // add ourself to the list of message display listeners so we get notified when we are about to display a + // message. + var listener = {}; + listener.onStartHeaders = onSMIMEStartHeaders; + listener.onEndHeaders = onSMIMEEndHeaders; + gMessageListeners.push(listener); + + gEncryptedURIService = + Components.classes["@mozilla.org/messenger-smime/smime-encrypted-uris-service;1"] + .getService(Components.interfaces.nsIEncryptedSMIMEURIsService); +} + +function msgHdrViewSMIMEOnUnload(event) +{ + window.crypto.enableSmartCardEvents = false; + document.removeEventListener("smartcard-insert", onSmartCardChange, false); + document.removeEventListener("smartcard-remove", onSmartCardChange, false); + forgetEncryptedURI(); + removeEventListener("messagepane-loaded", msgHdrViewSMIMEOnLoad, true); + removeEventListener("messagepane-unloaded", msgHdrViewSMIMEOnUnload, true); + removeEventListener("messagepane-hide", msgHdrViewSMIMEOnMessagePaneHide, true); + removeEventListener("messagepane-unhide", msgHdrViewSMIMEOnMessagePaneUnhide, true); +} + +function msgHdrViewSMIMEOnMessagePaneHide() +{ + gSMIMEContainer.collapsed = true; + gSignedUINode.collapsed = true; + gSignedStatusPanel.collapsed = true; + gEncryptedUINode.collapsed = true; + gEncryptedStatusPanel.collapsed = true; +} + +function msgHdrViewSMIMEOnMessagePaneUnhide() +{ + if (gEncryptionStatus != -1 || gSignatureStatus != -1) + { + gSMIMEContainer.collapsed = false; + + if (gSignatureStatus != -1) + { + gSignedUINode.collapsed = false; + gSignedStatusPanel.collapsed = false; + } + + if (gEncryptionStatus != -1) + { + gEncryptedUINode.collapsed = false; + gEncryptedStatusPanel.collapsed = false; + } + } +} + +addEventListener('messagepane-loaded', msgHdrViewSMIMEOnLoad, true); +addEventListener('messagepane-unloaded', msgHdrViewSMIMEOnUnload, true); +addEventListener('messagepane-hide', msgHdrViewSMIMEOnMessagePaneHide, true); +addEventListener('messagepane-unhide', msgHdrViewSMIMEOnMessagePaneUnhide, true); diff --git a/mailnews/extensions/smime/content/msgHdrViewSMIMEOverlay.xul b/mailnews/extensions/smime/content/msgHdrViewSMIMEOverlay.xul new file mode 100644 index 000000000..957d2a15b --- /dev/null +++ b/mailnews/extensions/smime/content/msgHdrViewSMIMEOverlay.xul @@ -0,0 +1,29 @@ +<?xml version="1.0"?> +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> + +<?xml-stylesheet href="chrome://messenger/skin/smime/msgHdrViewSMIMEOverlay.css" type="text/css"?> + +<overlay xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul"> + + <script type="application/javascript" src="chrome://messenger-smime/content/msgHdrViewSMIMEOverlay.js"/> +<!-- These stringbundles are already defined in msgReadSMIMEOverlay.xul! + <stringbundleset id="stringbundleset"> + <stringbundle id="bundle_read_smime" src="chrome://messenger-smime/locale/msgReadSMIMEOverlay.properties"/> + <stringbundle id="bundle_brand" src="chrome://branding/locale/brand.properties"/> + </stringbundleset> +--> + + <hbox id="expandedHeaderView"> + <vbox id="smimeBox" insertafter="expandedHeaders" collapsed="true"> + <spacer flex="1"/> + <image id="signedHdrIcon" + onclick="showMessageReadSecurityInfo();" collapsed="true"/> + <image id="encryptedHdrIcon" + onclick="showMessageReadSecurityInfo();" collapsed="true"/> + <spacer flex="1"/> + </vbox> + </hbox> +</overlay> + diff --git a/mailnews/extensions/smime/content/msgReadSMIMEOverlay.js b/mailnews/extensions/smime/content/msgReadSMIMEOverlay.js new file mode 100644 index 000000000..ab362d418 --- /dev/null +++ b/mailnews/extensions/smime/content/msgReadSMIMEOverlay.js @@ -0,0 +1,102 @@ +/* -*- Mode: Java; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +Components.utils.import("resource://gre/modules/Services.jsm"); + +var gEncryptionStatus = -1; +var gSignatureStatus = -1; +var gSignerCert = null; +var gEncryptionCert = null; + +addEventListener("load", smimeReadOnLoad, {capture: false, once: true}); + +function smimeReadOnLoad() +{ + top.controllers.appendController(SecurityController); + + addEventListener("unload", smimeReadOnUnload, {capture: false, once: true}); +} + +function smimeReadOnUnload() +{ + top.controllers.removeController(SecurityController); +} + +function showImapSignatureUnknown() +{ + let readSmimeBundle = document.getElementById("bundle_read_smime"); + let brandBundle = document.getElementById("bundle_brand"); + if (!readSmimeBundle || !brandBundle) + return; + + if (Services.prompt.confirm(window, brandBundle.getString("brandShortName"), + readSmimeBundle.getString("ImapOnDemand"))) + { + gDBView.reloadMessageWithAllParts(); + } +} + +function showMessageReadSecurityInfo() +{ + let gSignedUINode = document.getElementById("signedHdrIcon"); + if (gSignedUINode && gSignedUINode.getAttribute("signed") == "unknown") + { + showImapSignatureUnknown(); + return; + } + + let params = Components.classes["@mozilla.org/embedcomp/dialogparam;1"] + .createInstance(Components.interfaces.nsIDialogParamBlock); + params.objects = Components.classes["@mozilla.org/array;1"] + .createInstance(Components.interfaces.nsIMutableArray); + // Append even if null... the receiver must handle that. + params.objects.appendElement(gSignerCert, false); + params.objects.appendElement(gEncryptionCert, false); + + // int array starts with index 0, but that is used for window exit status + params.SetInt(1, gSignatureStatus); + params.SetInt(2, gEncryptionStatus); + + window.openDialog("chrome://messenger-smime/content/msgReadSecurityInfo.xul", + "", "chrome,resizable,modal,dialog,centerscreen", params); +} + +var SecurityController = +{ + supportsCommand: function(command) + { + switch (command) + { + case "cmd_viewSecurityStatus": + return true; + + default: + return false; + } + }, + + isCommandEnabled: function(command) + { + switch (command) + { + case "cmd_viewSecurityStatus": + if (document.documentElement.getAttribute('windowtype') == "mail:messageWindow") + return GetNumSelectedMessages() > 0; + + if (GetNumSelectedMessages() > 0 && gDBView) + { + let enabled = {value: false}; + let checkStatus = {}; + gDBView.getCommandStatus(nsMsgViewCommandType.cmdRequiringMsgBody, + enabled, checkStatus); + return enabled.value; + } + // else: fall through. + + default: + return false; + } + } +}; diff --git a/mailnews/extensions/smime/content/msgReadSMIMEOverlay.xul b/mailnews/extensions/smime/content/msgReadSMIMEOverlay.xul new file mode 100644 index 000000000..a55828c0f --- /dev/null +++ b/mailnews/extensions/smime/content/msgReadSMIMEOverlay.xul @@ -0,0 +1,34 @@ +<?xml version="1.0"?> +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> + +<?xml-stylesheet href="chrome://messenger/skin/smime/msgReadSMIMEOverlay.css" type="text/css"?> + +<!DOCTYPE overlay SYSTEM "chrome://messenger-smime/locale/msgReadSMIMEOverlay.dtd"> + +<overlay xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul"> + + <script type="application/javascript" src="chrome://messenger-smime/content/msgReadSMIMEOverlay.js"/> + + <commandset id="mailViewMenuItems"> + <command id="cmd_viewSecurityStatus" oncommand="showMessageReadSecurityInfo();" disabled="true"/> + </commandset> + + <menupopup id="menu_View_Popup"> + <menuitem insertafter="pageSourceMenuItem" label="&menu_securityStatus.label;" + accesskey="&menu_securityStatus.accesskey;" command="cmd_viewSecurityStatus"/> + </menupopup> + + <statusbar id="status-bar"> + <statusbarpanel insertbefore="offline-status" class="statusbarpanel-iconic" + id="signed-status" collapsed="true" oncommand="showMessageReadSecurityInfo();"/> + <statusbarpanel insertbefore="offline-status" class="statusbarpanel-iconic" + id="encrypted-status" collapsed="true" oncommand="showMessageReadSecurityInfo();"/> + <stringbundle id="bundle_read_smime" src="chrome://messenger-smime/locale/msgReadSMIMEOverlay.properties"/> +<!-- This stringbundle is already defined on top window level! + <stringbundle id="bundle_brand" src="chrome://branding/locale/brand.properties"/> +--> + </statusbar> + +</overlay> diff --git a/mailnews/extensions/smime/content/msgReadSecurityInfo.js b/mailnews/extensions/smime/content/msgReadSecurityInfo.js new file mode 100644 index 000000000..310cfc18a --- /dev/null +++ b/mailnews/extensions/smime/content/msgReadSecurityInfo.js @@ -0,0 +1,232 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +var nsIDialogParamBlock = Components.interfaces.nsIDialogParamBlock; +var nsIX509Cert = Components.interfaces.nsIX509Cert; +var nsICMSMessageErrors = Components.interfaces.nsICMSMessageErrors; +var nsICertificateDialogs = Components.interfaces.nsICertificateDialogs; +var nsCertificateDialogs = "@mozilla.org/nsCertificateDialogs;1" + +var gSignerCert = null; +var gEncryptionCert = null; + +var gSignatureStatus = -1; +var gEncryptionStatus = -1; + +function setText(id, value) { + var element = document.getElementById(id); + if (!element) + return; + if (element.hasChildNodes()) + element.firstChild.remove(); + var textNode = document.createTextNode(value); + element.appendChild(textNode); +} + +function onLoad() +{ + var paramBlock = window.arguments[0].QueryInterface(nsIDialogParamBlock); + paramBlock.objects.QueryInterface(Components.interfaces.nsIMutableArray); + try { + gSignerCert = paramBlock.objects.queryElementAt(0, nsIX509Cert); + } catch(e) { } // maybe null + try { + gEncryptionCert = paramBlock.objects.queryElementAt(1, nsIX509Cert); + } catch(e) { } // maybe null + + gSignatureStatus = paramBlock.GetInt(1); + gEncryptionStatus = paramBlock.GetInt(2); + + var bundle = document.getElementById("bundle_smime_read_info"); + + if (bundle) { + var sigInfoLabel = null; + var sigInfoHeader = null; + var sigInfo = null; + var sigInfo_clueless = false; + + switch (gSignatureStatus) { + case -1: + case nsICMSMessageErrors.VERIFY_NOT_SIGNED: + sigInfoLabel = "SINoneLabel"; + sigInfo = "SINone"; + break; + + case nsICMSMessageErrors.SUCCESS: + sigInfoLabel = "SIValidLabel"; + sigInfo = "SIValid"; + break; + + + case nsICMSMessageErrors.VERIFY_BAD_SIGNATURE: + case nsICMSMessageErrors.VERIFY_DIGEST_MISMATCH: + sigInfoLabel = "SIInvalidLabel"; + sigInfoHeader = "SIInvalidHeader"; + sigInfo = "SIContentAltered"; + break; + + case nsICMSMessageErrors.VERIFY_UNKNOWN_ALGO: + case nsICMSMessageErrors.VERIFY_UNSUPPORTED_ALGO: + sigInfoLabel = "SIInvalidLabel"; + sigInfoHeader = "SIInvalidHeader"; + sigInfo = "SIInvalidCipher"; + break; + + case nsICMSMessageErrors.VERIFY_HEADER_MISMATCH: + sigInfoLabel = "SIPartiallyValidLabel"; + sigInfoHeader = "SIPartiallyValidHeader"; + sigInfo = "SIHeaderMismatch"; + break; + + case nsICMSMessageErrors.VERIFY_CERT_WITHOUT_ADDRESS: + sigInfoLabel = "SIPartiallyValidLabel"; + sigInfoHeader = "SIPartiallyValidHeader"; + sigInfo = "SICertWithoutAddress"; + break; + + case nsICMSMessageErrors.VERIFY_UNTRUSTED: + sigInfoLabel = "SIInvalidLabel"; + sigInfoHeader = "SIInvalidHeader"; + sigInfo = "SIUntrustedCA"; + // XXX Need to extend to communicate better errors + // might also be: + // SIExpired SIRevoked SINotYetValid SIUnknownCA SIExpiredCA SIRevokedCA SINotYetValidCA + break; + + case nsICMSMessageErrors.VERIFY_NOT_YET_ATTEMPTED: + case nsICMSMessageErrors.GENERAL_ERROR: + case nsICMSMessageErrors.VERIFY_NO_CONTENT_INFO: + case nsICMSMessageErrors.VERIFY_BAD_DIGEST: + case nsICMSMessageErrors.VERIFY_NOCERT: + case nsICMSMessageErrors.VERIFY_ERROR_UNVERIFIED: + case nsICMSMessageErrors.VERIFY_ERROR_PROCESSING: + case nsICMSMessageErrors.VERIFY_MALFORMED_SIGNATURE: + sigInfoLabel = "SIInvalidLabel"; + sigInfoHeader = "SIInvalidHeader"; + sigInfo_clueless = true; + break; + default: + Components.utils.reportError("Unexpected gSignatureStatus: " + + gSignatureStatus); + } + + document.getElementById("signatureLabel").value = + bundle.getString(sigInfoLabel); + + var label; + if (sigInfoHeader) { + label = document.getElementById("signatureHeader"); + label.collapsed = false; + label.value = bundle.getString(sigInfoHeader); + } + + var str; + if (sigInfo) { + str = bundle.getString(sigInfo); + } + else if (sigInfo_clueless) { + str = bundle.getString("SIClueless") + " (" + gSignatureStatus + ")"; + } + setText("signatureExplanation", str); + + var encInfoLabel = null; + var encInfoHeader = null; + var encInfo = null; + var encInfo_clueless = false; + + switch (gEncryptionStatus) { + case -1: + encInfoLabel = "EINoneLabel2"; + encInfo = "EINone"; + break; + + case nsICMSMessageErrors.SUCCESS: + encInfoLabel = "EIValidLabel"; + encInfo = "EIValid"; + break; + + case nsICMSMessageErrors.ENCRYPT_INCOMPLETE: + encInfoLabel = "EIInvalidLabel"; + encInfo = "EIContentAltered"; + break; + + case nsICMSMessageErrors.GENERAL_ERROR: + encInfoLabel = "EIInvalidLabel"; + encInfoHeader = "EIInvalidHeader"; + encInfo_clueless = 1; + break; + default: + Components.utils.reportError("Unexpected gEncryptionStatus: " + + gEncryptionStatus); + } + + document.getElementById("encryptionLabel").value = + bundle.getString(encInfoLabel); + + if (encInfoHeader) { + label = document.getElementById("encryptionHeader"); + label.collapsed = false; + label.value = bundle.getString(encInfoHeader); + } + + if (encInfo) { + str = bundle.getString(encInfo); + } + else if (encInfo_clueless) { + str = bundle.getString("EIClueless"); + } + setText("encryptionExplanation", str); + } + + if (gSignerCert) { + document.getElementById("signatureCert").collapsed = false; + if (gSignerCert.subjectName) { + document.getElementById("signedBy").value = gSignerCert.commonName; + } + if (gSignerCert.emailAddress) { + document.getElementById("signerEmail").value = gSignerCert.emailAddress; + } + if (gSignerCert.issuerName) { + document.getElementById("sigCertIssuedBy").value = gSignerCert.issuerCommonName; + } + } + + if (gEncryptionCert) { + document.getElementById("encryptionCert").collapsed = false; + if (gEncryptionCert.subjectName) { + document.getElementById("encryptedFor").value = gEncryptionCert.commonName; + } + if (gEncryptionCert.emailAddress) { + document.getElementById("recipientEmail").value = gEncryptionCert.emailAddress; + } + if (gEncryptionCert.issuerName) { + document.getElementById("encCertIssuedBy").value = gEncryptionCert.issuerCommonName; + } + } +} + +function viewCertHelper(parent, cert) { + var cd = Components.classes[nsCertificateDialogs].getService(nsICertificateDialogs); + cd.viewCert(parent, cert); +} + +function viewSignatureCert() +{ + if (gSignerCert) { + viewCertHelper(window, gSignerCert); + } +} + +function viewEncryptionCert() +{ + if (gEncryptionCert) { + viewCertHelper(window, gEncryptionCert); + } +} + +function doHelpButton() +{ + openHelp('received_security'); +} diff --git a/mailnews/extensions/smime/content/msgReadSecurityInfo.xul b/mailnews/extensions/smime/content/msgReadSecurityInfo.xul new file mode 100644 index 000000000..8e0a1f5f5 --- /dev/null +++ b/mailnews/extensions/smime/content/msgReadSecurityInfo.xul @@ -0,0 +1,68 @@ +<?xml version="1.0"?> +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> + +<?xml-stylesheet href="chrome://global/skin/" type="text/css"?> +<?xml-stylesheet href="chrome://messenger/skin/smime/msgReadSecurityInfo.css" type="text/css"?> + +<!DOCTYPE dialog SYSTEM "chrome://messenger-smime/locale/msgReadSecurityInfo.dtd"> + +<dialog id="msgReadSecurityInfo" title="&status.label;" + xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul" + style="width: 40em;" + buttons="accept" + onload="onLoad();"> + + <script type="application/javascript" src="chrome://messenger-smime/content/msgReadSecurityInfo.js"/> + + <stringbundle id="bundle_smime_read_info" src="chrome://messenger-smime/locale/msgSecurityInfo.properties"/> + + <vbox flex="1"> + <label id="signatureLabel"/> + <label id="signatureHeader" collapsed="true"/> + <description id="signatureExplanation"/> + <vbox id="signatureCert" collapsed="true"> + <hbox> + <label id="signedByLabel">&signer.name;</label> + <description id="signedBy"/> + </hbox> + <hbox> + <label id="signerEmailLabel">&email.address;</label> + <description id="signerEmail"/> + </hbox> + <hbox> + <label id="sigCertIssuedByLabel">&issuer.name;</label> + <description id="sigCertIssuedBy"/> + </hbox> + <hbox> + <button id="signatureCertView" label="&signatureCert.label;" + oncommand="viewSignatureCert()"/> + </hbox> + </vbox> + + <separator/> + + <label id="encryptionLabel"/> + <label id="encryptionHeader" collapsed="true"/> + <description id="encryptionExplanation"/> + <vbox id="encryptionCert" collapsed="true"> + <hbox> + <label id="encryptedForLabel">&recipient.name;</label> + <description id="encryptedFor"/> + </hbox> + <hbox> + <label id="recipientEmailLabel">&email.address;</label> + <description id="recipientEmail"/> + </hbox> + <hbox> + <label id="encCertIssuedByLabel">&issuer.name;</label> + <description id="encCertIssuedBy"/> + </hbox> + <hbox> + <button id="encryptionCertView" label="&encryptionCert.label;" + oncommand="viewEncryptionCert()"/> + </hbox> + </vbox> + </vbox> +</dialog> diff --git a/mailnews/extensions/smime/content/smime.js b/mailnews/extensions/smime/content/smime.js new file mode 100644 index 000000000..8259ead1a --- /dev/null +++ b/mailnews/extensions/smime/content/smime.js @@ -0,0 +1,14 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + Add any default pref values we want for smime +*/ + +pref("mail.identity.default.encryption_cert_name",""); +pref("mail.identity.default.encryptionpolicy", 0); +pref("mail.identity.default.signing_cert_name", ""); +pref("mail.identity.default.sign_mail", false); + + diff --git a/mailnews/extensions/smime/jar.mn b/mailnews/extensions/smime/jar.mn new file mode 100644 index 000000000..548fef63c --- /dev/null +++ b/mailnews/extensions/smime/jar.mn @@ -0,0 +1,30 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifdef MOZ_SUITE +messenger.jar: +% content messenger-smime %content/messenger-smime/ +% overlay chrome://messenger/content/messengercompose/messengercompose.xul chrome://messenger-smime/content/msgCompSMIMEOverlay.xul +% overlay chrome://messenger/content/msgHdrViewOverlay.xul chrome://messenger-smime/content/msgHdrViewSMIMEOverlay.xul +% overlay chrome://messenger/content/mailWindowOverlay.xul chrome://messenger-smime/content/msgReadSMIMEOverlay.xul +% overlay chrome://messenger/content/am-identity-edit.xul chrome://messenger/content/am-smimeIdentityEditOverlay.xul + content/messenger/am-smime.xul (content/am-smime.xul) + content/messenger/am-smime.js (content/am-smime.js) + content/messenger/am-smimeIdentityEditOverlay.xul (content/am-smimeIdentityEditOverlay.xul) + content/messenger/am-smimeOverlay.xul (content/am-smimeOverlay.xul) + content/messenger/certpicker.js (content/certpicker.js) + content/messenger/certpicker.xul (content/certpicker.xul) + content/messenger-smime/msgCompSMIMEOverlay.js (content/msgCompSMIMEOverlay.js) + content/messenger-smime/msgCompSMIMEOverlay.xul (content/msgCompSMIMEOverlay.xul) + content/messenger-smime/msgReadSMIMEOverlay.js (content/msgReadSMIMEOverlay.js) + content/messenger-smime/msgReadSMIMEOverlay.xul (content/msgReadSMIMEOverlay.xul) + content/messenger-smime/msgHdrViewSMIMEOverlay.xul (content/msgHdrViewSMIMEOverlay.xul) + content/messenger-smime/msgHdrViewSMIMEOverlay.js (content/msgHdrViewSMIMEOverlay.js) + content/messenger-smime/msgCompSecurityInfo.xul (content/msgCompSecurityInfo.xul) + content/messenger-smime/msgCompSecurityInfo.js (content/msgCompSecurityInfo.js) + content/messenger-smime/msgReadSecurityInfo.xul (content/msgReadSecurityInfo.xul) + content/messenger-smime/msgReadSecurityInfo.js (content/msgReadSecurityInfo.js) + content/messenger-smime/certFetchingStatus.xul (content/certFetchingStatus.xul) + content/messenger-smime/certFetchingStatus.js (content/certFetchingStatus.js) +#endif diff --git a/mailnews/extensions/smime/moz.build b/mailnews/extensions/smime/moz.build new file mode 100644 index 000000000..3adf6a50e --- /dev/null +++ b/mailnews/extensions/smime/moz.build @@ -0,0 +1,15 @@ +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +DIRS += [ + 'public', + 'src', +] + +JAR_MANIFESTS += ['jar.mn'] + +JS_PREFERENCE_FILES += [ + 'content/smime.js', +] diff --git a/mailnews/extensions/smime/public/moz.build b/mailnews/extensions/smime/public/moz.build new file mode 100644 index 000000000..b7acbb0b2 --- /dev/null +++ b/mailnews/extensions/smime/public/moz.build @@ -0,0 +1,15 @@ +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +XPIDL_SOURCES += [ + 'nsICertPickDialogs.idl', + 'nsIEncryptedSMIMEURIsSrvc.idl', + 'nsIMsgSMIMECompFields.idl', + 'nsIMsgSMIMEHeaderSink.idl', + 'nsISMimeJSHelper.idl', + 'nsIUserCertPicker.idl', +] + +XPIDL_MODULE = 'msgsmime' diff --git a/mailnews/extensions/smime/public/nsICertPickDialogs.idl b/mailnews/extensions/smime/public/nsICertPickDialogs.idl new file mode 100644 index 000000000..01a7f3712 --- /dev/null +++ b/mailnews/extensions/smime/public/nsICertPickDialogs.idl @@ -0,0 +1,30 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" + +interface nsIInterfaceRequestor; + +/** + * nsICertPickDialogs + * Provides generic UI for choosing a certificate + */ +[scriptable, uuid(51d59b08-1dd2-11b2-ad4a-a51b92f8a184)] +interface nsICertPickDialogs : nsISupports +{ + /** + * PickCertificate + * General purpose certificate prompter + */ + void PickCertificate(in nsIInterfaceRequestor ctx, + [array, size_is(count)] in wstring certNickList, + [array, size_is(count)] in wstring certDetailsList, + in unsigned long count, + inout long selectedIndex, + out boolean canceled); +}; + +%{C++ +#define NS_CERTPICKDIALOGS_CONTRACTID "@mozilla.org/nsCertPickDialogs;1" +%} diff --git a/mailnews/extensions/smime/public/nsIEncryptedSMIMEURIsSrvc.idl b/mailnews/extensions/smime/public/nsIEncryptedSMIMEURIsSrvc.idl new file mode 100644 index 000000000..4b2b7c25c --- /dev/null +++ b/mailnews/extensions/smime/public/nsIEncryptedSMIMEURIsSrvc.idl @@ -0,0 +1,24 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* This is a private interface used exclusively by SMIME. + It provides functionality to the JS UI code, + that is only accessible from C++. +*/ + +#include "nsISupports.idl" + +[scriptable, uuid(f86e55c9-530b-483f-91a7-10fb5b852488)] +interface nsIEncryptedSMIMEURIsService : nsISupports +{ + /// Remember that this URI is encrypted. + void rememberEncrypted(in AUTF8String uri); + + /// Forget that this URI is encrypted. + void forgetEncrypted(in AUTF8String uri); + + /// Check if this URI is encrypted. + boolean isEncrypted(in AUTF8String uri); +}; diff --git a/mailnews/extensions/smime/public/nsIMsgSMIMECompFields.idl b/mailnews/extensions/smime/public/nsIMsgSMIMECompFields.idl new file mode 100644 index 000000000..0688afd76 --- /dev/null +++ b/mailnews/extensions/smime/public/nsIMsgSMIMECompFields.idl @@ -0,0 +1,18 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + + +/* This is a private interface used exclusively by SMIME. NO ONE outside of extensions/smime + should have any knowledge nor should be referring to this interface. +*/ + +#include "nsISupports.idl" + +[scriptable, uuid(338E91F9-5970-4f81-B771-0822A32B1161)] +interface nsIMsgSMIMECompFields : nsISupports +{ + attribute boolean signMessage; + attribute boolean requireEncryptMessage; +}; diff --git a/mailnews/extensions/smime/public/nsIMsgSMIMEHeaderSink.idl b/mailnews/extensions/smime/public/nsIMsgSMIMEHeaderSink.idl new file mode 100644 index 000000000..9bfa41a04 --- /dev/null +++ b/mailnews/extensions/smime/public/nsIMsgSMIMEHeaderSink.idl @@ -0,0 +1,23 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + + +/* This is a private interface used exclusively by SMIME. NO ONE outside of extensions/smime + or the hard coded smime decryption files in mime/src should have any knowledge nor should + be referring to this interface. +*/ + +#include "nsISupports.idl" + +interface nsIX509Cert; + +[scriptable, uuid(25380FA1-E70C-4e82-B0BC-F31C2F41C470)] +interface nsIMsgSMIMEHeaderSink : nsISupports +{ + void signedStatus(in long aNestingLevel, in long aSignatureStatus, in nsIX509Cert aSignerCert); + void encryptionStatus(in long aNestingLevel, in long aEncryptionStatus, in nsIX509Cert aReceipientCert); + + long maxWantedNesting(); // 1 == only info on outermost nesting level wanted +}; diff --git a/mailnews/extensions/smime/public/nsISMimeJSHelper.idl b/mailnews/extensions/smime/public/nsISMimeJSHelper.idl new file mode 100644 index 000000000..c29a77939 --- /dev/null +++ b/mailnews/extensions/smime/public/nsISMimeJSHelper.idl @@ -0,0 +1,73 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* This is a private interface used exclusively by SMIME. + It provides functionality to the JS UI code, + that is only accessible from C++. +*/ + +#include "nsISupports.idl" + +interface nsIMsgCompFields; +interface nsIX509Cert; + +[scriptable, uuid(a54e3c8f-a000-4901-898f-fafb297b1546)] +interface nsISMimeJSHelper : nsISupports +{ + /** + * Obtains detailed information about the certificate availability + * status of email recipients. + * + * @param compFields - Attributes of the composed message + * + * @param count - The number of entries in returned arrays + * + * @param emailAddresses - The list of all recipient email addresses + * + * @param certVerification - The verification/validity status of recipient certs + * + * @param certIssuedInfos - If a recipient cert was found, when has it been issued? + * + * @param certExpiredInfos - If a recipient cert was found, when will it expire? + * + * @param certs - The recipient certificates, which can contain null for not found + * + * @param canEncrypt - whether valid certificates have been found for all recipients + * + * @exception NS_ERROR_FAILURE - unexptected failure + * + * @exception NS_ERROR_OUT_OF_MEMORY - could not create the out list + * + * @exception NS_ERROR_INVALID_ARG + */ + void getRecipientCertsInfo(in nsIMsgCompFields compFields, + out unsigned long count, + [array, size_is(count)] out wstring emailAddresses, + [array, size_is(count)] out long certVerification, + [array, size_is(count)] out wstring certIssuedInfos, + [array, size_is(count)] out wstring certExpiresInfos, + [array, size_is(count)] out nsIX509Cert certs, + out boolean canEncrypt); + + /** + * Obtains a list of email addresses where valid email recipient certificates + * are not yet available. + * + * @param compFields - Attributes of the composed message + * + * @param count - The number of returned email addresses + * + * @param emailAddresses - The list of email addresses without valid certs + * + * @exception NS_ERROR_FAILURE - unexptected failure + * + * @exception NS_ERROR_OUT_OF_MEMORY - could not create the out list + * + * @exception NS_ERROR_INVALID_ARG + */ + void getNoCertAddresses(in nsIMsgCompFields compFields, + out unsigned long count, + [array, size_is(count)] out wstring emailAddresses); +}; diff --git a/mailnews/extensions/smime/public/nsIUserCertPicker.idl b/mailnews/extensions/smime/public/nsIUserCertPicker.idl new file mode 100644 index 000000000..666941c29 --- /dev/null +++ b/mailnews/extensions/smime/public/nsIUserCertPicker.idl @@ -0,0 +1,28 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" + +interface nsIX509Cert; +interface nsIInterfaceRequestor; + +[scriptable, uuid(92396323-23f2-49e0-bf98-a25a725231ab)] +interface nsIUserCertPicker : nsISupports { + nsIX509Cert pickByUsage(in nsIInterfaceRequestor ctx, + in wstring selectedNickname, + in long certUsage, // as defined by NSS enum SECCertUsage + in boolean allowInvalid, + in boolean allowDuplicateNicknames, + in AString emailAddress, // optional - if non-empty, + // skip certificates which + // have at least one e-mail + // address but do not + // include this specific one + out boolean canceled); +}; + +%{C++ +#define NS_CERT_PICKER_CONTRACTID "@mozilla.org/user_cert_picker;1" +%} diff --git a/mailnews/extensions/smime/src/moz.build b/mailnews/extensions/smime/src/moz.build new file mode 100644 index 000000000..f3e888dd4 --- /dev/null +++ b/mailnews/extensions/smime/src/moz.build @@ -0,0 +1,23 @@ +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +SOURCES += [ + 'nsCertPicker.cpp', + 'nsEncryptedSMIMEURIsService.cpp', + 'nsMsgComposeSecure.cpp', + 'nsSMimeJSHelper.cpp', +] + +EXTRA_COMPONENTS += [ + 'smime-service.js', + 'smime-service.manifest', +] + +FINAL_LIBRARY = 'mail' + +LOCAL_INCLUDES += [ + '/mozilla/security/manager/pki', + '/mozilla/security/pkix/include' +] diff --git a/mailnews/extensions/smime/src/nsCertPicker.cpp b/mailnews/extensions/smime/src/nsCertPicker.cpp new file mode 100644 index 000000000..183f9605c --- /dev/null +++ b/mailnews/extensions/smime/src/nsCertPicker.cpp @@ -0,0 +1,471 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsCertPicker.h" + +#include "MainThreadUtils.h" +#include "ScopedNSSTypes.h" +#include "cert.h" +#include "mozilla/RefPtr.h" +#include "nsCOMPtr.h" +#include "nsICertPickDialogs.h" +#include "nsIDOMWindow.h" +#include "nsIDialogParamBlock.h" +#include "nsIInterfaceRequestor.h" +#include "nsIServiceManager.h" +#include "nsIX509CertValidity.h" +#include "nsMemory.h" +#include "nsMsgComposeSecure.h" +#include "nsNSSCertificate.h" +#include "nsNSSComponent.h" +#include "nsNSSDialogHelper.h" +#include "nsNSSHelper.h" +#include "nsNSSShutDown.h" +#include "nsReadableUtils.h" +#include "nsString.h" +#include "pkix/pkixtypes.h" + +using namespace mozilla; + +MOZ_TYPE_SPECIFIC_UNIQUE_PTR_TEMPLATE(UniqueCERTCertNicknames, + CERTCertNicknames, + CERT_FreeNicknames) + +CERTCertNicknames* +getNSSCertNicknamesFromCertList(const UniqueCERTCertList& certList) +{ + static NS_DEFINE_CID(kNSSComponentCID, NS_NSSCOMPONENT_CID); + + nsresult rv; + + nsCOMPtr<nsINSSComponent> nssComponent(do_GetService(kNSSComponentCID, &rv)); + if (NS_FAILED(rv)) + return nullptr; + + nsAutoString expiredString, notYetValidString; + nsAutoString expiredStringLeadingSpace, notYetValidStringLeadingSpace; + + nssComponent->GetPIPNSSBundleString("NicknameExpired", expiredString); + nssComponent->GetPIPNSSBundleString("NicknameNotYetValid", notYetValidString); + + expiredStringLeadingSpace.Append(' '); + expiredStringLeadingSpace.Append(expiredString); + + notYetValidStringLeadingSpace.Append(' '); + notYetValidStringLeadingSpace.Append(notYetValidString); + + NS_ConvertUTF16toUTF8 aUtf8ExpiredString(expiredStringLeadingSpace); + NS_ConvertUTF16toUTF8 aUtf8NotYetValidString(notYetValidStringLeadingSpace); + + return CERT_NicknameStringsFromCertList(certList.get(), + const_cast<char*>(aUtf8ExpiredString.get()), + const_cast<char*>(aUtf8NotYetValidString.get())); +} + +nsresult +FormatUIStrings(nsIX509Cert* cert, const nsAutoString& nickname, + nsAutoString& nickWithSerial, nsAutoString& details) +{ + if (!NS_IsMainThread()) { + NS_ERROR("nsNSSCertificate::FormatUIStrings called off the main thread"); + return NS_ERROR_NOT_SAME_THREAD; + } + + RefPtr<nsMsgComposeSecure> mcs = new nsMsgComposeSecure; + if (!mcs) { + return NS_ERROR_FAILURE; + } + + nsAutoString info; + nsAutoString temp1; + + nickWithSerial.Append(nickname); + + if (NS_SUCCEEDED(mcs->GetSMIMEBundleString(u"CertInfoIssuedFor", info))) { + details.Append(info); + details.Append(char16_t(' ')); + if (NS_SUCCEEDED(cert->GetSubjectName(temp1)) && !temp1.IsEmpty()) { + details.Append(temp1); + } + details.Append(char16_t('\n')); + } + + if (NS_SUCCEEDED(cert->GetSerialNumber(temp1)) && !temp1.IsEmpty()) { + details.AppendLiteral(" "); + if (NS_SUCCEEDED(mcs->GetSMIMEBundleString(u"CertDumpSerialNo", info))) { + details.Append(info); + details.AppendLiteral(": "); + } + details.Append(temp1); + + nickWithSerial.AppendLiteral(" ["); + nickWithSerial.Append(temp1); + nickWithSerial.Append(char16_t(']')); + + details.Append(char16_t('\n')); + } + + nsCOMPtr<nsIX509CertValidity> validity; + nsresult rv = cert->GetValidity(getter_AddRefs(validity)); + if (NS_SUCCEEDED(rv) && validity) { + details.AppendLiteral(" "); + if (NS_SUCCEEDED(mcs->GetSMIMEBundleString(u"CertInfoValid", info))) { + details.Append(info); + } + + if (NS_SUCCEEDED(validity->GetNotBeforeLocalTime(temp1)) && !temp1.IsEmpty()) { + details.Append(char16_t(' ')); + if (NS_SUCCEEDED(mcs->GetSMIMEBundleString(u"CertInfoFrom", info))) { + details.Append(info); + details.Append(char16_t(' ')); + } + details.Append(temp1); + } + + if (NS_SUCCEEDED(validity->GetNotAfterLocalTime(temp1)) && !temp1.IsEmpty()) { + details.Append(char16_t(' ')); + if (NS_SUCCEEDED(mcs->GetSMIMEBundleString(u"CertInfoTo", info))) { + details.Append(info); + details.Append(char16_t(' ')); + } + details.Append(temp1); + } + + details.Append(char16_t('\n')); + } + + if (NS_SUCCEEDED(cert->GetKeyUsages(temp1)) && !temp1.IsEmpty()) { + details.AppendLiteral(" "); + if (NS_SUCCEEDED(mcs->GetSMIMEBundleString(u"CertDumpKeyUsage", info))) { + details.Append(info); + details.AppendLiteral(": "); + } + details.Append(temp1); + details.Append(char16_t('\n')); + } + + UniqueCERTCertificate nssCert(cert->GetCert()); + if (!nssCert) { + return NS_ERROR_FAILURE; + } + + nsAutoString firstEmail; + const char* aWalkAddr; + for (aWalkAddr = CERT_GetFirstEmailAddress(nssCert.get()) + ; + aWalkAddr + ; + aWalkAddr = CERT_GetNextEmailAddress(nssCert.get(), aWalkAddr)) + { + NS_ConvertUTF8toUTF16 email(aWalkAddr); + if (email.IsEmpty()) + continue; + + if (firstEmail.IsEmpty()) { + // If the first email address from the subject DN is also present + // in the subjectAltName extension, GetEmailAddresses() will return + // it twice (as received from NSS). Remember the first address so that + // we can filter out duplicates later on. + firstEmail = email; + + details.AppendLiteral(" "); + if (NS_SUCCEEDED(mcs->GetSMIMEBundleString(u"CertInfoEmail", info))) { + details.Append(info); + details.AppendLiteral(": "); + } + details.Append(email); + } + else { + // Append current address if it's different from the first one. + if (!firstEmail.Equals(email)) { + details.AppendLiteral(", "); + details.Append(email); + } + } + } + + if (!firstEmail.IsEmpty()) { + // We got at least one email address, so we want a newline + details.Append(char16_t('\n')); + } + + if (NS_SUCCEEDED(mcs->GetSMIMEBundleString(u"CertInfoIssuedBy", info))) { + details.Append(info); + details.Append(char16_t(' ')); + + if (NS_SUCCEEDED(cert->GetIssuerName(temp1)) && !temp1.IsEmpty()) { + details.Append(temp1); + } + + details.Append(char16_t('\n')); + } + + if (NS_SUCCEEDED(mcs->GetSMIMEBundleString(u"CertInfoStoredIn", info))) { + details.Append(info); + details.Append(char16_t(' ')); + + if (NS_SUCCEEDED(cert->GetTokenName(temp1)) && !temp1.IsEmpty()) { + details.Append(temp1); + } + } + + // the above produces the following output: + // + // Issued to: $subjectName + // Serial number: $serialNumber + // Valid from: $starting_date to $expiration_date + // Certificate Key usage: $usages + // Email: $address(es) + // Issued by: $issuerName + // Stored in: $token + + return rv; +} + +NS_IMPL_ISUPPORTS(nsCertPicker, nsICertPickDialogs, nsIUserCertPicker) + +nsCertPicker::nsCertPicker() +{ +} + +nsCertPicker::~nsCertPicker() +{ + nsNSSShutDownPreventionLock locker; + if (isAlreadyShutDown()) { + return; + } + + shutdown(ShutdownCalledFrom::Object); +} + +nsresult +nsCertPicker::Init() +{ + nsresult rv; + nsCOMPtr<nsISupports> psm = do_GetService("@mozilla.org/psm;1", &rv); + return rv; +} + +NS_IMETHODIMP +nsCertPicker::PickCertificate(nsIInterfaceRequestor *ctx, + const char16_t **certNickList, + const char16_t **certDetailsList, + uint32_t count, + int32_t *selectedIndex, + bool *canceled) +{ + nsresult rv; + uint32_t i; + + *canceled = false; + + nsCOMPtr<nsIDialogParamBlock> block = + do_CreateInstance(NS_DIALOGPARAMBLOCK_CONTRACTID); + if (!block) return NS_ERROR_FAILURE; + + block->SetNumberStrings(1+count*2); + + for (i = 0; i < count; i++) { + rv = block->SetString(i, certNickList[i]); + if (NS_FAILED(rv)) return rv; + } + + for (i = 0; i < count; i++) { + rv = block->SetString(i+count, certDetailsList[i]); + if (NS_FAILED(rv)) return rv; + } + + rv = block->SetInt(0, count); + if (NS_FAILED(rv)) return rv; + + rv = block->SetInt(1, *selectedIndex); + if (NS_FAILED(rv)) return rv; + + rv = nsNSSDialogHelper::openDialog(nullptr, + "chrome://messenger/content/certpicker.xul", + block); + if (NS_FAILED(rv)) return rv; + + int32_t status; + + rv = block->GetInt(0, &status); + if (NS_FAILED(rv)) return rv; + + *canceled = (status == 0)?true:false; + if (!*canceled) { + rv = block->GetInt(1, selectedIndex); + } + return rv; +} + +NS_IMETHODIMP nsCertPicker::PickByUsage(nsIInterfaceRequestor *ctx, + const char16_t *selectedNickname, + int32_t certUsage, + bool allowInvalid, + bool allowDuplicateNicknames, + const nsAString &emailAddress, + bool *canceled, + nsIX509Cert **_retval) +{ + nsNSSShutDownPreventionLock locker; + if (isAlreadyShutDown()) { + return NS_ERROR_NOT_AVAILABLE; + } + + int32_t selectedIndex = -1; + bool selectionFound = false; + char16_t **certNicknameList = nullptr; + char16_t **certDetailsList = nullptr; + CERTCertListNode* node = nullptr; + nsresult rv = NS_OK; + + { + // Iterate over all certs. This assures that user is logged in to all hardware tokens. + nsCOMPtr<nsIInterfaceRequestor> ctx = new PipUIContext(); + UniqueCERTCertList allcerts(PK11_ListCerts(PK11CertListUnique, ctx)); + } + + /* find all user certs that are valid for the specified usage */ + /* note that we are allowing expired certs in this list */ + UniqueCERTCertList certList( + CERT_FindUserCertsByUsage(CERT_GetDefaultCertDB(), + (SECCertUsage)certUsage, + !allowDuplicateNicknames, + !allowInvalid, + ctx)); + if (!certList) { + return NS_ERROR_NOT_AVAILABLE; + } + + /* if a (non-empty) emailAddress argument is supplied to PickByUsage, */ + /* remove non-matching certificates from the candidate list */ + + if (!emailAddress.IsEmpty()) { + node = CERT_LIST_HEAD(certList); + while (!CERT_LIST_END(node, certList)) { + /* if the cert has at least one e-mail address, check if suitable */ + if (CERT_GetFirstEmailAddress(node->cert)) { + RefPtr<nsNSSCertificate> tempCert(nsNSSCertificate::Create(node->cert)); + bool match = false; + rv = tempCert->ContainsEmailAddress(emailAddress, &match); + if (NS_FAILED(rv)) { + return rv; + } + if (!match) { + /* doesn't contain the specified address, so remove from the list */ + CERTCertListNode* freenode = node; + node = CERT_LIST_NEXT(node); + CERT_RemoveCertListNode(freenode); + continue; + } + } + node = CERT_LIST_NEXT(node); + } + } + + UniqueCERTCertNicknames nicknames(getNSSCertNicknamesFromCertList(certList)); + if (!nicknames) { + return NS_ERROR_NOT_AVAILABLE; + } + + certNicknameList = (char16_t **)moz_xmalloc(sizeof(char16_t *) * nicknames->numnicknames); + certDetailsList = (char16_t **)moz_xmalloc(sizeof(char16_t *) * nicknames->numnicknames); + + if (!certNicknameList || !certDetailsList) { + free(certNicknameList); + free(certDetailsList); + return NS_ERROR_OUT_OF_MEMORY; + } + + int32_t CertsToUse; + + for (CertsToUse = 0, node = CERT_LIST_HEAD(certList.get()); + !CERT_LIST_END(node, certList.get()) && + CertsToUse < nicknames->numnicknames; + node = CERT_LIST_NEXT(node) + ) + { + RefPtr<nsNSSCertificate> tempCert(nsNSSCertificate::Create(node->cert)); + + if (tempCert) { + + nsAutoString i_nickname(NS_ConvertUTF8toUTF16(nicknames->nicknames[CertsToUse])); + nsAutoString nickWithSerial; + nsAutoString details; + + if (!selectionFound) { + /* for the case when selectedNickname refers to a bare nickname */ + if (i_nickname == nsDependentString(selectedNickname)) { + selectedIndex = CertsToUse; + selectionFound = true; + } + } + + if (NS_SUCCEEDED(FormatUIStrings(tempCert, i_nickname, nickWithSerial, + details))) { + certNicknameList[CertsToUse] = ToNewUnicode(nickWithSerial); + certDetailsList[CertsToUse] = ToNewUnicode(details); + if (!selectionFound) { + /* for the case when selectedNickname refers to nickname + serial */ + if (nickWithSerial == nsDependentString(selectedNickname)) { + selectedIndex = CertsToUse; + selectionFound = true; + } + } + } + else { + certNicknameList[CertsToUse] = nullptr; + certDetailsList[CertsToUse] = nullptr; + } + + ++CertsToUse; + } + } + + if (CertsToUse) { + nsCOMPtr<nsICertPickDialogs> dialogs; + rv = getNSSDialogs(getter_AddRefs(dialogs), NS_GET_IID(nsICertPickDialogs), + NS_CERTPICKDIALOGS_CONTRACTID); + + if (NS_SUCCEEDED(rv)) { + // Show the cert picker dialog and get the index of the selected cert. + rv = dialogs->PickCertificate(ctx, (const char16_t**)certNicknameList, + (const char16_t**)certDetailsList, + CertsToUse, &selectedIndex, canceled); + } + } + + int32_t i; + for (i = 0; i < CertsToUse; ++i) { + free(certNicknameList[i]); + free(certDetailsList[i]); + } + free(certNicknameList); + free(certDetailsList); + + if (!CertsToUse) { + return NS_ERROR_NOT_AVAILABLE; + } + + if (NS_SUCCEEDED(rv) && !*canceled) { + for (i = 0, node = CERT_LIST_HEAD(certList); + !CERT_LIST_END(node, certList); + ++i, node = CERT_LIST_NEXT(node)) { + + if (i == selectedIndex) { + RefPtr<nsNSSCertificate> cert = nsNSSCertificate::Create(node->cert); + if (!cert) { + rv = NS_ERROR_OUT_OF_MEMORY; + break; + } + + cert.forget(_retval); + break; + } + } + } + + return rv; +} diff --git a/mailnews/extensions/smime/src/nsCertPicker.h b/mailnews/extensions/smime/src/nsCertPicker.h new file mode 100644 index 000000000..e5881f14f --- /dev/null +++ b/mailnews/extensions/smime/src/nsCertPicker.h @@ -0,0 +1,36 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef nsCertPicker_h +#define nsCertPicker_h + +#include "nsICertPickDialogs.h" +#include "nsIUserCertPicker.h" +#include "nsNSSShutDown.h" + +#define NS_CERT_PICKER_CID \ + { 0x735959a1, 0xaf01, 0x447e, { 0xb0, 0x2d, 0x56, 0xe9, 0x68, 0xfa, 0x52, 0xb4 } } + +class nsCertPicker : public nsICertPickDialogs + , public nsIUserCertPicker + , public nsNSSShutDownObject +{ +public: + NS_DECL_ISUPPORTS + NS_DECL_NSICERTPICKDIALOGS + NS_DECL_NSIUSERCERTPICKER + + nsCertPicker(); + + // Nothing to actually release. + virtual void virtualDestroyNSSReference() override {} + + nsresult Init(); + +protected: + virtual ~nsCertPicker(); +}; + +#endif // nsCertPicker_h diff --git a/mailnews/extensions/smime/src/nsEncryptedSMIMEURIsService.cpp b/mailnews/extensions/smime/src/nsEncryptedSMIMEURIsService.cpp new file mode 100644 index 000000000..9276a07a6 --- /dev/null +++ b/mailnews/extensions/smime/src/nsEncryptedSMIMEURIsService.cpp @@ -0,0 +1,36 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsEncryptedSMIMEURIsService.h" + +NS_IMPL_ISUPPORTS(nsEncryptedSMIMEURIsService, nsIEncryptedSMIMEURIsService) + +nsEncryptedSMIMEURIsService::nsEncryptedSMIMEURIsService() +{ +} + +nsEncryptedSMIMEURIsService::~nsEncryptedSMIMEURIsService() +{ +} + +NS_IMETHODIMP nsEncryptedSMIMEURIsService::RememberEncrypted(const nsACString & uri) +{ + // Assuming duplicates are allowed. + mEncryptedURIs.AppendElement(uri); + return NS_OK; +} + +NS_IMETHODIMP nsEncryptedSMIMEURIsService::ForgetEncrypted(const nsACString & uri) +{ + // Assuming, this will only remove one copy of the string, if the array + // contains multiple copies of the same string. + mEncryptedURIs.RemoveElement(uri); + return NS_OK; +} + +NS_IMETHODIMP nsEncryptedSMIMEURIsService::IsEncrypted(const nsACString & uri, bool *_retval) +{ + *_retval = mEncryptedURIs.Contains(uri); + return NS_OK; +} diff --git a/mailnews/extensions/smime/src/nsEncryptedSMIMEURIsService.h b/mailnews/extensions/smime/src/nsEncryptedSMIMEURIsService.h new file mode 100644 index 000000000..429acbf0a --- /dev/null +++ b/mailnews/extensions/smime/src/nsEncryptedSMIMEURIsService.h @@ -0,0 +1,25 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef _nsEncryptedSMIMEURIsService_H_ +#define _nsEncryptedSMIMEURIsService_H_ + +#include "nsIEncryptedSMIMEURIsSrvc.h" +#include "nsTArray.h" +#include "nsStringGlue.h" + +class nsEncryptedSMIMEURIsService : public nsIEncryptedSMIMEURIsService +{ +public: + NS_DECL_ISUPPORTS + NS_DECL_NSIENCRYPTEDSMIMEURISSERVICE + + nsEncryptedSMIMEURIsService(); + +protected: + virtual ~nsEncryptedSMIMEURIsService(); + nsTArray<nsCString> mEncryptedURIs; +}; + +#endif diff --git a/mailnews/extensions/smime/src/nsMsgComposeSecure.cpp b/mailnews/extensions/smime/src/nsMsgComposeSecure.cpp new file mode 100644 index 000000000..55383c828 --- /dev/null +++ b/mailnews/extensions/smime/src/nsMsgComposeSecure.cpp @@ -0,0 +1,1203 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsMsgComposeSecure.h" + +#include <algorithm> + +#include "ScopedNSSTypes.h" +#include "cert.h" +#include "keyhi.h" +#include "mozilla/RefPtr.h" +#include "mozilla/Services.h" +#include "mozilla/mailnews/MimeEncoder.h" +#include "mozilla/mailnews/MimeHeaderParser.h" +#include "msgCore.h" +#include "nsAlgorithm.h" +#include "nsComponentManagerUtils.h" +#include "nsICryptoHash.h" +#include "nsIMimeConverter.h" +#include "nsIMsgCompFields.h" +#include "nsIMsgIdentity.h" +#include "nsIX509CertDB.h" +#include "nsMemory.h" +#include "nsMimeTypes.h" +#include "nsMsgMimeCID.h" +#include "nsNSSComponent.h" +#include "nsServiceManagerUtils.h" +#include "nspr.h" +#include "pkix/Result.h" + +using namespace mozilla::mailnews; +using namespace mozilla; +using namespace mozilla::psm; + +#define MK_MIME_ERROR_WRITING_FILE -1 + +#define SMIME_STRBUNDLE_URL "chrome://messenger/locale/am-smime.properties" + +// It doesn't make sense to encode the message because the message will be +// displayed only if the MUA doesn't support MIME. +// We need to consider what to do in case the server doesn't support 8BITMIME. +// In short, we can't use non-ASCII characters here. +static const char crypto_multipart_blurb[] = "This is a cryptographically signed message in MIME format."; + +static void mime_crypto_write_base64 (void *closure, const char *buf, + unsigned long size); +static nsresult mime_encoder_output_fn(const char *buf, int32_t size, + void *closure); +static nsresult mime_nested_encoder_output_fn(const char *buf, int32_t size, + void *closure); +static nsresult make_multipart_signed_header_string(bool outer_p, + char **header_return, + char **boundary_return, + int16_t hash_type); +static char *mime_make_separator(const char *prefix); + + +static void +GenerateGlobalRandomBytes(unsigned char *buf, int32_t len) +{ + static bool firstTime = true; + + if (firstTime) + { + // Seed the random-number generator with current time so that + // the numbers will be different every time we run. + srand( (unsigned)PR_Now() ); + firstTime = false; + } + + for( int32_t i = 0; i < len; i++ ) + buf[i] = rand() % 10; +} + +char +*mime_make_separator(const char *prefix) +{ + unsigned char rand_buf[13]; + GenerateGlobalRandomBytes(rand_buf, 12); + + return PR_smprintf("------------%s" + "%02X%02X%02X%02X" + "%02X%02X%02X%02X" + "%02X%02X%02X%02X", + prefix, + rand_buf[0], rand_buf[1], rand_buf[2], rand_buf[3], + rand_buf[4], rand_buf[5], rand_buf[6], rand_buf[7], + rand_buf[8], rand_buf[9], rand_buf[10], rand_buf[11]); +} + +// end of copied code which needs fixed.... + +///////////////////////////////////////////////////////////////////////////////////////// +// Implementation of nsMsgSMIMEComposeFields +///////////////////////////////////////////////////////////////////////////////////////// + +NS_IMPL_ISUPPORTS(nsMsgSMIMEComposeFields, nsIMsgSMIMECompFields) + +nsMsgSMIMEComposeFields::nsMsgSMIMEComposeFields() +:mSignMessage(false), mAlwaysEncryptMessage(false) +{ +} + +nsMsgSMIMEComposeFields::~nsMsgSMIMEComposeFields() +{ +} + +NS_IMETHODIMP nsMsgSMIMEComposeFields::SetSignMessage(bool value) +{ + mSignMessage = value; + return NS_OK; +} + +NS_IMETHODIMP nsMsgSMIMEComposeFields::GetSignMessage(bool *_retval) +{ + *_retval = mSignMessage; + return NS_OK; +} + +NS_IMETHODIMP nsMsgSMIMEComposeFields::SetRequireEncryptMessage(bool value) +{ + mAlwaysEncryptMessage = value; + return NS_OK; +} + +NS_IMETHODIMP nsMsgSMIMEComposeFields::GetRequireEncryptMessage(bool *_retval) +{ + *_retval = mAlwaysEncryptMessage; + return NS_OK; +} + +///////////////////////////////////////////////////////////////////////////////////////// +// Implementation of nsMsgComposeSecure +///////////////////////////////////////////////////////////////////////////////////////// + +NS_IMPL_ISUPPORTS(nsMsgComposeSecure, nsIMsgComposeSecure) + +nsMsgComposeSecure::nsMsgComposeSecure() +{ + /* member initializers and constructor code */ + mMultipartSignedBoundary = 0; + mBuffer = 0; + mBufferedBytes = 0; + mHashType = 0; +} + +nsMsgComposeSecure::~nsMsgComposeSecure() +{ + /* destructor code */ + if (mEncryptionContext) { + if (mBufferedBytes) { + mEncryptionContext->Update(mBuffer, mBufferedBytes); + mBufferedBytes = 0; + } + mEncryptionContext->Finish(); + } + + delete [] mBuffer; + + PR_FREEIF(mMultipartSignedBoundary); +} + +NS_IMETHODIMP nsMsgComposeSecure::RequiresCryptoEncapsulation(nsIMsgIdentity * aIdentity, nsIMsgCompFields * aCompFields, bool * aRequiresEncryptionWork) +{ + NS_ENSURE_ARG_POINTER(aRequiresEncryptionWork); + + *aRequiresEncryptionWork = false; + + bool alwaysEncryptMessages = false; + bool signMessage = false; + nsresult rv = ExtractEncryptionState(aIdentity, aCompFields, &signMessage, &alwaysEncryptMessages); + NS_ENSURE_SUCCESS(rv, rv); + + if (alwaysEncryptMessages || signMessage) + *aRequiresEncryptionWork = true; + + return NS_OK; +} + + +nsresult nsMsgComposeSecure::GetSMIMEBundleString(const char16_t *name, + nsString &outString) +{ + outString.Truncate(); + + NS_ENSURE_ARG_POINTER(name); + + NS_ENSURE_TRUE(InitializeSMIMEBundle(), NS_ERROR_FAILURE); + + return mSMIMEBundle->GetStringFromName(name, getter_Copies(outString)); +} + +nsresult +nsMsgComposeSecure:: +SMIMEBundleFormatStringFromName(const char16_t *name, + const char16_t **params, + uint32_t numParams, + char16_t **outString) +{ + NS_ENSURE_ARG_POINTER(name); + + if (!InitializeSMIMEBundle()) + return NS_ERROR_FAILURE; + + return mSMIMEBundle->FormatStringFromName(name, params, + numParams, outString); +} + +bool nsMsgComposeSecure::InitializeSMIMEBundle() +{ + if (mSMIMEBundle) + return true; + + nsCOMPtr<nsIStringBundleService> bundleService = + mozilla::services::GetStringBundleService(); + nsresult rv = bundleService->CreateBundle(SMIME_STRBUNDLE_URL, + getter_AddRefs(mSMIMEBundle)); + NS_ENSURE_SUCCESS(rv, false); + + return true; +} + +void nsMsgComposeSecure::SetError(nsIMsgSendReport *sendReport, const char16_t *bundle_string) +{ + if (!sendReport || !bundle_string) + return; + + if (mErrorAlreadyReported) + return; + + mErrorAlreadyReported = true; + + nsString errorString; + nsresult res = GetSMIMEBundleString(bundle_string, errorString); + if (NS_SUCCEEDED(res) && !errorString.IsEmpty()) + { + sendReport->SetMessage(nsIMsgSendReport::process_Current, + errorString.get(), + true); + } +} + +void nsMsgComposeSecure::SetErrorWithParam(nsIMsgSendReport *sendReport, const char16_t *bundle_string, const char *param) +{ + if (!sendReport || !bundle_string || !param) + return; + + if (mErrorAlreadyReported) + return; + + mErrorAlreadyReported = true; + + nsString errorString; + nsresult res; + const char16_t *params[1]; + + NS_ConvertASCIItoUTF16 ucs2(param); + params[0]= ucs2.get(); + + res = SMIMEBundleFormatStringFromName(bundle_string, + params, + 1, + getter_Copies(errorString)); + + if (NS_SUCCEEDED(res) && !errorString.IsEmpty()) + { + sendReport->SetMessage(nsIMsgSendReport::process_Current, + errorString.get(), + true); + } +} + +nsresult nsMsgComposeSecure::ExtractEncryptionState(nsIMsgIdentity * aIdentity, nsIMsgCompFields * aComposeFields, bool * aSignMessage, bool * aEncrypt) +{ + if (!aComposeFields && !aIdentity) + return NS_ERROR_FAILURE; // kick out...invalid args.... + + NS_ENSURE_ARG_POINTER(aSignMessage); + NS_ENSURE_ARG_POINTER(aEncrypt); + + nsCOMPtr<nsISupports> securityInfo; + if (aComposeFields) + aComposeFields->GetSecurityInfo(getter_AddRefs(securityInfo)); + + if (securityInfo) // if we were given security comp fields, use them..... + { + nsCOMPtr<nsIMsgSMIMECompFields> smimeCompFields = do_QueryInterface(securityInfo); + if (smimeCompFields) + { + smimeCompFields->GetSignMessage(aSignMessage); + smimeCompFields->GetRequireEncryptMessage(aEncrypt); + return NS_OK; + } + } + + // get the default info from the identity.... + int32_t ep = 0; + nsresult testrv = aIdentity->GetIntAttribute("encryptionpolicy", &ep); + if (NS_FAILED(testrv)) { + *aEncrypt = false; + } + else { + *aEncrypt = (ep > 0); + } + + testrv = aIdentity->GetBoolAttribute("sign_mail", aSignMessage); + if (NS_FAILED(testrv)) + { + *aSignMessage = false; + } + return NS_OK; +} + +// Select a hash algorithm to sign message +// based on subject public key type and size. +static nsresult +GetSigningHashFunction(nsIX509Cert *aSigningCert, int16_t *hashType) +{ + // Get the signing certificate + CERTCertificate *scert = nullptr; + if (aSigningCert) { + scert = aSigningCert->GetCert(); + } + if (!scert) { + return NS_ERROR_FAILURE; + } + + UniqueSECKEYPublicKey scertPublicKey(CERT_ExtractPublicKey(scert)); + if (!scertPublicKey) { + return mozilla::MapSECStatus(SECFailure); + } + KeyType subjectPublicKeyType = SECKEY_GetPublicKeyType(scertPublicKey.get()); + + // Get the length of the signature in bits. + unsigned siglen = SECKEY_SignatureLen(scertPublicKey.get()) * 8; + if (!siglen) { + return mozilla::MapSECStatus(SECFailure); + } + + // Select a hash function for signature generation whose security strength + // meets or exceeds the security strength of the public key, using NIST + // Special Publication 800-57, Recommendation for Key Management - Part 1: + // General (Revision 3), where Table 2 specifies the security strength of + // the public key and Table 3 lists acceptable hash functions. (The security + // strength of the hash (for digital signatures) is half the length of the + // output.) + // [SP 800-57 is available at http://csrc.nist.gov/publications/PubsSPs.html.] + if (subjectPublicKeyType == rsaKey) { + // For RSA, siglen is the same as the length of the modulus. + + // SHA-1 provides equivalent security strength for up to 1024 bits + // SHA-256 provides equivalent security strength for up to 3072 bits + + if (siglen > 3072) { + *hashType = nsICryptoHash::SHA512; + } else if (siglen > 1024) { + *hashType = nsICryptoHash::SHA256; + } else { + *hashType = nsICryptoHash::SHA1; + } + } else if (subjectPublicKeyType == dsaKey) { + // For DSA, siglen is twice the length of the q parameter of the key. + // The security strength of the key is half the length (in bits) of + // the q parameter of the key. + + // NSS only supports SHA-1, SHA-224, and SHA-256 for DSA signatures. + // The S/MIME code does not support SHA-224. + + if (siglen >= 512) { // 512-bit signature = 256-bit q parameter + *hashType = nsICryptoHash::SHA256; + } else { + *hashType = nsICryptoHash::SHA1; + } + } else if (subjectPublicKeyType == ecKey) { + // For ECDSA, siglen is twice the length of the field size. The security + // strength of the key is half the length (in bits) of the field size. + + if (siglen >= 1024) { // 1024-bit signature = 512-bit field size + *hashType = nsICryptoHash::SHA512; + } else if (siglen >= 768) { // 768-bit signature = 384-bit field size + *hashType = nsICryptoHash::SHA384; + } else if (siglen >= 512) { // 512-bit signature = 256-bit field size + *hashType = nsICryptoHash::SHA256; + } else { + *hashType = nsICryptoHash::SHA1; + } + } else { + // Unknown key type + *hashType = nsICryptoHash::SHA256; + NS_WARNING("GetSigningHashFunction: Subject public key type unknown."); + } + return NS_OK; +} + +/* void beginCryptoEncapsulation (in nsOutputFileStream aStream, in boolean aEncrypt, in boolean aSign, in string aRecipeints, in boolean aIsDraft); */ +NS_IMETHODIMP nsMsgComposeSecure::BeginCryptoEncapsulation(nsIOutputStream * aStream, + const char * aRecipients, + nsIMsgCompFields * aCompFields, + nsIMsgIdentity * aIdentity, + nsIMsgSendReport *sendReport, + bool aIsDraft) +{ + mErrorAlreadyReported = false; + nsresult rv = NS_OK; + + bool encryptMessages = false; + bool signMessage = false; + ExtractEncryptionState(aIdentity, aCompFields, &signMessage, &encryptMessages); + + if (!signMessage && !encryptMessages) return NS_ERROR_FAILURE; + + mStream = aStream; + mIsDraft = aIsDraft; + + if (encryptMessages && signMessage) + mCryptoState = mime_crypto_signed_encrypted; + else if (encryptMessages) + mCryptoState = mime_crypto_encrypted; + else if (signMessage) + mCryptoState = mime_crypto_clear_signed; + else + PR_ASSERT(0); + + aIdentity->GetUnicharAttribute("signing_cert_name", mSigningCertName); + aIdentity->GetCharAttribute("signing_cert_dbkey", mSigningCertDBKey); + aIdentity->GetUnicharAttribute("encryption_cert_name", mEncryptionCertName); + aIdentity->GetCharAttribute("encryption_cert_dbkey", mEncryptionCertDBKey); + + rv = MimeCryptoHackCerts(aRecipients, sendReport, encryptMessages, signMessage, aIdentity); + if (NS_FAILED(rv)) { + goto FAIL; + } + + if (signMessage && mSelfSigningCert) { + rv = GetSigningHashFunction(mSelfSigningCert, &mHashType); + NS_ENSURE_SUCCESS(rv, rv); + } + + switch (mCryptoState) + { + case mime_crypto_clear_signed: + rv = MimeInitMultipartSigned(true, sendReport); + break; + case mime_crypto_opaque_signed: + PR_ASSERT(0); /* #### no api for this yet */ + rv = NS_ERROR_NOT_IMPLEMENTED; + break; + case mime_crypto_signed_encrypted: + rv = MimeInitEncryption(true, sendReport); + break; + case mime_crypto_encrypted: + rv = MimeInitEncryption(false, sendReport); + break; + case mime_crypto_none: + /* This can happen if mime_crypto_hack_certs() decided to turn off + encryption (by asking the user.) */ + // XXX 1 is not a valid nsresult + rv = static_cast<nsresult>(1); + break; + default: + PR_ASSERT(0); + break; + } + +FAIL: + return rv; +} + +/* void finishCryptoEncapsulation (in boolean aAbort); */ +NS_IMETHODIMP nsMsgComposeSecure::FinishCryptoEncapsulation(bool aAbort, nsIMsgSendReport *sendReport) +{ + nsresult rv = NS_OK; + + if (!aAbort) { + switch (mCryptoState) { + case mime_crypto_clear_signed: + rv = MimeFinishMultipartSigned (true, sendReport); + break; + case mime_crypto_opaque_signed: + PR_ASSERT(0); /* #### no api for this yet */ + rv = NS_ERROR_FAILURE; + break; + case mime_crypto_signed_encrypted: + rv = MimeFinishEncryption (true, sendReport); + break; + case mime_crypto_encrypted: + rv = MimeFinishEncryption (false, sendReport); + break; + default: + PR_ASSERT(0); + rv = NS_ERROR_FAILURE; + break; + } + } + return rv; +} + +nsresult nsMsgComposeSecure::MimeInitMultipartSigned(bool aOuter, nsIMsgSendReport *sendReport) +{ + /* First, construct and write out the multipart/signed MIME header data. + */ + nsresult rv = NS_OK; + char *header = 0; + uint32_t L; + + rv = make_multipart_signed_header_string(aOuter, &header, + &mMultipartSignedBoundary, mHashType); + + NS_ENSURE_SUCCESS(rv, rv); + + L = strlen(header); + + if (aOuter){ + /* If this is the outer block, write it to the file. */ + uint32_t n; + rv = mStream->Write(header, L, &n); + if (NS_FAILED(rv) || n < L) { + // XXX This is -1, not an nsresult + rv = static_cast<nsresult>(MK_MIME_ERROR_WRITING_FILE); + } + } else { + /* If this is an inner block, feed it through the crypto stream. */ + rv = MimeCryptoWriteBlock (header, L); + } + + PR_Free(header); + NS_ENSURE_SUCCESS(rv, rv); + + /* Now initialize the crypto library, so that we can compute a hash + on the object which we are signing. + */ + + PR_SetError(0,0); + mDataHash = do_CreateInstance("@mozilla.org/security/hash;1", &rv); + NS_ENSURE_SUCCESS(rv, rv); + + rv = mDataHash->Init(mHashType); + NS_ENSURE_SUCCESS(rv, rv); + + PR_SetError(0,0); + return rv; +} + +nsresult nsMsgComposeSecure::MimeInitEncryption(bool aSign, nsIMsgSendReport *sendReport) +{ + nsresult rv; + nsCOMPtr<nsIStringBundleService> bundleSvc = + mozilla::services::GetStringBundleService(); + NS_ENSURE_TRUE(bundleSvc, NS_ERROR_UNEXPECTED); + + nsCOMPtr<nsIStringBundle> sMIMEBundle; + nsString mime_smime_enc_content_desc; + + bundleSvc->CreateBundle(SMIME_STRBUNDLE_URL, getter_AddRefs(sMIMEBundle)); + + if (!sMIMEBundle) + return NS_ERROR_FAILURE; + + sMIMEBundle->GetStringFromName(u"mime_smimeEncryptedContentDesc", + getter_Copies(mime_smime_enc_content_desc)); + NS_ConvertUTF16toUTF8 enc_content_desc_utf8(mime_smime_enc_content_desc); + + nsCOMPtr<nsIMimeConverter> mimeConverter = + do_GetService(NS_MIME_CONVERTER_CONTRACTID, &rv); + NS_ENSURE_SUCCESS(rv, rv); + nsCString encodedContentDescription; + mimeConverter->EncodeMimePartIIStr_UTF8(enc_content_desc_utf8, false, "UTF-8", + sizeof("Content-Description: "), + nsIMimeConverter::MIME_ENCODED_WORD_SIZE, + encodedContentDescription); + + /* First, construct and write out the opaque-crypto-blob MIME header data. + */ + + char *s = + PR_smprintf("Content-Type: " APPLICATION_PKCS7_MIME + "; name=\"smime.p7m\"; smime-type=enveloped-data" CRLF + "Content-Transfer-Encoding: " ENCODING_BASE64 CRLF + "Content-Disposition: attachment" + "; filename=\"smime.p7m\"" CRLF + "Content-Description: %s" CRLF + CRLF, + encodedContentDescription.get()); + + uint32_t L; + if (!s) return NS_ERROR_OUT_OF_MEMORY; + L = strlen(s); + uint32_t n; + rv = mStream->Write(s, L, &n); + if (NS_FAILED(rv) || n < L) { + return NS_ERROR_FAILURE; + } + PR_Free(s); + s = 0; + + /* Now initialize the crypto library, so that we can filter the object + to be encrypted through it. + */ + + if (!mIsDraft) { + uint32_t numCerts; + mCerts->GetLength(&numCerts); + PR_ASSERT(numCerts > 0); + if (numCerts == 0) return NS_ERROR_FAILURE; + } + + // Initialize the base64 encoder + MOZ_ASSERT(!mCryptoEncoder, "Shouldn't have an encoder already"); + mCryptoEncoder = MimeEncoder::GetBase64Encoder(mime_encoder_output_fn, + this); + + /* Initialize the encrypter (and add the sender's cert.) */ + PR_ASSERT(mSelfEncryptionCert); + PR_SetError(0,0); + mEncryptionCinfo = do_CreateInstance(NS_CMSMESSAGE_CONTRACTID, &rv); + if (NS_FAILED(rv)) return rv; + rv = mEncryptionCinfo->CreateEncrypted(mCerts); + if (NS_FAILED(rv)) { + SetError(sendReport, u"ErrorEncryptMail"); + goto FAIL; + } + + mEncryptionContext = do_CreateInstance(NS_CMSENCODER_CONTRACTID, &rv); + if (NS_FAILED(rv)) return rv; + + if (!mBuffer) { + mBuffer = new char[eBufferSize]; + if (!mBuffer) + return NS_ERROR_OUT_OF_MEMORY; + } + + mBufferedBytes = 0; + + rv = mEncryptionContext->Start(mEncryptionCinfo, mime_crypto_write_base64, mCryptoEncoder); + if (NS_FAILED(rv)) { + SetError(sendReport, u"ErrorEncryptMail"); + goto FAIL; + } + + /* If we're signing, tack a multipart/signed header onto the front of + the data to be encrypted, and initialize the sign-hashing code too. + */ + if (aSign) { + rv = MimeInitMultipartSigned(false, sendReport); + if (NS_FAILED(rv)) goto FAIL; + } + + FAIL: + return rv; +} + +nsresult nsMsgComposeSecure::MimeFinishMultipartSigned (bool aOuter, nsIMsgSendReport *sendReport) +{ + int status; + nsresult rv; + nsCOMPtr<nsICMSMessage> cinfo = do_CreateInstance(NS_CMSMESSAGE_CONTRACTID, &rv); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr<nsICMSEncoder> encoder = do_CreateInstance(NS_CMSENCODER_CONTRACTID, &rv); + NS_ENSURE_SUCCESS(rv, rv); + + char * header = nullptr; + nsCOMPtr<nsIStringBundleService> bundleSvc = + mozilla::services::GetStringBundleService(); + NS_ENSURE_TRUE(bundleSvc, NS_ERROR_UNEXPECTED); + + nsCOMPtr<nsIStringBundle> sMIMEBundle; + nsString mime_smime_sig_content_desc; + + bundleSvc->CreateBundle(SMIME_STRBUNDLE_URL, getter_AddRefs(sMIMEBundle)); + + if (!sMIMEBundle) + return NS_ERROR_FAILURE; + + sMIMEBundle->GetStringFromName(u"mime_smimeSignatureContentDesc", + getter_Copies(mime_smime_sig_content_desc)); + + NS_ConvertUTF16toUTF8 sig_content_desc_utf8(mime_smime_sig_content_desc); + + /* Compute the hash... + */ + + nsAutoCString hashString; + mDataHash->Finish(false, hashString); + + mDataHash = nullptr; + + status = PR_GetError(); + if (status < 0) goto FAIL; + + /* Write out the headers for the signature. + */ + uint32_t L; + header = + PR_smprintf(CRLF + "--%s" CRLF + "Content-Type: " APPLICATION_PKCS7_SIGNATURE + "; name=\"smime.p7s\"" CRLF + "Content-Transfer-Encoding: " ENCODING_BASE64 CRLF + "Content-Disposition: attachment; " + "filename=\"smime.p7s\"" CRLF + "Content-Description: %s" CRLF + CRLF, + mMultipartSignedBoundary, + sig_content_desc_utf8.get()); + + if (!header) { + rv = NS_ERROR_OUT_OF_MEMORY; + goto FAIL; + } + + L = strlen(header); + if (aOuter) { + /* If this is the outer block, write it to the file. */ + uint32_t n; + rv = mStream->Write(header, L, &n); + if (NS_FAILED(rv) || n < L) { + // XXX This is -1, not an nsresult + rv = static_cast<nsresult>(MK_MIME_ERROR_WRITING_FILE); + } + } else { + /* If this is an inner block, feed it through the crypto stream. */ + rv = MimeCryptoWriteBlock (header, L); + } + + PR_Free(header); + + /* Create the signature... + */ + + NS_ASSERTION(mHashType, "Hash function for signature has not been set."); + + PR_ASSERT (mSelfSigningCert); + PR_SetError(0,0); + + rv = cinfo->CreateSigned(mSelfSigningCert, mSelfEncryptionCert, + (unsigned char*)hashString.get(), hashString.Length(), mHashType); + if (NS_FAILED(rv)) { + SetError(sendReport, u"ErrorCanNotSignMail"); + goto FAIL; + } + + // Initialize the base64 encoder for the signature data. + MOZ_ASSERT(!mSigEncoder, "Shouldn't already have a mSigEncoder"); + mSigEncoder = MimeEncoder::GetBase64Encoder( + (aOuter ? mime_encoder_output_fn : mime_nested_encoder_output_fn), this); + + /* Write out the signature. + */ + PR_SetError(0,0); + rv = encoder->Start(cinfo, mime_crypto_write_base64, mSigEncoder); + if (NS_FAILED(rv)) { + SetError(sendReport, u"ErrorCanNotSignMail"); + goto FAIL; + } + + // We're not passing in any data, so no update needed. + rv = encoder->Finish(); + if (NS_FAILED(rv)) { + SetError(sendReport, u"ErrorCanNotSignMail"); + goto FAIL; + } + + // Shut down the sig's base64 encoder. + rv = mSigEncoder->Flush(); + mSigEncoder = nullptr; + if (NS_FAILED(rv)) { + goto FAIL; + } + + /* Now write out the terminating boundary. + */ + { + uint32_t L; + char *header = PR_smprintf(CRLF "--%s--" CRLF, + mMultipartSignedBoundary); + PR_Free(mMultipartSignedBoundary); + mMultipartSignedBoundary = 0; + + if (!header) { + rv = NS_ERROR_OUT_OF_MEMORY; + goto FAIL; + } + L = strlen(header); + if (aOuter) { + /* If this is the outer block, write it to the file. */ + uint32_t n; + rv = mStream->Write(header, L, &n); + if (NS_FAILED(rv) || n < L) + // XXX This is -1, not an nsresult + rv = static_cast<nsresult>(MK_MIME_ERROR_WRITING_FILE); + } else { + /* If this is an inner block, feed it through the crypto stream. */ + rv = MimeCryptoWriteBlock (header, L); + } + } + +FAIL: + return rv; +} + + +/* Helper function for mime_finish_crypto_encapsulation() to close off + an opaque crypto object (for encrypted or signed-and-encrypted messages.) + */ +nsresult nsMsgComposeSecure::MimeFinishEncryption (bool aSign, nsIMsgSendReport *sendReport) +{ + nsresult rv; + + /* If this object is both encrypted and signed, close off the + signature first (since it's inside.) */ + if (aSign) { + rv = MimeFinishMultipartSigned (false, sendReport); + if (NS_FAILED(rv)) { + goto FAIL; + } + } + + /* Close off the opaque encrypted blob. + */ + PR_ASSERT(mEncryptionContext); + + if (mBufferedBytes) { + rv = mEncryptionContext->Update(mBuffer, mBufferedBytes); + mBufferedBytes = 0; + if (NS_FAILED(rv)) { + PR_ASSERT(PR_GetError() < 0); + goto FAIL; + } + } + + rv = mEncryptionContext->Finish(); + if (NS_FAILED(rv)) { + SetError(sendReport, u"ErrorEncryptMail"); + goto FAIL; + } + + mEncryptionContext = nullptr; + + PR_ASSERT(mEncryptionCinfo); + if (!mEncryptionCinfo) { + rv = NS_ERROR_FAILURE; + } + if (mEncryptionCinfo) { + mEncryptionCinfo = nullptr; + } + + // Shut down the base64 encoder. + mCryptoEncoder->Flush(); + mCryptoEncoder = nullptr; + + uint32_t n; + rv = mStream->Write(CRLF, 2, &n); + if (NS_FAILED(rv) || n < 2) + rv = NS_ERROR_FAILURE; + + FAIL: + return rv; +} + +/* Used to figure out what certs should be used when encrypting this message. + */ +nsresult nsMsgComposeSecure::MimeCryptoHackCerts(const char *aRecipients, + nsIMsgSendReport *sendReport, + bool aEncrypt, + bool aSign, + nsIMsgIdentity *aIdentity) +{ + nsCOMPtr<nsIX509CertDB> certdb = do_GetService(NS_X509CERTDB_CONTRACTID); + nsresult res; + + mCerts = do_CreateInstance(NS_ARRAY_CONTRACTID, &res); + if (NS_FAILED(res)) { + return res; + } + + PR_ASSERT(aEncrypt || aSign); + + /* + Signing and encryption certs use the following (per-identity) preferences: + - "signing_cert_name"/"encryption_cert_name": a string specifying the + nickname of the certificate + - "signing_cert_dbkey"/"encryption_cert_dbkey": a Base64 encoded blob + specifying an nsIX509Cert dbKey (represents serial number + and issuer DN, which is considered to be unique for X.509 certificates) + + When retrieving the prefs, we try (in this order): + 1) *_cert_dbkey, if available + 2) *_cert_name (for maintaining backwards compatibility with preference + attributes written by earlier versions) + */ + + RefPtr<SharedCertVerifier> certVerifier(GetDefaultCertVerifier()); + NS_ENSURE_TRUE(certVerifier, NS_ERROR_UNEXPECTED); + + UniqueCERTCertList builtChain; + if (!mEncryptionCertDBKey.IsEmpty()) { + certdb->FindCertByDBKey(mEncryptionCertDBKey.get(), + getter_AddRefs(mSelfEncryptionCert)); + if (mSelfEncryptionCert && + (certVerifier->VerifyCert(mSelfEncryptionCert->GetCert(), + certificateUsageEmailRecipient, + mozilla::pkix::Now(), + nullptr, nullptr, + builtChain) != mozilla::pkix::Success)) { + // not suitable for encryption, so unset cert and clear pref + mSelfEncryptionCert = nullptr; + mEncryptionCertDBKey.Truncate(); + aIdentity->SetCharAttribute("encryption_cert_dbkey", + mEncryptionCertDBKey); + } + } + if (!mSelfEncryptionCert) { + certdb->FindEmailEncryptionCert(mEncryptionCertName, + getter_AddRefs(mSelfEncryptionCert)); + } + + // same procedure for the signing cert + if (!mSigningCertDBKey.IsEmpty()) { + certdb->FindCertByDBKey(mSigningCertDBKey.get(), + getter_AddRefs(mSelfSigningCert)); + if (mSelfSigningCert && + (certVerifier->VerifyCert(mSelfSigningCert->GetCert(), + certificateUsageEmailSigner, + mozilla::pkix::Now(), + nullptr, nullptr, + builtChain) != mozilla::pkix::Success)) { + // not suitable for signing, so unset cert and clear pref + mSelfSigningCert = nullptr; + mSigningCertDBKey.Truncate(); + aIdentity->SetCharAttribute("signing_cert_dbkey", mSigningCertDBKey); + } + } + if (!mSelfSigningCert) { + certdb->FindEmailSigningCert(mSigningCertName, + getter_AddRefs(mSelfSigningCert)); + } + + // must have both the signing and encryption certs to sign + if (!mSelfSigningCert && aSign) { + SetError(sendReport, u"NoSenderSigningCert"); + return NS_ERROR_FAILURE; + } + + if (!mSelfEncryptionCert && aEncrypt) { + SetError(sendReport, u"NoSenderEncryptionCert"); + return NS_ERROR_FAILURE; + } + + + if (aEncrypt && mSelfEncryptionCert) { + // Make sure self's configured cert is prepared for being used + // as an email recipient cert. + UniqueCERTCertificate nsscert(mSelfEncryptionCert->GetCert()); + if (!nsscert) { + return NS_ERROR_FAILURE; + } + // XXX: This does not respect the nsNSSShutDownObject protocol. + if (CERT_SaveSMimeProfile(nsscert.get(), nullptr, nullptr) != SECSuccess) { + return NS_ERROR_FAILURE; + } + } + + /* If the message is to be encrypted, then get the recipient certs */ + if (aEncrypt) { + nsTArray<nsCString> mailboxes; + ExtractEmails(EncodedHeader(nsDependentCString(aRecipients)), + UTF16ArrayAdapter<>(mailboxes)); + uint32_t count = mailboxes.Length(); + + bool already_added_self_cert = false; + + for (uint32_t i = 0; i < count; i++) { + nsCString mailbox_lowercase; + ToLowerCase(mailboxes[i], mailbox_lowercase); + nsCOMPtr<nsIX509Cert> cert; + res = certdb->FindCertByEmailAddress(mailbox_lowercase.get(), + getter_AddRefs(cert)); + if (NS_FAILED(res)) { + // Failure to find a valid encryption cert is fatal. + // Here I assume that mailbox is ascii rather than utf8. + SetErrorWithParam(sendReport, + u"MissingRecipientEncryptionCert", + mailboxes[i].get()); + + return res; + } + + /* #### see if recipient requests `signedData'. + if (...) no_clearsigning_p = true; + (This is the only reason we even bother looking up the certs + of the recipients if we're sending a signed-but-not-encrypted + message.) + */ + + bool isSame; + if (NS_SUCCEEDED(cert->Equals(mSelfEncryptionCert, &isSame)) + && isSame) { + already_added_self_cert = true; + } + + mCerts->AppendElement(cert, false); + } + + if (!already_added_self_cert) { + mCerts->AppendElement(mSelfEncryptionCert, false); + } + } + return res; +} + +NS_IMETHODIMP nsMsgComposeSecure::MimeCryptoWriteBlock (const char *buf, int32_t size) +{ + int status = 0; + nsresult rv; + + /* If this is a From line, mangle it before signing it. You just know + that something somewhere is going to mangle it later, and that's + going to cause the signature check to fail. + + (This assumes that, in the cases where From-mangling must happen, + this function is called a line at a time. That happens to be the + case.) + */ + if (size >= 5 && buf[0] == 'F' && !strncmp(buf, "From ", 5)) { + char mangle[] = ">"; + nsresult res = MimeCryptoWriteBlock (mangle, 1); + if (NS_FAILED(res)) + return res; + // This value will actually be cast back to an nsresult before use, so this + // cast is reasonable under the circumstances. + status = static_cast<int>(res); + } + + /* If we're signing, or signing-and-encrypting, feed this data into + the computation of the hash. */ + if (mDataHash) { + PR_SetError(0,0); + mDataHash->Update((const uint8_t*) buf, size); + status = PR_GetError(); + if (status < 0) goto FAIL; + } + + PR_SetError(0,0); + if (mEncryptionContext) { + /* If we're encrypting, or signing-and-encrypting, write this data + by filtering it through the crypto library. */ + + /* We want to create equally sized encryption strings */ + const char *inputBytesIterator = buf; + uint32_t inputBytesLeft = size; + + while (inputBytesLeft) { + const uint32_t spaceLeftInBuffer = eBufferSize - mBufferedBytes; + const uint32_t bytesToAppend = std::min(inputBytesLeft, spaceLeftInBuffer); + + memcpy(mBuffer+mBufferedBytes, inputBytesIterator, bytesToAppend); + mBufferedBytes += bytesToAppend; + + inputBytesIterator += bytesToAppend; + inputBytesLeft -= bytesToAppend; + + if (eBufferSize == mBufferedBytes) { + rv = mEncryptionContext->Update(mBuffer, mBufferedBytes); + mBufferedBytes = 0; + if (NS_FAILED(rv)) { + status = PR_GetError(); + PR_ASSERT(status < 0); + if (status >= 0) status = -1; + goto FAIL; + } + } + } + } else { + /* If we're not encrypting (presumably just signing) then write this + data directly to the file. */ + + uint32_t n; + rv = mStream->Write(buf, size, &n); + if (NS_FAILED(rv) || n < (uint32_t)size) { + // XXX MK_MIME_ERROR_WRITING_FILE is -1, which is not a valid nsresult + return static_cast<nsresult>(MK_MIME_ERROR_WRITING_FILE); + } + } + FAIL: + // XXX status sometimes has invalid nsresults like -1 or PR_GetError() + // assigned to it + return static_cast<nsresult>(status); +} + +/* Returns a string consisting of a Content-Type header, and a boundary + string, suitable for moving from the header block, down into the body + of a multipart object. The boundary itself is also returned (so that + the caller knows what to write to close it off.) + */ +static nsresult +make_multipart_signed_header_string(bool outer_p, + char **header_return, + char **boundary_return, + int16_t hash_type) +{ + const char *hashStr; + *header_return = 0; + *boundary_return = mime_make_separator("ms"); + + if (!*boundary_return) + return NS_ERROR_OUT_OF_MEMORY; + + switch (hash_type) { + case nsICryptoHash::SHA1: + hashStr = PARAM_MICALG_SHA1; + break; + case nsICryptoHash::SHA256: + hashStr = PARAM_MICALG_SHA256; + break; + case nsICryptoHash::SHA384: + hashStr = PARAM_MICALG_SHA384; + break; + case nsICryptoHash::SHA512: + hashStr = PARAM_MICALG_SHA512; + break; + default: + return NS_ERROR_INVALID_ARG; + } + + *header_return = PR_smprintf( + "Content-Type: " MULTIPART_SIGNED "; " + "protocol=\"" APPLICATION_PKCS7_SIGNATURE "\"; " + "micalg=%s; " + "boundary=\"%s\"" CRLF + CRLF + "%s%s" + "--%s" CRLF, + hashStr, + *boundary_return, + (outer_p ? crypto_multipart_blurb : ""), + (outer_p ? CRLF CRLF : ""), + *boundary_return); + + if (!*header_return) { + PR_Free(*boundary_return); + *boundary_return = 0; + return NS_ERROR_OUT_OF_MEMORY; + } + + return NS_OK; +} + +/* Used as the output function of a SEC_PKCS7EncoderContext -- we feed + plaintext into the crypto engine, and it calls this function with encrypted + data; then this function writes a base64-encoded representation of that + data to the file (by filtering it through the given MimeEncoder object.) + + Also used as the output function of SEC_PKCS7Encode() -- but in that case, + it's used to write the encoded representation of the signature. The only + difference is which MimeEncoder object is used. + */ +static void +mime_crypto_write_base64 (void *closure, const char *buf, unsigned long size) +{ + MimeEncoder *encoder = (MimeEncoder *) closure; + nsresult rv = encoder->Write(buf, size); + PR_SetError(NS_FAILED(rv) ? static_cast<uint32_t>(rv) : 0, 0); +} + + +/* Used as the output function of MimeEncoder -- when we have generated + the signature for a multipart/signed object, this is used to write the + base64-encoded representation of the signature to the file. + */ +// TODO: size should probably be converted to uint32_t +nsresult mime_encoder_output_fn(const char *buf, int32_t size, void *closure) +{ + nsMsgComposeSecure *state = (nsMsgComposeSecure *) closure; + nsCOMPtr<nsIOutputStream> stream; + state->GetOutputStream(getter_AddRefs(stream)); + uint32_t n; + nsresult rv = stream->Write((char *) buf, size, &n); + if (NS_FAILED(rv) || n < (uint32_t)size) + return NS_ERROR_FAILURE; + else + return NS_OK; +} + +/* Like mime_encoder_output_fn, except this is used for the case where we + are both signing and encrypting -- the base64-encoded output of the + signature should be fed into the crypto engine, rather than being written + directly to the file. + */ +static nsresult +mime_nested_encoder_output_fn (const char *buf, int32_t size, void *closure) +{ + nsMsgComposeSecure *state = (nsMsgComposeSecure *) closure; + + // Copy to new null-terminated string so JS glue doesn't crash when + // MimeCryptoWriteBlock() is implemented in JS. + nsCString bufWithNull; + bufWithNull.Assign(buf, size); + return state->MimeCryptoWriteBlock(bufWithNull.get(), size); +} diff --git a/mailnews/extensions/smime/src/nsMsgComposeSecure.h b/mailnews/extensions/smime/src/nsMsgComposeSecure.h new file mode 100644 index 000000000..0f3b9ac60 --- /dev/null +++ b/mailnews/extensions/smime/src/nsMsgComposeSecure.h @@ -0,0 +1,106 @@ +/* -*- Mode: idl; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#ifndef _nsMsgComposeSecure_H_ +#define _nsMsgComposeSecure_H_ + +#include "nsIMsgComposeSecure.h" +#include "nsIMsgSMIMECompFields.h" +#include "nsCOMPtr.h" +#include "nsICMSEncoder.h" +#include "nsIX509Cert.h" +#include "nsIStringBundle.h" +#include "nsICryptoHash.h" +#include "nsICMSMessage.h" +#include "nsIMutableArray.h" +#include "nsStringGlue.h" +#include "nsIOutputStream.h" +#include "nsAutoPtr.h" + +class nsIMsgCompFields; +namespace mozilla { +namespace mailnews { +class MimeEncoder; +} +} + +class nsMsgSMIMEComposeFields : public nsIMsgSMIMECompFields +{ +public: + NS_DECL_ISUPPORTS + NS_DECL_NSIMSGSMIMECOMPFIELDS + + nsMsgSMIMEComposeFields(); + +private: + virtual ~nsMsgSMIMEComposeFields(); + bool mSignMessage; + bool mAlwaysEncryptMessage; +}; + +typedef enum { + mime_crypto_none, /* normal unencapsulated MIME message */ + mime_crypto_clear_signed, /* multipart/signed encapsulation */ + mime_crypto_opaque_signed, /* application/x-pkcs7-mime (signedData) */ + mime_crypto_encrypted, /* application/x-pkcs7-mime */ + mime_crypto_signed_encrypted /* application/x-pkcs7-mime */ +} mimeDeliveryCryptoState; + +class nsMsgComposeSecure : public nsIMsgComposeSecure +{ +public: + NS_DECL_ISUPPORTS + NS_DECL_NSIMSGCOMPOSESECURE + + nsMsgComposeSecure(); + + void GetOutputStream(nsIOutputStream **stream) { NS_IF_ADDREF(*stream = mStream);} + nsresult GetSMIMEBundleString(const char16_t *name, nsString &outString); + +private: + virtual ~nsMsgComposeSecure(); + typedef mozilla::mailnews::MimeEncoder MimeEncoder; + nsresult MimeInitMultipartSigned(bool aOuter, nsIMsgSendReport *sendReport); + nsresult MimeInitEncryption(bool aSign, nsIMsgSendReport *sendReport); + nsresult MimeFinishMultipartSigned (bool aOuter, nsIMsgSendReport *sendReport); + nsresult MimeFinishEncryption (bool aSign, nsIMsgSendReport *sendReport); + nsresult MimeCryptoHackCerts(const char *aRecipients, nsIMsgSendReport *sendReport, bool aEncrypt, bool aSign, nsIMsgIdentity *aIdentity); + bool InitializeSMIMEBundle(); + nsresult SMIMEBundleFormatStringFromName(const char16_t *name, + const char16_t **params, + uint32_t numParams, + char16_t **outString); + nsresult ExtractEncryptionState(nsIMsgIdentity * aIdentity, nsIMsgCompFields * aComposeFields, bool * aSignMessage, bool * aEncrypt); + + mimeDeliveryCryptoState mCryptoState; + nsCOMPtr<nsIOutputStream> mStream; + int16_t mHashType; + nsCOMPtr<nsICryptoHash> mDataHash; + nsAutoPtr<MimeEncoder> mSigEncoder; + char *mMultipartSignedBoundary; + nsString mSigningCertName; + nsAutoCString mSigningCertDBKey; + nsCOMPtr<nsIX509Cert> mSelfSigningCert; + nsString mEncryptionCertName; + nsAutoCString mEncryptionCertDBKey; + nsCOMPtr<nsIX509Cert> mSelfEncryptionCert; + nsCOMPtr<nsIMutableArray> mCerts; + nsCOMPtr<nsICMSMessage> mEncryptionCinfo; + nsCOMPtr<nsICMSEncoder> mEncryptionContext; + nsCOMPtr<nsIStringBundle> mSMIMEBundle; + + nsAutoPtr<MimeEncoder> mCryptoEncoder; + bool mIsDraft; + + enum {eBufferSize = 8192}; + char *mBuffer; + uint32_t mBufferedBytes; + + bool mErrorAlreadyReported; + void SetError(nsIMsgSendReport *sendReport, const char16_t *bundle_string); + void SetErrorWithParam(nsIMsgSendReport *sendReport, const char16_t *bundle_string, const char *param); +}; + +#endif diff --git a/mailnews/extensions/smime/src/nsMsgSMIMECID.h b/mailnews/extensions/smime/src/nsMsgSMIMECID.h new file mode 100644 index 000000000..0fbf2d1bf --- /dev/null +++ b/mailnews/extensions/smime/src/nsMsgSMIMECID.h @@ -0,0 +1,42 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef nsMsgSMIMECID_h__ +#define nsMsgSMIMECID_h__ + +#include "nsISupports.h" +#include "nsIFactory.h" +#include "nsIComponentManager.h" + +#define NS_MSGSMIMECOMPFIELDS_CONTRACTID \ + "@mozilla.org/messenger-smime/composefields;1" + +#define NS_MSGSMIMECOMPFIELDS_CID \ +{ /* 122C919C-96B7-49a0-BBC8-0ABC67EEFFE0 */ \ + 0x122c919c, 0x96b7, 0x49a0, \ + { 0xbb, 0xc8, 0xa, 0xbc, 0x67, 0xee, 0xff, 0xe0 }} + +#define NS_MSGCOMPOSESECURE_CID \ +{ /* dd753201-9a23-4e08-957f-b3616bf7e012 */ \ + 0xdd753201, 0x9a23, 0x4e08, \ + {0x95, 0x7f, 0xb3, 0x61, 0x6b, 0xf7, 0xe0, 0x12 }} + +#define NS_SMIMEJSHELPER_CONTRACTID \ + "@mozilla.org/messenger-smime/smimejshelper;1" + +#define NS_SMIMEJSJELPER_CID \ +{ /* d57d928c-60e4-4f81-999d-5c762e611205 */ \ + 0xd57d928c, 0x60e4, 0x4f81, \ + {0x99, 0x9d, 0x5c, 0x76, 0x2e, 0x61, 0x12, 0x05 }} + +#define NS_SMIMEENCRYPTURISERVICE_CONTRACTID \ + "@mozilla.org/messenger-smime/smime-encrypted-uris-service;1" + +#define NS_SMIMEENCRYPTURISERVICE_CID \ +{ /* a0134d58-018f-4d40-a099-fa079e5024a6 */ \ + 0xa0134d58, 0x018f, 0x4d40, \ + {0xa0, 0x99, 0xfa, 0x07, 0x9e, 0x50, 0x24, 0xa6 }} + +#endif // nsMsgSMIMECID_h__ diff --git a/mailnews/extensions/smime/src/nsSMimeJSHelper.cpp b/mailnews/extensions/smime/src/nsSMimeJSHelper.cpp new file mode 100644 index 000000000..c392980b6 --- /dev/null +++ b/mailnews/extensions/smime/src/nsSMimeJSHelper.cpp @@ -0,0 +1,335 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/mailnews/MimeHeaderParser.h" +#include "nspr.h" +#include "nsSMimeJSHelper.h" +#include "nsCOMPtr.h" +#include "nsMemory.h" +#include "nsStringGlue.h" +#include "nsIX509CertDB.h" +#include "nsIX509CertValidity.h" +#include "nsIServiceManager.h" +#include "nsServiceManagerUtils.h" +#include "nsCRTGlue.h" + +using namespace mozilla::mailnews; + +NS_IMPL_ISUPPORTS(nsSMimeJSHelper, nsISMimeJSHelper) + +nsSMimeJSHelper::nsSMimeJSHelper() +{ +} + +nsSMimeJSHelper::~nsSMimeJSHelper() +{ +} + +NS_IMETHODIMP nsSMimeJSHelper::GetRecipientCertsInfo( + nsIMsgCompFields *compFields, + uint32_t *count, + char16_t ***emailAddresses, + int32_t **certVerification, + char16_t ***certIssuedInfos, + char16_t ***certExpiresInfos, + nsIX509Cert ***certs, + bool *canEncrypt) +{ + NS_ENSURE_ARG_POINTER(count); + *count = 0; + + NS_ENSURE_ARG_POINTER(emailAddresses); + NS_ENSURE_ARG_POINTER(certVerification); + NS_ENSURE_ARG_POINTER(certIssuedInfos); + NS_ENSURE_ARG_POINTER(certExpiresInfos); + NS_ENSURE_ARG_POINTER(certs); + NS_ENSURE_ARG_POINTER(canEncrypt); + + NS_ENSURE_ARG_POINTER(compFields); + + nsTArray<nsCString> mailboxes; + nsresult rv = getMailboxList(compFields, mailboxes); + NS_ENSURE_SUCCESS(rv, rv); + + uint32_t mailbox_count = mailboxes.Length(); + + nsCOMPtr<nsIX509CertDB> certdb = do_GetService(NS_X509CERTDB_CONTRACTID); + + *count = mailbox_count; + *canEncrypt = false; + rv = NS_OK; + + if (mailbox_count) + { + char16_t **outEA = static_cast<char16_t **>(moz_xmalloc(mailbox_count * sizeof(char16_t *))); + int32_t *outCV = static_cast<int32_t *>(moz_xmalloc(mailbox_count * sizeof(int32_t))); + char16_t **outCII = static_cast<char16_t **>(moz_xmalloc(mailbox_count * sizeof(char16_t *))); + char16_t **outCEI = static_cast<char16_t **>(moz_xmalloc(mailbox_count * sizeof(char16_t *))); + nsIX509Cert **outCerts = static_cast<nsIX509Cert **>(moz_xmalloc(mailbox_count * sizeof(nsIX509Cert *))); + + if (!outEA || !outCV || !outCII || !outCEI || !outCerts) + { + free(outEA); + free(outCV); + free(outCII); + free(outCEI); + free(outCerts); + rv = NS_ERROR_OUT_OF_MEMORY; + } + else + { + char16_t **iEA = outEA; + int32_t *iCV = outCV; + char16_t **iCII = outCII; + char16_t **iCEI = outCEI; + nsIX509Cert **iCert = outCerts; + + bool found_blocker = false; + bool memory_failure = false; + + for (uint32_t i = 0; + i < mailbox_count; + ++i, ++iEA, ++iCV, ++iCII, ++iCEI, ++iCert) + { + *iCert = nullptr; + *iCV = 0; + *iCII = nullptr; + *iCEI = nullptr; + + if (memory_failure) { + *iEA = nullptr; + continue; + } + + nsCString &email = mailboxes[i]; + *iEA = ToNewUnicode(NS_ConvertUTF8toUTF16(email)); + if (!*iEA) { + memory_failure = true; + continue; + } + + nsCString email_lowercase; + ToLowerCase(email, email_lowercase); + + nsCOMPtr<nsIX509Cert> cert; + if (NS_SUCCEEDED(certdb->FindCertByEmailAddress( + email_lowercase.get(), getter_AddRefs(cert)))) + { + *iCert = cert; + NS_ADDREF(*iCert); + + nsCOMPtr<nsIX509CertValidity> validity; + rv = cert->GetValidity(getter_AddRefs(validity)); + + if (NS_SUCCEEDED(rv)) { + nsString id, ed; + + if (NS_SUCCEEDED(validity->GetNotBeforeLocalDay(id))) + { + *iCII = ToNewUnicode(id); + if (!*iCII) { + memory_failure = true; + continue; + } + } + + if (NS_SUCCEEDED(validity->GetNotAfterLocalDay(ed))) + { + *iCEI = ToNewUnicode(ed); + if (!*iCEI) { + memory_failure = true; + continue; + } + } + } + } + else + { + found_blocker = true; + } + } + + if (memory_failure) { + NS_FREE_XPCOM_ALLOCATED_POINTER_ARRAY(mailbox_count, outEA); + NS_FREE_XPCOM_ALLOCATED_POINTER_ARRAY(mailbox_count, outCII); + NS_FREE_XPCOM_ALLOCATED_POINTER_ARRAY(mailbox_count, outCEI); + NS_FREE_XPCOM_ISUPPORTS_POINTER_ARRAY(mailbox_count, outCerts); + free(outCV); + rv = NS_ERROR_OUT_OF_MEMORY; + } + else { + if (mailbox_count > 0 && !found_blocker) + { + *canEncrypt = true; + } + + *emailAddresses = outEA; + *certVerification = outCV; + *certIssuedInfos = outCII; + *certExpiresInfos = outCEI; + *certs = outCerts; + } + } + } + return rv; +} + +NS_IMETHODIMP nsSMimeJSHelper::GetNoCertAddresses( + nsIMsgCompFields *compFields, + uint32_t *count, + char16_t ***emailAddresses) +{ + NS_ENSURE_ARG_POINTER(count); + *count = 0; + + NS_ENSURE_ARG_POINTER(emailAddresses); + + NS_ENSURE_ARG_POINTER(compFields); + + nsTArray<nsCString> mailboxes; + nsresult rv = getMailboxList(compFields, mailboxes); + NS_ENSURE_SUCCESS(rv, rv); + + uint32_t mailbox_count = mailboxes.Length(); + + if (!mailbox_count) + { + *count = 0; + *emailAddresses = nullptr; + return NS_OK; + } + + nsCOMPtr<nsIX509CertDB> certdb = do_GetService(NS_X509CERTDB_CONTRACTID); + + uint32_t missing_count = 0; + bool *haveCert = new bool[mailbox_count]; + if (!haveCert) + { + return NS_ERROR_OUT_OF_MEMORY; + } + + rv = NS_OK; + + if (mailbox_count) + { + for (uint32_t i = 0; i < mailbox_count; ++i) + { + haveCert[i] = false; + + nsCString email_lowercase; + ToLowerCase(mailboxes[i], email_lowercase); + + nsCOMPtr<nsIX509Cert> cert; + if (NS_SUCCEEDED(certdb->FindCertByEmailAddress( + email_lowercase.get(), getter_AddRefs(cert)))) + haveCert[i] = true; + + if (!haveCert[i]) + ++missing_count; + } + } + + *count = missing_count; + + if (missing_count) + { + char16_t **outEA = static_cast<char16_t **>(moz_xmalloc(missing_count * sizeof(char16_t *))); + if (!outEA ) + { + rv = NS_ERROR_OUT_OF_MEMORY; + } + else + { + char16_t **iEA = outEA; + + bool memory_failure = false; + + for (uint32_t i = 0; i < mailbox_count; ++i) + { + if (!haveCert[i]) + { + if (memory_failure) { + *iEA = nullptr; + } + else { + *iEA = ToNewUnicode(NS_ConvertUTF8toUTF16(mailboxes[i])); + if (!*iEA) { + memory_failure = true; + } + } + ++iEA; + } + } + + if (memory_failure) { + NS_FREE_XPCOM_ALLOCATED_POINTER_ARRAY(missing_count, outEA); + rv = NS_ERROR_OUT_OF_MEMORY; + } + else { + *emailAddresses = outEA; + } + } + } + else + { + *emailAddresses = nullptr; + } + + delete [] haveCert; + return rv; +} + +nsresult nsSMimeJSHelper::getMailboxList(nsIMsgCompFields *compFields, + nsTArray<nsCString> &mailboxes) +{ + if (!compFields) + return NS_ERROR_INVALID_ARG; + + nsresult res; + nsString to, cc, bcc, ng; + + res = compFields->GetTo(to); + if (NS_FAILED(res)) + return res; + + res = compFields->GetCc(cc); + if (NS_FAILED(res)) + return res; + + res = compFields->GetBcc(bcc); + if (NS_FAILED(res)) + return res; + + res = compFields->GetNewsgroups(ng); + if (NS_FAILED(res)) + return res; + + { + nsCString all_recipients; + + if (!to.IsEmpty()) { + all_recipients.Append(NS_ConvertUTF16toUTF8(to)); + all_recipients.Append(','); + } + + if (!cc.IsEmpty()) { + all_recipients.Append(NS_ConvertUTF16toUTF8(cc)); + all_recipients.Append(','); + } + + if (!bcc.IsEmpty()) { + all_recipients.Append(NS_ConvertUTF16toUTF8(bcc)); + all_recipients.Append(','); + } + + if (!ng.IsEmpty()) + all_recipients.Append(NS_ConvertUTF16toUTF8(ng)); + + ExtractEmails(EncodedHeader(all_recipients), + UTF16ArrayAdapter<>(mailboxes)); + } + + return NS_OK; +} diff --git a/mailnews/extensions/smime/src/nsSMimeJSHelper.h b/mailnews/extensions/smime/src/nsSMimeJSHelper.h new file mode 100644 index 000000000..403ab2098 --- /dev/null +++ b/mailnews/extensions/smime/src/nsSMimeJSHelper.h @@ -0,0 +1,26 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef _nsSMimeJSHelper_H_ +#define _nsSMimeJSHelper_H_ + +#include "nsISMimeJSHelper.h" +#include "nsIX509Cert.h" +#include "nsIMsgCompFields.h" + +class nsSMimeJSHelper : public nsISMimeJSHelper +{ +public: + NS_DECL_ISUPPORTS + NS_DECL_NSISMIMEJSHELPER + + nsSMimeJSHelper(); + +private: + virtual ~nsSMimeJSHelper(); + nsresult getMailboxList(nsIMsgCompFields *compFields, + nsTArray<nsCString> &mailboxes); +}; + +#endif diff --git a/mailnews/extensions/smime/src/smime-service.js b/mailnews/extensions/smime/src/smime-service.js new file mode 100644 index 000000000..9fa618fd6 --- /dev/null +++ b/mailnews/extensions/smime/src/smime-service.js @@ -0,0 +1,24 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +Components.utils.import("resource://gre/modules/XPCOMUtils.jsm"); + +function SMIMEService() {} + +SMIMEService.prototype = { + name: "smime", + chromePackageName: "messenger", + showPanel: function(server) { + // don't show the panel for news, rss, or local accounts + return (server.type != "nntp" && server.type != "rss" && + server.type != "im" && server.type != "none"); + }, + + QueryInterface: XPCOMUtils.generateQI([Components.interfaces.nsIMsgAccountManagerExtension]), + classID: Components.ID("{f2809796-1dd1-11b2-8c1b-8f15f007c699}"), +}; + +var components = [SMIMEService]; +var NSGetFactory = XPCOMUtils.generateNSGetFactory(components); diff --git a/mailnews/extensions/smime/src/smime-service.manifest b/mailnews/extensions/smime/src/smime-service.manifest new file mode 100644 index 000000000..1b799ed7b --- /dev/null +++ b/mailnews/extensions/smime/src/smime-service.manifest @@ -0,0 +1,3 @@ +component {f2809796-1dd1-11b2-8c1b-8f15f007c699} smime-service.js +contract @mozilla.org/accountmanager/extension;1?name=smime {f2809796-1dd1-11b2-8c1b-8f15f007c699} +category mailnews-accountmanager-extensions smime-account-manager-extension @mozilla.org/accountmanager/extension;1?name=smime |