diff options
Diffstat (limited to '')
139 files changed, 38188 insertions, 0 deletions
diff --git a/comm/mailnews/extensions/bayesian-spam-filter/components.conf b/comm/mailnews/extensions/bayesian-spam-filter/components.conf new file mode 100644 index 0000000000..98fe2d6aeb --- /dev/null +++ b/comm/mailnews/extensions/bayesian-spam-filter/components.conf @@ -0,0 +1,15 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, you can obtain one at http://mozilla.org/MPL/2.0/. + +Classes = [ + { + "cid": "{f1070bfa-d539-11d6-90ca-00039310a47a}", + "contract_ids": ["@mozilla.org/messenger/filter-plugin;1?name=bayesianfilter"], + "type": "nsBayesianFilter", + "init_method": "Init", + "headers": [ + "/comm/mailnews/extensions/bayesian-spam-filter/nsBayesianFilter.h" + ], + }, +] diff --git a/comm/mailnews/extensions/bayesian-spam-filter/moz.build b/comm/mailnews/extensions/bayesian-spam-filter/moz.build new file mode 100644 index 0000000000..329fdcafa4 --- /dev/null +++ b/comm/mailnews/extensions/bayesian-spam-filter/moz.build @@ -0,0 +1,16 @@ +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +SOURCES += [ + "nsBayesianFilter.cpp", +] + +FINAL_LIBRARY = "mail" + +XPCSHELL_TESTS_MANIFESTS += ["test/unit/xpcshell.ini"] + +XPCOM_MANIFESTS += [ + "components.conf", +] diff --git a/comm/mailnews/extensions/bayesian-spam-filter/nsBayesianFilter.cpp b/comm/mailnews/extensions/bayesian-spam-filter/nsBayesianFilter.cpp new file mode 100644 index 0000000000..8a4cca905b --- /dev/null +++ b/comm/mailnews/extensions/bayesian-spam-filter/nsBayesianFilter.cpp @@ -0,0 +1,2548 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsBayesianFilter.h" +#include "nsIInputStream.h" +#include "nsIStreamListener.h" +#include "nsNetUtil.h" +#include "nsQuickSort.h" +#include "nsIMsgMessageService.h" +#include "nsMsgUtils.h" // for GetMessageServiceFromURI +#include "prnetdb.h" +#include "nsIMsgWindow.h" +#include "mozilla/Logging.h" +#include "nsAppDirectoryServiceDefs.h" +#include "nsUnicharUtils.h" +#include "nsDirectoryServiceUtils.h" +#include "nsIMIMEHeaderParam.h" +#include "nsNetCID.h" +#include "nsIMsgMailNewsUrl.h" +#include "nsIPrefService.h" +#include "nsIPrefBranch.h" +#include "nsIStringEnumerator.h" +#include "nsIObserverService.h" +#include "nsIChannel.h" +#include "nsIMailChannel.h" +#include "nsDependentSubstring.h" +#include "nsMemory.h" +#include "nsUnicodeProperties.h" + +#include "mozilla/ArenaAllocatorExtensions.h" // for ArenaStrdup + +using namespace mozilla; +using mozilla::intl::Script; +using mozilla::intl::UnicodeProperties; + +// needed to mark attachment flag on the db hdr +#include "nsIMsgHdr.h" + +// needed to strip html out of the body +#include "nsLayoutCID.h" +#include "nsIParserUtils.h" +#include "nsIDocumentEncoder.h" + +#include "nsIncompleteGamma.h" +#include <math.h> +#include <prmem.h> +#include "nsIMsgTraitService.h" +#include "mozilla/Services.h" +#include "mozilla/Attributes.h" +#include <cstdlib> // for std::abs(int/long) +#include <cmath> // for std::abs(float/double) + +static mozilla::LazyLogModule BayesianFilterLogModule("BayesianFilter"); + +#define kDefaultJunkThreshold .99 // we override this value via a pref +static const char* kBayesianFilterTokenDelimiters = " \t\n\r\f."; +static unsigned int kMinLengthForToken = + 3; // lower bound on the number of characters in a word before we treat it + // as a token +static unsigned int kMaxLengthForToken = + 12; // upper bound on the number of characters in a word to be declared as + // a token + +#define FORGED_RECEIVED_HEADER_HINT "may be forged"_ns + +#ifndef M_LN2 +# define M_LN2 0.69314718055994530942 +#endif + +#ifndef M_E +# define M_E 2.7182818284590452354 +#endif + +// provide base implementation of hash lookup of a string +struct BaseToken : public PLDHashEntryHdr { + const char* mWord; +}; + +// token for a particular message +// mCount, mAnalysisLink are initialized to zero by the hash code +struct Token : public BaseToken { + uint32_t mCount; + uint32_t mAnalysisLink; // index in mAnalysisStore of the AnalysisPerToken + // object for the first trait for this token + // Helper to support Tokenizer::copyTokens() + void clone(const Token& other) { + mWord = other.mWord; + mCount = other.mCount; + mAnalysisLink = other.mAnalysisLink; + } +}; + +// token stored in a training file for a group of messages +// mTraitLink is initialized to 0 by the hash code +struct CorpusToken : public BaseToken { + uint32_t mTraitLink; // index in mTraitStore of the TraitPerToken + // object for the first trait for this token +}; + +// set the value of a TraitPerToken object +TraitPerToken::TraitPerToken(uint32_t aTraitId, uint32_t aCount) + : mId(aTraitId), mCount(aCount), mNextLink(0) {} + +// shorthand representations of trait ids for junk and good +static const uint32_t kJunkTrait = nsIJunkMailPlugin::JUNK_TRAIT; +static const uint32_t kGoodTrait = nsIJunkMailPlugin::GOOD_TRAIT; + +// set the value of an AnalysisPerToken object +AnalysisPerToken::AnalysisPerToken(uint32_t aTraitIndex, double aDistance, + double aProbability) + : mTraitIndex(aTraitIndex), + mDistance(aDistance), + mProbability(aProbability), + mNextLink(0) {} + +// the initial size of the AnalysisPerToken linked list storage +const uint32_t kAnalysisStoreCapacity = 2048; + +// the initial size of the TraitPerToken linked list storage +const uint32_t kTraitStoreCapacity = 16384; + +// Size of Auto arrays representing per trait information +const uint32_t kTraitAutoCapacity = 10; + +TokenEnumeration::TokenEnumeration(PLDHashTable* table) + : mIterator(table->Iter()) {} + +inline bool TokenEnumeration::hasMoreTokens() { return !mIterator.Done(); } + +inline BaseToken* TokenEnumeration::nextToken() { + auto token = static_cast<BaseToken*>(mIterator.Get()); + mIterator.Next(); + return token; +} + +// member variables +static const PLDHashTableOps gTokenTableOps = { + PLDHashTable::HashStringKey, PLDHashTable::MatchStringKey, + PLDHashTable::MoveEntryStub, PLDHashTable::ClearEntryStub, nullptr}; + +TokenHash::TokenHash(uint32_t aEntrySize) + : mTokenTable(&gTokenTableOps, aEntrySize, 128) { + mEntrySize = aEntrySize; +} + +TokenHash::~TokenHash() {} + +nsresult TokenHash::clearTokens() { + // we re-use the tokenizer when classifying multiple messages, + // so this gets called after every message classification. + mTokenTable.ClearAndPrepareForLength(128); + mWordPool.Clear(); + return NS_OK; +} + +char* TokenHash::copyWord(const char* word, uint32_t len) { + return ArenaStrdup(Substring(word, len), mWordPool); +} + +inline BaseToken* TokenHash::get(const char* word) { + PLDHashEntryHdr* entry = mTokenTable.Search(word); + if (entry) return static_cast<BaseToken*>(entry); + return NULL; +} + +BaseToken* TokenHash::add(const char* word) { + if (!word || !*word) { + NS_ERROR("Trying to add a null word"); + return nullptr; + } + + MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, ("add word: %s", word)); + + PLDHashEntryHdr* entry = mTokenTable.Add(word, mozilla::fallible); + BaseToken* token = static_cast<BaseToken*>(entry); + if (token) { + if (token->mWord == NULL) { + uint32_t len = strlen(word); + NS_ASSERTION(len != 0, "adding zero length word to tokenizer"); + if (!len) + MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, + ("adding zero length word to tokenizer")); + token->mWord = copyWord(word, len); + NS_ASSERTION(token->mWord, "copyWord failed"); + if (!token->mWord) { + MOZ_LOG(BayesianFilterLogModule, LogLevel::Error, + ("copyWord failed: %s (%d)", word, len)); + mTokenTable.RawRemove(entry); + return NULL; + } + } + } + return token; +} + +inline uint32_t TokenHash::countTokens() { return mTokenTable.EntryCount(); } + +inline TokenEnumeration TokenHash::getTokens() { + return TokenEnumeration(&mTokenTable); +} + +Tokenizer::Tokenizer() + : TokenHash(sizeof(Token)), + mBodyDelimiters(kBayesianFilterTokenDelimiters), + mHeaderDelimiters(kBayesianFilterTokenDelimiters), + mCustomHeaderTokenization(false), + mMaxLengthForToken(kMaxLengthForToken), + mIframeToDiv(false) { + nsresult rv; + nsCOMPtr<nsIPrefService> prefs = + do_GetService(NS_PREFSERVICE_CONTRACTID, &rv); + NS_ENSURE_SUCCESS_VOID(rv); + + nsCOMPtr<nsIPrefBranch> prefBranch; + rv = prefs->GetBranch("mailnews.bayesian_spam_filter.", + getter_AddRefs(prefBranch)); + NS_ENSURE_SUCCESS_VOID(rv); // no branch defined, just use defaults + + /* + * RSS feeds store their summary as alternate content of an iframe. But due + * to bug 365953, this is not seen by the serializer. As a workaround, allow + * the tokenizer to replace the iframe with div for tokenization. + */ + rv = prefBranch->GetBoolPref("iframe_to_div", &mIframeToDiv); + if (NS_FAILED(rv)) mIframeToDiv = false; + + /* + * the list of delimiters used to tokenize the message and body + * defaults to the value in kBayesianFilterTokenDelimiters, but may be + * set with the following preferences for the body and header + * separately. + * + * \t, \n, \v, \f, \r, and \\ will be escaped to their normal + * C-library values, all other two-letter combinations beginning with \ + * will be ignored. + */ + + prefBranch->GetCharPref("body_delimiters", mBodyDelimiters); + if (!mBodyDelimiters.IsEmpty()) + UnescapeCString(mBodyDelimiters); + else // prefBranch empties the result when it fails :( + mBodyDelimiters.Assign(kBayesianFilterTokenDelimiters); + + prefBranch->GetCharPref("header_delimiters", mHeaderDelimiters); + if (!mHeaderDelimiters.IsEmpty()) + UnescapeCString(mHeaderDelimiters); + else + mHeaderDelimiters.Assign(kBayesianFilterTokenDelimiters); + + /* + * Extensions may wish to enable or disable tokenization of certain headers. + * Define any headers to enable/disable in a string preference like this: + * "mailnews.bayesian_spam_filter.tokenizeheader.headername" + * + * where "headername" is the header to tokenize. For example, to tokenize the + * header "x-spam-status" use the preference: + * + * "mailnews.bayesian_spam_filter.tokenizeheader.x-spam-status" + * + * The value of the string preference will be interpreted in one of + * four ways, depending on the value: + * + * If "false" then do not tokenize that header + * If "full" then add the entire header value as a token, + * without breaking up into subtokens using delimiters + * If "standard" then tokenize the header using as delimiters the current + * value of the generic header delimiters + * Any other string is interpreted as a list of delimiters to use to parse + * the header. \t, \n, \v, \f, \r, and \\ will be escaped to their normal + * C-library values, all other two-letter combinations beginning with \ + * will be ignored. + * + * Header names in the preference should be all lower case + * + * Extensions may also set the maximum length of a token (default is + * kMaxLengthForToken) by setting the int preference: + * "mailnews.bayesian_spam_filter.maxlengthfortoken" + */ + + nsTArray<nsCString> headers; + + // get customized maximum token length + int32_t maxLengthForToken; + rv = prefBranch->GetIntPref("maxlengthfortoken", &maxLengthForToken); + mMaxLengthForToken = + NS_SUCCEEDED(rv) ? uint32_t(maxLengthForToken) : kMaxLengthForToken; + + rv = prefs->GetBranch("mailnews.bayesian_spam_filter.tokenizeheader.", + getter_AddRefs(prefBranch)); + if (NS_SUCCEEDED(rv)) rv = prefBranch->GetChildList("", headers); + + if (NS_SUCCEEDED(rv)) { + mCustomHeaderTokenization = true; + for (auto& header : headers) { + nsCString value; + prefBranch->GetCharPref(header.get(), value); + if (value.EqualsLiteral("false")) { + mDisabledHeaders.AppendElement(header); + continue; + } + mEnabledHeaders.AppendElement(header); + if (value.EqualsLiteral("standard")) + value.SetIsVoid(true); // Void means use default delimiter + else if (value.EqualsLiteral("full")) + value.Truncate(); // Empty means add full header + else + UnescapeCString(value); + mEnabledHeadersDelimiters.AppendElement(value); + } + } +} + +Tokenizer::~Tokenizer() {} + +inline Token* Tokenizer::get(const char* word) { + return static_cast<Token*>(TokenHash::get(word)); +} + +Token* Tokenizer::add(const char* word, uint32_t count) { + MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, + ("add word: %s (count=%d)", word, count)); + + Token* token = static_cast<Token*>(TokenHash::add(word)); + if (token) { + token->mCount += count; // hash code initializes this to zero + MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, + ("adding word to tokenizer: %s (count=%d) (mCount=%d)", word, count, + token->mCount)); + } + return token; +} + +static bool isDecimalNumber(const char* word) { + const char* p = word; + if (*p == '-') ++p; + char c; + while ((c = *p++)) { + if (!isdigit((unsigned char)c)) return false; + } + return true; +} + +static bool isASCII(const char* word) { + const unsigned char* p = (const unsigned char*)word; + unsigned char c; + while ((c = *p++)) { + if (c > 127) return false; + } + return true; +} + +inline bool isUpperCase(char c) { return ('A' <= c) && (c <= 'Z'); } + +static char* toLowerCase(char* str) { + char c, *p = str; + while ((c = *p++)) { + if (isUpperCase(c)) p[-1] = c + ('a' - 'A'); + } + return str; +} + +void Tokenizer::addTokenForHeader(const char* aTokenPrefix, nsACString& aValue, + bool aTokenizeValue, + const char* aDelimiters) { + if (aValue.Length()) { + ToLowerCase(aValue); + if (!aTokenizeValue) { + nsCString tmpStr; + tmpStr.Assign(aTokenPrefix); + tmpStr.Append(':'); + tmpStr.Append(aValue); + + add(tmpStr.get()); + } else { + char* word; + nsCString str(aValue); + char* next = str.BeginWriting(); + const char* delimiters = + !aDelimiters ? mHeaderDelimiters.get() : aDelimiters; + while ((word = NS_strtok(delimiters, &next)) != NULL) { + if (strlen(word) < kMinLengthForToken) continue; + if (isDecimalNumber(word)) continue; + if (isASCII(word)) { + nsCString tmpStr; + tmpStr.Assign(aTokenPrefix); + tmpStr.Append(':'); + tmpStr.Append(word); + add(tmpStr.get()); + } + } + } + } +} + +void Tokenizer::tokenizeAttachments( + nsTArray<RefPtr<nsIPropertyBag2>>& attachments) { + for (auto attachment : attachments) { + nsCString contentType; + ToLowerCase(contentType); + attachment->GetPropertyAsAUTF8String(u"contentType"_ns, contentType); + addTokenForHeader("attachment/content-type", contentType); + + nsCString displayName; + attachment->GetPropertyAsAUTF8String(u"displayName"_ns, displayName); + ToLowerCase(displayName); + addTokenForHeader("attachment/filename", displayName); + } +} + +void Tokenizer::tokenizeHeaders(nsTArray<nsCString>& aHeaderNames, + nsTArray<nsCString>& aHeaderValues) { + nsCString headerValue; + nsAutoCString + headerName; // we'll be normalizing all header names to lower case + + for (uint32_t i = 0; i < aHeaderNames.Length(); i++) { + headerName = aHeaderNames[i]; + ToLowerCase(headerName); + headerValue = aHeaderValues[i]; + + bool headerProcessed = false; + if (mCustomHeaderTokenization) { + // Process any exceptions set from preferences + for (uint32_t i = 0; i < mEnabledHeaders.Length(); i++) + if (headerName.Equals(mEnabledHeaders[i])) { + if (mEnabledHeadersDelimiters[i].IsVoid()) + // tokenize with standard delimiters for all headers + addTokenForHeader(headerName.get(), headerValue, true); + else if (mEnabledHeadersDelimiters[i].IsEmpty()) + // do not break the header into tokens + addTokenForHeader(headerName.get(), headerValue); + else + // use the delimiter in mEnabledHeadersDelimiters + addTokenForHeader(headerName.get(), headerValue, true, + mEnabledHeadersDelimiters[i].get()); + headerProcessed = true; + break; // we found the header, no need to look for more custom values + } + + for (uint32_t i = 0; i < mDisabledHeaders.Length(); i++) { + if (headerName.Equals(mDisabledHeaders[i])) { + headerProcessed = true; + break; + } + } + + if (headerProcessed) continue; + } + + switch (headerName.First()) { + case 'c': + if (headerName.EqualsLiteral("content-type")) { + nsresult rv; + nsCOMPtr<nsIMIMEHeaderParam> mimehdrpar = + do_GetService(NS_MIMEHEADERPARAM_CONTRACTID, &rv); + if (NS_FAILED(rv)) break; + + // extract the charset parameter + nsCString parameterValue; + mimehdrpar->GetParameterInternal(headerValue, "charset", nullptr, + nullptr, + getter_Copies(parameterValue)); + addTokenForHeader("charset", parameterValue); + + // create a token containing just the content type + mimehdrpar->GetParameterInternal(headerValue, "type", nullptr, + nullptr, + getter_Copies(parameterValue)); + if (!parameterValue.Length()) + mimehdrpar->GetParameterInternal( + headerValue, nullptr /* use first unnamed param */, nullptr, + nullptr, getter_Copies(parameterValue)); + addTokenForHeader("content-type/type", parameterValue); + + // XXX: should we add a token for the entire content-type header as + // well or just these parts we have extracted? + } + break; + case 'r': + if (headerName.EqualsLiteral("received")) { + // look for the string "may be forged" in the received headers. + // sendmail sometimes adds this hint This does not compile on linux + // yet. Need to figure out why. Commenting out for now if + // (FindInReadable(FORGED_RECEIVED_HEADER_HINT, headerValue)) + // addTokenForHeader(headerName.get(), FORGED_RECEIVED_HEADER_HINT); + } + + // leave out reply-to + break; + case 's': + if (headerName.EqualsLiteral("subject")) { + // we want to tokenize the subject + addTokenForHeader(headerName.get(), headerValue, true); + } + + // important: leave out sender field. Too strong of an indicator + break; + case 'x': // (2) X-Mailer / user-agent works best if it is untokenized, + // just fold the case and any leading/trailing white space + // all headers beginning with x-mozilla are being changed by us, so + // ignore + if (StringBeginsWith(headerName, "x-mozilla"_ns)) break; + // fall through + [[fallthrough]]; + case 'u': + addTokenForHeader(headerName.get(), headerValue); + break; + default: + addTokenForHeader(headerName.get(), headerValue); + break; + } // end switch + } +} + +void Tokenizer::tokenize_ascii_word(char* aWord) { + // always deal with normalized lower case strings + toLowerCase(aWord); + uint32_t wordLength = strlen(aWord); + + // if the wordLength is within our accepted token limit, then add it + if (wordLength >= kMinLengthForToken && wordLength <= mMaxLengthForToken) + add(aWord); + else if (wordLength > mMaxLengthForToken) { + // don't skip over the word if it looks like an email address, + // there is value in adding tokens for addresses + nsDependentCString word(aWord, + wordLength); // CHEAP, no allocation occurs here... + + // XXX: i think the 40 byte check is just for perf reasons...if the email + // address is longer than that then forget about it. + const char* atSign = strchr(aWord, '@'); + if (wordLength < 40 && strchr(aWord, '.') && atSign && + !strchr(atSign + 1, '@')) { + uint32_t numBytesToSep = atSign - aWord; + if (numBytesToSep < + wordLength - 1) // if the @ sign is the last character, it must not + // be an email address + { + // split the john@foo.com into john and foo.com, treat them as separate + // tokens + nsCString emailNameToken; + emailNameToken.AssignLiteral("email name:"); + emailNameToken.Append(Substring(word, 0, numBytesToSep++)); + add(emailNameToken.get()); + nsCString emailAddrToken; + emailAddrToken.AssignLiteral("email addr:"); + emailAddrToken.Append( + Substring(word, numBytesToSep, wordLength - numBytesToSep)); + add(emailAddrToken.get()); + return; + } + } + + // there is value in generating a token indicating the number + // of characters we are skipping. We'll round to the nearest 10 + nsCString skipToken; + skipToken.AssignLiteral("skip:"); + skipToken.Append(word[0]); + skipToken.Append(' '); + skipToken.AppendInt((wordLength / 10) * 10); + add(skipToken.get()); + } +} + +// Copied from mozilla/intl/lwbrk/WordBreaker.cpp + +#define ASCII_IS_ALPHA(c) \ + ((('a' <= (c)) && ((c) <= 'z')) || (('A' <= (c)) && ((c) <= 'Z'))) +#define ASCII_IS_DIGIT(c) (('0' <= (c)) && ((c) <= '9')) +#define ASCII_IS_SPACE(c) \ + ((' ' == (c)) || ('\t' == (c)) || ('\r' == (c)) || ('\n' == (c))) +#define IS_ALPHABETICAL_SCRIPT(c) ((c) < 0x2E80) + +// we change the beginning of IS_HAN from 0x4e00 to 0x3400 to relfect +// Unicode 3.0 +#define IS_HAN(c) \ + ((0x3400 <= (c)) && ((c) <= 0x9fff)) || ((0xf900 <= (c)) && ((c) <= 0xfaff)) +#define IS_KATAKANA(c) ((0x30A0 <= (c)) && ((c) <= 0x30FF)) +#define IS_HIRAGANA(c) ((0x3040 <= (c)) && ((c) <= 0x309F)) +#define IS_HALFWIDTHKATAKANA(c) ((0xFF60 <= (c)) && ((c) <= 0xFF9F)) + +// Return true if aChar belongs to a SEAsian script that is written without +// word spaces, so we need to use the "complex breaker" to find possible word +// boundaries. (https://en.wikipedia.org/wiki/Scriptio_continua) +// (How well this works depends on the level of platform support for finding +// possible line breaks - or possible word boundaries - in the particular +// script. Thai, at least, works pretty well on the major desktop OSes. If +// the script is not supported by the platform, we just won't find any useful +// boundaries.) +static bool IsScriptioContinua(char16_t aChar) { + Script sc = UnicodeProperties::GetScriptCode(aChar); + return sc == Script::THAI || sc == Script::MYANMAR || sc == Script::KHMER || + sc == Script::JAVANESE || sc == Script::BALINESE || + sc == Script::SUNDANESE || sc == Script::LAO; +} + +// one subtract and one conditional jump should be faster than two conditional +// jump on most recent system. +#define IN_RANGE(x, low, high) ((uint16_t)((x) - (low)) <= (high) - (low)) + +#define IS_JA_HIRAGANA(x) IN_RANGE(x, 0x3040, 0x309F) +// swapping the range using xor operation to reduce conditional jump. +#define IS_JA_KATAKANA(x) \ + (IN_RANGE(x ^ 0x0004, 0x30A0, 0x30FE) || (IN_RANGE(x, 0xFF66, 0xFF9F))) +#define IS_JA_KANJI(x) \ + (IN_RANGE(x, 0x2E80, 0x2FDF) || IN_RANGE(x, 0x4E00, 0x9FAF)) +#define IS_JA_KUTEN(x) (((x) == 0x3001) || ((x) == 0xFF64) || ((x) == 0xFF0E)) +#define IS_JA_TOUTEN(x) (((x) == 0x3002) || ((x) == 0xFF61) || ((x) == 0xFF0C)) +#define IS_JA_SPACE(x) ((x) == 0x3000) +#define IS_JA_FWLATAIN(x) IN_RANGE(x, 0xFF01, 0xFF5E) +#define IS_JA_FWNUMERAL(x) IN_RANGE(x, 0xFF10, 0xFF19) + +#define IS_JAPANESE_SPECIFIC(x) \ + (IN_RANGE(x, 0x3040, 0x30FF) || IN_RANGE(x, 0xFF01, 0xFF9F)) + +enum char_class { + others = 0, + space, + hiragana, + katakana, + kanji, + kuten, + touten, + kigou, + fwlatain, + ascii +}; + +static char_class getCharClass(char16_t c) { + char_class charClass = others; + + if (IS_JA_HIRAGANA(c)) + charClass = hiragana; + else if (IS_JA_KATAKANA(c)) + charClass = katakana; + else if (IS_JA_KANJI(c)) + charClass = kanji; + else if (IS_JA_KUTEN(c)) + charClass = kuten; + else if (IS_JA_TOUTEN(c)) + charClass = touten; + else if (IS_JA_FWLATAIN(c)) + charClass = fwlatain; + + return charClass; +} + +static bool isJapanese(const char* word) { + nsString text = NS_ConvertUTF8toUTF16(word); + const char16_t* p = (const char16_t*)text.get(); + char16_t c; + + // it is japanese chunk if it contains any hiragana or katakana. + while ((c = *p++)) + if (IS_JAPANESE_SPECIFIC(c)) return true; + + return false; +} + +static bool isFWNumeral(const char16_t* p1, const char16_t* p2) { + for (; p1 < p2; p1++) + if (!IS_JA_FWNUMERAL(*p1)) return false; + + return true; +} + +// The japanese tokenizer was added as part of Bug #277354 +void Tokenizer::tokenize_japanese_word(char* chunk) { + MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, + ("entering tokenize_japanese_word(%s)", chunk)); + + nsString srcStr = NS_ConvertUTF8toUTF16(chunk); + const char16_t* p1 = srcStr.get(); + const char16_t* p2 = p1; + if (!*p2) return; + + char_class cc = getCharClass(*p2); + while (*(++p2)) { + if (cc == getCharClass(*p2)) continue; + + nsCString token = NS_ConvertUTF16toUTF8(p1, p2 - p1); + if ((!isDecimalNumber(token.get())) && (!isFWNumeral(p1, p2))) { + nsCString tmpStr; + tmpStr.AppendLiteral("JA:"); + tmpStr.Append(token); + add(tmpStr.get()); + } + + cc = getCharClass(*p2); + p1 = p2; + } +} + +nsresult Tokenizer::stripHTML(const nsAString& inString, nsAString& outString) { + uint32_t flags = nsIDocumentEncoder::OutputLFLineBreak | + nsIDocumentEncoder::OutputNoScriptContent | + nsIDocumentEncoder::OutputNoFramesContent | + nsIDocumentEncoder::OutputBodyOnly; + nsCOMPtr<nsIParserUtils> utils = do_GetService(NS_PARSERUTILS_CONTRACTID); + return utils->ConvertToPlainText(inString, flags, 80, outString); +} + +// Copied from WorfdBreker.cpp due to changes in bug 1728708. +enum WordBreakClass : uint8_t { + kWbClassSpace = 0, + kWbClassAlphaLetter, + kWbClassPunct, + kWbClassHanLetter, + kWbClassKatakanaLetter, + kWbClassHiraganaLetter, + kWbClassHWKatakanaLetter, + kWbClassScriptioContinua +}; + +WordBreakClass GetWordBreakClass(char16_t c) { + // begin of the hack + + if (IS_ALPHABETICAL_SCRIPT(c)) { + if (IS_ASCII(c)) { + if (ASCII_IS_SPACE(c)) { + return WordBreakClass::kWbClassSpace; + } + if (ASCII_IS_ALPHA(c) || ASCII_IS_DIGIT(c) || (c == '_')) { + return WordBreakClass::kWbClassAlphaLetter; + } + return WordBreakClass::kWbClassPunct; + } + if (c == 0x00A0 /*NBSP*/) { + return WordBreakClass::kWbClassSpace; + } + if (mozilla::unicode::GetGenCategory(c) == nsUGenCategory::kPunctuation) { + return WordBreakClass::kWbClassPunct; + } + if (IsScriptioContinua(c)) { + return WordBreakClass::kWbClassScriptioContinua; + } + return WordBreakClass::kWbClassAlphaLetter; + } + if (IS_HAN(c)) { + return WordBreakClass::kWbClassHanLetter; + } + if (IS_KATAKANA(c)) { + return kWbClassKatakanaLetter; + } + if (IS_HIRAGANA(c)) { + return WordBreakClass::kWbClassHiraganaLetter; + } + if (IS_HALFWIDTHKATAKANA(c)) { + return WordBreakClass::kWbClassHWKatakanaLetter; + } + if (mozilla::unicode::GetGenCategory(c) == nsUGenCategory::kPunctuation) { + return WordBreakClass::kWbClassPunct; + } + if (IsScriptioContinua(c)) { + return WordBreakClass::kWbClassScriptioContinua; + } + return WordBreakClass::kWbClassAlphaLetter; +} + +// Copied from nsSemanticUnitScanner.cpp which was removed in bug 1368418. +nsresult Tokenizer::ScannerNext(const char16_t* text, int32_t length, + int32_t pos, bool isLastBuffer, int32_t* begin, + int32_t* end, bool* _retval) { + // if we reach the end, just return + if (pos >= length) { + *begin = pos; + *end = pos; + *_retval = false; + return NS_OK; + } + + WordBreakClass char_class = GetWordBreakClass(text[pos]); + + // If we are in Chinese mode, return one Han letter at a time. + // We should not do this if we are in Japanese or Korean mode. + if (WordBreakClass::kWbClassHanLetter == char_class) { + *begin = pos; + *end = pos + 1; + *_retval = true; + return NS_OK; + } + + int32_t next; + // Find the next "word". + next = + mozilla::intl::WordBreaker::Next(text, (uint32_t)length, (uint32_t)pos); + + // If we don't have enough text to make decision, return. + if (next == NS_WORDBREAKER_NEED_MORE_TEXT) { + *begin = pos; + *end = isLastBuffer ? length : pos; + *_retval = isLastBuffer; + return NS_OK; + } + + // If what we got is space or punct, look at the next break. + if (char_class == WordBreakClass::kWbClassSpace || + char_class == WordBreakClass::kWbClassPunct) { + // If the next "word" is not letters, + // call itself recursively with the new pos. + return ScannerNext(text, length, next, isLastBuffer, begin, end, _retval); + } + + // For the rest, return. + *begin = pos; + *end = next; + *_retval = true; + return NS_OK; +} + +void Tokenizer::tokenize(const char* aText) { + MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, ("tokenize: %s", aText)); + + // strip out HTML tags before we begin processing + // uggh but first we have to blow up our string into UCS2 + // since that's what the document encoder wants. UTF8/UCS2, I wish we all + // spoke the same language here.. + nsString text = NS_ConvertUTF8toUTF16(aText); + nsString strippedUCS2; + + // RSS feeds store their summary information as an iframe. But due to + // bug 365953, we can't see those in the plaintext serializer. As a + // workaround, allow an option to replace iframe with div in the message + // text. We disable by default, since most people won't be applying bayes + // to RSS + + if (mIframeToDiv) { + text.ReplaceSubstring(u"<iframe"_ns, u"<div"_ns); + text.ReplaceSubstring(u"/iframe>"_ns, u"/div>"_ns); + } + + stripHTML(text, strippedUCS2); + + // convert 0x3000(full width space) into 0x0020 + char16_t* substr_start = strippedUCS2.BeginWriting(); + char16_t* substr_end = strippedUCS2.EndWriting(); + while (substr_start != substr_end) { + if (*substr_start == 0x3000) *substr_start = 0x0020; + ++substr_start; + } + + nsCString strippedStr = NS_ConvertUTF16toUTF8(strippedUCS2); + char* strippedText = strippedStr.BeginWriting(); + MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, + ("tokenize stripped html: %s", strippedText)); + + char* word; + char* next = strippedText; + while ((word = NS_strtok(mBodyDelimiters.get(), &next)) != NULL) { + if (!*word) continue; + if (isDecimalNumber(word)) continue; + if (isASCII(word)) + tokenize_ascii_word(word); + else if (isJapanese(word)) + tokenize_japanese_word(word); + else { + nsresult rv; + // Convert this word from UTF-8 into UCS2. + NS_ConvertUTF8toUTF16 uword(word); + ToLowerCase(uword); + const char16_t* utext = uword.get(); + int32_t len = uword.Length(), pos = 0, begin, end; + bool gotUnit; + while (pos < len) { + rv = ScannerNext(utext, len, pos, true, &begin, &end, &gotUnit); + if (NS_SUCCEEDED(rv) && gotUnit) { + NS_ConvertUTF16toUTF8 utfUnit(utext + begin, end - begin); + add(utfUnit.get()); + // Advance to end of current unit. + pos = end; + } else { + break; + } + } + } + } +} + +// helper function to un-escape \n, \t, etc from a CString +void Tokenizer::UnescapeCString(nsCString& aCString) { + nsAutoCString result; + + const char* readEnd = aCString.EndReading(); + result.SetLength(aCString.Length()); + char* writeStart = result.BeginWriting(); + char* writeIter = writeStart; + + bool inEscape = false; + for (const char* readIter = aCString.BeginReading(); readIter != readEnd; + readIter++) { + if (!inEscape) { + if (*readIter == '\\') + inEscape = true; + else + *(writeIter++) = *readIter; + } else { + inEscape = false; + switch (*readIter) { + case '\\': + *(writeIter++) = '\\'; + break; + case 't': + *(writeIter++) = '\t'; + break; + case 'n': + *(writeIter++) = '\n'; + break; + case 'v': + *(writeIter++) = '\v'; + break; + case 'f': + *(writeIter++) = '\f'; + break; + case 'r': + *(writeIter++) = '\r'; + break; + default: + // all other escapes are ignored + break; + } + } + } + result.Truncate(writeIter - writeStart); + aCString.Assign(result); +} + +Token* Tokenizer::copyTokens() { + uint32_t count = countTokens(); + if (count > 0) { + Token* tokens = new Token[count]; + if (tokens) { + Token* tp = tokens; + TokenEnumeration e(&mTokenTable); + while (e.hasMoreTokens()) { + Token* src = static_cast<Token*>(e.nextToken()); + tp->clone(*src); + ++tp; + } + } + return tokens; + } + return NULL; +} + +class TokenAnalyzer { + public: + virtual ~TokenAnalyzer() {} + + virtual void analyzeTokens(Tokenizer& tokenizer) = 0; + void setTokenListener(nsIStreamListener* aTokenListener) { + mTokenListener = aTokenListener; + } + + void setSource(const nsACString& sourceURI) { mTokenSource = sourceURI; } + + nsCOMPtr<nsIStreamListener> mTokenListener; + nsCString mTokenSource; +}; + +/** + * This class downloads the raw content of an email message, buffering until + * complete segments are seen, that is until a linefeed is seen, although + * any of the valid token separators would do. This could be a further + * refinement. + */ +class TokenStreamListener : public nsIStreamListener { + public: + NS_DECL_ISUPPORTS + NS_DECL_NSIREQUESTOBSERVER + NS_DECL_NSISTREAMLISTENER + + explicit TokenStreamListener(TokenAnalyzer* analyzer); + + protected: + virtual ~TokenStreamListener(); + TokenAnalyzer* mAnalyzer; + char* mBuffer; + uint32_t mBufferSize; + uint32_t mLeftOverCount; + Tokenizer mTokenizer; + bool mSetAttachmentFlag; +}; + +const uint32_t kBufferSize = 16384; + +TokenStreamListener::TokenStreamListener(TokenAnalyzer* analyzer) + : mAnalyzer(analyzer), + mBuffer(NULL), + mBufferSize(kBufferSize), + mLeftOverCount(0), + mSetAttachmentFlag(false) {} + +TokenStreamListener::~TokenStreamListener() { + delete[] mBuffer; + delete mAnalyzer; +} + +NS_IMPL_ISUPPORTS(TokenStreamListener, nsIRequestObserver, nsIStreamListener) + +/* void onStartRequest (in nsIRequest aRequest); */ +NS_IMETHODIMP TokenStreamListener::OnStartRequest(nsIRequest* aRequest) { + mLeftOverCount = 0; + if (!mBuffer) { + mBuffer = new char[mBufferSize]; + NS_ENSURE_TRUE(mBuffer, NS_ERROR_OUT_OF_MEMORY); + } + + return NS_OK; +} + +/* void onDataAvailable (in nsIRequest aRequest, in nsIInputStream aInputStream, + * in unsigned long long aOffset, in unsigned long aCount); */ +NS_IMETHODIMP TokenStreamListener::OnDataAvailable(nsIRequest* aRequest, + nsIInputStream* aInputStream, + uint64_t aOffset, + uint32_t aCount) { + nsresult rv = NS_OK; + + while (aCount > 0) { + uint32_t readCount, totalCount = (aCount + mLeftOverCount); + if (totalCount >= mBufferSize) { + readCount = mBufferSize - mLeftOverCount - 1; + } else { + readCount = aCount; + } + + // mBuffer is supposed to be allocated in onStartRequest. But something + // is causing that to not happen, so as a last-ditch attempt we'll + // do it here. + if (!mBuffer) { + mBuffer = new char[mBufferSize]; + NS_ENSURE_TRUE(mBuffer, NS_ERROR_OUT_OF_MEMORY); + } + + char* buffer = mBuffer; + rv = aInputStream->Read(buffer + mLeftOverCount, readCount, &readCount); + if (NS_FAILED(rv)) break; + + if (readCount == 0) { + rv = NS_ERROR_UNEXPECTED; + NS_WARNING("failed to tokenize"); + break; + } + + aCount -= readCount; + + /* consume the tokens up to the last legal token delimiter in the buffer. */ + totalCount = (readCount + mLeftOverCount); + buffer[totalCount] = '\0'; + char* lastDelimiter = NULL; + char* scan = buffer + totalCount; + while (scan > buffer) { + if (strchr(mTokenizer.mBodyDelimiters.get(), *--scan)) { + lastDelimiter = scan; + break; + } + } + + if (lastDelimiter) { + *lastDelimiter = '\0'; + mTokenizer.tokenize(buffer); + + uint32_t consumedCount = 1 + (lastDelimiter - buffer); + mLeftOverCount = totalCount - consumedCount; + if (mLeftOverCount) + memmove(buffer, buffer + consumedCount, mLeftOverCount); + } else { + /* didn't find a delimiter, keep the whole buffer around. */ + mLeftOverCount = totalCount; + if (totalCount >= (mBufferSize / 2)) { + uint32_t newBufferSize = mBufferSize * 2; + char* newBuffer = new char[newBufferSize]; + NS_ENSURE_TRUE(newBuffer, NS_ERROR_OUT_OF_MEMORY); + memcpy(newBuffer, mBuffer, mLeftOverCount); + delete[] mBuffer; + mBuffer = newBuffer; + mBufferSize = newBufferSize; + } + } + } + + return rv; +} + +/* void onStopRequest (in nsIRequest aRequest, in nsresult aStatusCode); */ +NS_IMETHODIMP TokenStreamListener::OnStopRequest(nsIRequest* aRequest, + nsresult aStatusCode) { + nsCOMPtr<nsIMailChannel> mailChannel = do_QueryInterface(aRequest); + if (mailChannel) { + nsTArray<nsCString> headerNames; + nsTArray<nsCString> headerValues; + mailChannel->GetHeaderNames(headerNames); + mailChannel->GetHeaderValues(headerValues); + mTokenizer.tokenizeHeaders(headerNames, headerValues); + + nsTArray<RefPtr<nsIPropertyBag2>> attachments; + mailChannel->GetAttachments(attachments); + mTokenizer.tokenizeAttachments(attachments); + } + + if (mLeftOverCount) { + /* assume final buffer is complete. */ + mBuffer[mLeftOverCount] = '\0'; + mTokenizer.tokenize(mBuffer); + } + + /* finally, analyze the tokenized message. */ + MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, + ("analyze the tokenized message")); + if (mAnalyzer) mAnalyzer->analyzeTokens(mTokenizer); + + return NS_OK; +} + +/* Implementation file */ + +NS_IMPL_ISUPPORTS(nsBayesianFilter, nsIMsgFilterPlugin, nsIJunkMailPlugin, + nsIMsgCorpus, nsISupportsWeakReference, nsIObserver) + +nsBayesianFilter::nsBayesianFilter() : mTrainingDataDirty(false) { + int32_t junkThreshold = 0; + nsresult rv; + nsCOMPtr<nsIPrefBranch> pPrefBranch( + do_GetService(NS_PREFSERVICE_CONTRACTID, &rv)); + if (pPrefBranch) + pPrefBranch->GetIntPref("mail.adaptivefilters.junk_threshold", + &junkThreshold); + + mJunkProbabilityThreshold = (static_cast<double>(junkThreshold)) / 100.0; + if (mJunkProbabilityThreshold == 0 || mJunkProbabilityThreshold >= 1) + mJunkProbabilityThreshold = kDefaultJunkThreshold; + + MOZ_LOG(BayesianFilterLogModule, LogLevel::Warning, + ("junk probability threshold: %f", mJunkProbabilityThreshold)); + + mCorpus.readTrainingData(); + + // get parameters for training data flushing, from the prefs + + nsCOMPtr<nsIPrefBranch> prefBranch; + + nsCOMPtr<nsIPrefService> prefs = + do_GetService(NS_PREFSERVICE_CONTRACTID, &rv); + NS_ASSERTION(NS_SUCCEEDED(rv), "failed accessing preferences service"); + rv = prefs->GetBranch(nullptr, getter_AddRefs(prefBranch)); + NS_ASSERTION(NS_SUCCEEDED(rv), "failed getting preferences branch"); + + rv = prefBranch->GetIntPref( + "mailnews.bayesian_spam_filter.flush.minimum_interval", + &mMinFlushInterval); + // it is not a good idea to allow a minimum interval of under 1 second + if (NS_FAILED(rv) || (mMinFlushInterval <= 1000)) + mMinFlushInterval = DEFAULT_MIN_INTERVAL_BETWEEN_WRITES; + + rv = prefBranch->GetIntPref("mailnews.bayesian_spam_filter.junk_maxtokens", + &mMaximumTokenCount); + if (NS_FAILED(rv)) + mMaximumTokenCount = 0; // which means do not limit token counts + MOZ_LOG(BayesianFilterLogModule, LogLevel::Warning, + ("maximum junk tokens: %d", mMaximumTokenCount)); + + // give a default capacity to the memory structure used to store + // per-message/per-trait token data + mAnalysisStore.SetCapacity(kAnalysisStoreCapacity); + + // dummy 0th element. Index 0 means "end of list" so we need to + // start from 1 + AnalysisPerToken analysisPT(0, 0.0, 0.0); + mAnalysisStore.AppendElement(analysisPT); + mNextAnalysisIndex = 1; +} + +nsresult nsBayesianFilter::Init() { + nsCOMPtr<nsIObserverService> observerService = + mozilla::services::GetObserverService(); + if (observerService) + observerService->AddObserver(this, "profile-before-change", true); + return NS_OK; +} + +void nsBayesianFilter::TimerCallback(nsITimer* aTimer, void* aClosure) { + // we will flush the training data to disk after enough time has passed + // since the first time a message has been classified after the last flush + + nsBayesianFilter* filter = static_cast<nsBayesianFilter*>(aClosure); + filter->mCorpus.writeTrainingData(filter->mMaximumTokenCount); + filter->mTrainingDataDirty = false; +} + +nsBayesianFilter::~nsBayesianFilter() { + if (mTimer) { + mTimer->Cancel(); + mTimer = nullptr; + } + // call shutdown when we are going away in case we need + // to flush the training set to disk + Shutdown(); +} + +// this object is used for one call to classifyMessage or classifyMessages(). +// So if we're classifying multiple messages, this object will be used for each +// message. It's going to hold a reference to itself, basically, to stay in +// memory. +class MessageClassifier : public TokenAnalyzer { + public: + // full classifier with arbitrary traits + MessageClassifier(nsBayesianFilter* aFilter, + nsIJunkMailClassificationListener* aJunkListener, + nsIMsgTraitClassificationListener* aTraitListener, + nsIMsgTraitDetailListener* aDetailListener, + const nsTArray<uint32_t>& aProTraits, + const nsTArray<uint32_t>& aAntiTraits, + nsIMsgWindow* aMsgWindow, + const nsTArray<nsCString>& aMessageURIs) + : mFilter(aFilter), + mJunkMailPlugin(aFilter), + mJunkListener(aJunkListener), + mTraitListener(aTraitListener), + mDetailListener(aDetailListener), + mProTraits(aProTraits.Clone()), + mAntiTraits(aAntiTraits.Clone()), + mMsgWindow(aMsgWindow), + mMessageURIs(aMessageURIs.Clone()), + mCurMessageToClassify(0) { + MOZ_ASSERT(aProTraits.Length() == aAntiTraits.Length()); + } + + // junk-only classifier + MessageClassifier(nsBayesianFilter* aFilter, + nsIJunkMailClassificationListener* aJunkListener, + nsIMsgWindow* aMsgWindow, + const nsTArray<nsCString>& aMessageURIs) + : mFilter(aFilter), + mJunkMailPlugin(aFilter), + mJunkListener(aJunkListener), + mTraitListener(nullptr), + mDetailListener(nullptr), + mMsgWindow(aMsgWindow), + mMessageURIs(aMessageURIs.Clone()), + mCurMessageToClassify(0) { + mProTraits.AppendElement(kJunkTrait); + mAntiTraits.AppendElement(kGoodTrait); + } + + virtual ~MessageClassifier() {} + virtual void analyzeTokens(Tokenizer& tokenizer) { + mFilter->classifyMessage(tokenizer, mTokenSource, mProTraits, mAntiTraits, + mJunkListener, mTraitListener, mDetailListener); + tokenizer.clearTokens(); + classifyNextMessage(); + } + + virtual void classifyNextMessage() { + if (++mCurMessageToClassify < mMessageURIs.Length()) { + MOZ_LOG(BayesianFilterLogModule, LogLevel::Warning, + ("classifyNextMessage(%s)", + mMessageURIs[mCurMessageToClassify].get())); + mFilter->tokenizeMessage(mMessageURIs[mCurMessageToClassify], mMsgWindow, + this); + } else { + // call all listeners with null parameters to signify end of batch + if (mJunkListener) + mJunkListener->OnMessageClassified(EmptyCString(), + nsIJunkMailPlugin::UNCLASSIFIED, 0); + if (mTraitListener) { + nsTArray<uint32_t> nullTraits; + nsTArray<uint32_t> nullPercents; + mTraitListener->OnMessageTraitsClassified(EmptyCString(), nullTraits, + nullPercents); + } + mTokenListener = + nullptr; // this breaks the circular ref that keeps this object alive + // so we will be destroyed as a result. + } + } + + private: + nsBayesianFilter* mFilter; + nsCOMPtr<nsIJunkMailPlugin> mJunkMailPlugin; + nsCOMPtr<nsIJunkMailClassificationListener> mJunkListener; + nsCOMPtr<nsIMsgTraitClassificationListener> mTraitListener; + nsCOMPtr<nsIMsgTraitDetailListener> mDetailListener; + nsTArray<uint32_t> mProTraits; + nsTArray<uint32_t> mAntiTraits; + nsCOMPtr<nsIMsgWindow> mMsgWindow; + nsTArray<nsCString> mMessageURIs; + uint32_t mCurMessageToClassify; // 0-based index +}; + +nsresult nsBayesianFilter::tokenizeMessage(const nsACString& aMessageURI, + nsIMsgWindow* aMsgWindow, + TokenAnalyzer* aAnalyzer) { + nsCOMPtr<nsIMsgMessageService> msgService; + nsresult rv = + GetMessageServiceFromURI(aMessageURI, getter_AddRefs(msgService)); + NS_ENSURE_SUCCESS(rv, rv); + + aAnalyzer->setSource(aMessageURI); + nsCOMPtr<nsIURI> dummyNull; + return msgService->StreamMessage( + aMessageURI, aAnalyzer->mTokenListener, aMsgWindow, nullptr, + true /* convert data */, "filter"_ns, false, getter_AddRefs(dummyNull)); +} + +// a TraitAnalysis is the per-token representation of the statistical +// calculations, basically created to group information that is then +// sorted by mDistance +struct TraitAnalysis { + uint32_t mTokenIndex; + double mDistance; + double mProbability; +}; + +// comparator required to sort an nsTArray +class compareTraitAnalysis { + public: + bool Equals(const TraitAnalysis& a, const TraitAnalysis& b) const { + return a.mDistance == b.mDistance; + } + bool LessThan(const TraitAnalysis& a, const TraitAnalysis& b) const { + return a.mDistance < b.mDistance; + } +}; + +inline double dmax(double x, double y) { return (x > y ? x : y); } +inline double dmin(double x, double y) { return (x < y ? x : y); } + +// Chi square functions are implemented by an incomplete gamma function. +// Note that chi2P's callers multiply the arguments by 2 but chi2P +// divides them by 2 again. Inlining chi2P gives the compiler a +// chance to notice this. + +// Both chi2P and nsIncompleteGammaP set *error negative on domain +// errors and nsIncompleteGammaP sets it posivive on internal errors. +// This may be useful but the chi2P callers treat any error as fatal. + +// Note that converting unsigned ints to floating point can be slow on +// some platforms (like Intel) so use signed quantities for the numeric +// routines. +static inline double chi2P(double chi2, double nu, int32_t* error) { + // domain checks; set error and return a dummy value + if (chi2 < 0.0 || nu <= 0.0) { + *error = -1; + return 0.0; + } + // reversing the arguments is intentional + return nsIncompleteGammaP(nu / 2.0, chi2 / 2.0, error); +} + +void nsBayesianFilter::classifyMessage( + Tokenizer& tokenizer, const nsACString& messageURI, + nsTArray<uint32_t>& aProTraits, nsTArray<uint32_t>& aAntiTraits, + nsIJunkMailClassificationListener* listener, + nsIMsgTraitClassificationListener* aTraitListener, + nsIMsgTraitDetailListener* aDetailListener) { + if (aProTraits.Length() != aAntiTraits.Length()) { + NS_ERROR("Each Pro trait needs a matching Anti trait"); + return; + } + Token* tokens = tokenizer.copyTokens(); + uint32_t tokenCount; + if (!tokens) { + // This can happen with problems with UTF conversion + NS_ERROR("Trying to classify a null or invalid message"); + tokenCount = 0; + // don't return so that we still call the listeners + } else { + tokenCount = tokenizer.countTokens(); + } + + /* this part is similar to the Graham algorithm with some adjustments. */ + uint32_t traitCount = aProTraits.Length(); + + // pro message counts per trait index + AutoTArray<uint32_t, kTraitAutoCapacity> numProMessages; + // anti message counts per trait index + AutoTArray<uint32_t, kTraitAutoCapacity> numAntiMessages; + // array of pro aliases per trait index + AutoTArray<nsTArray<uint32_t>, kTraitAutoCapacity> proAliasArrays; + // array of anti aliases per trait index + AutoTArray<nsTArray<uint32_t>, kTraitAutoCapacity> antiAliasArrays; + // construct the outgoing listener arrays + AutoTArray<uint32_t, kTraitAutoCapacity> traits; + AutoTArray<uint32_t, kTraitAutoCapacity> percents; + if (traitCount > kTraitAutoCapacity) { + traits.SetCapacity(traitCount); + percents.SetCapacity(traitCount); + numProMessages.SetCapacity(traitCount); + numAntiMessages.SetCapacity(traitCount); + proAliasArrays.SetCapacity(traitCount); + antiAliasArrays.SetCapacity(traitCount); + } + + nsresult rv; + nsCOMPtr<nsIMsgTraitService> traitService( + do_GetService("@mozilla.org/msg-trait-service;1", &rv)); + if (NS_FAILED(rv)) { + NS_ERROR("Failed to get trait service"); + MOZ_LOG(BayesianFilterLogModule, LogLevel::Error, + ("Failed to get trait service")); + } + + // get aliases and message counts for the pro and anti traits + for (uint32_t traitIndex = 0; traitIndex < traitCount; traitIndex++) { + nsresult rv; + + // pro trait + nsTArray<uint32_t> proAliases; + uint32_t proTrait = aProTraits[traitIndex]; + if (traitService) { + rv = traitService->GetAliases(proTrait, proAliases); + if (NS_FAILED(rv)) { + NS_ERROR("trait service failed to get aliases"); + MOZ_LOG(BayesianFilterLogModule, LogLevel::Error, + ("trait service failed to get aliases")); + } + } + proAliasArrays.AppendElement(proAliases.Clone()); + uint32_t proMessageCount = mCorpus.getMessageCount(proTrait); + for (uint32_t aliasIndex = 0; aliasIndex < proAliases.Length(); + aliasIndex++) + proMessageCount += mCorpus.getMessageCount(proAliases[aliasIndex]); + numProMessages.AppendElement(proMessageCount); + + // anti trait + nsTArray<uint32_t> antiAliases; + uint32_t antiTrait = aAntiTraits[traitIndex]; + if (traitService) { + rv = traitService->GetAliases(antiTrait, antiAliases); + if (NS_FAILED(rv)) { + NS_ERROR("trait service failed to get aliases"); + MOZ_LOG(BayesianFilterLogModule, LogLevel::Error, + ("trait service failed to get aliases")); + } + } + antiAliasArrays.AppendElement(antiAliases.Clone()); + uint32_t antiMessageCount = mCorpus.getMessageCount(antiTrait); + for (uint32_t aliasIndex = 0; aliasIndex < antiAliases.Length(); + aliasIndex++) + antiMessageCount += mCorpus.getMessageCount(antiAliases[aliasIndex]); + numAntiMessages.AppendElement(antiMessageCount); + } + + for (uint32_t i = 0; i < tokenCount; ++i) { + Token& token = tokens[i]; + CorpusToken* t = mCorpus.get(token.mWord); + if (!t) continue; + for (uint32_t traitIndex = 0; traitIndex < traitCount; traitIndex++) { + uint32_t iProCount = mCorpus.getTraitCount(t, aProTraits[traitIndex]); + // add in any counts for aliases to proTrait + for (uint32_t aliasIndex = 0; + aliasIndex < proAliasArrays[traitIndex].Length(); aliasIndex++) + iProCount += + mCorpus.getTraitCount(t, proAliasArrays[traitIndex][aliasIndex]); + double proCount = static_cast<double>(iProCount); + + uint32_t iAntiCount = mCorpus.getTraitCount(t, aAntiTraits[traitIndex]); + // add in any counts for aliases to antiTrait + for (uint32_t aliasIndex = 0; + aliasIndex < antiAliasArrays[traitIndex].Length(); aliasIndex++) + iAntiCount += + mCorpus.getTraitCount(t, antiAliasArrays[traitIndex][aliasIndex]); + double antiCount = static_cast<double>(iAntiCount); + + double prob, denom; + // Prevent a divide by zero error by setting defaults for prob + + // If there are no matching tokens at all, ignore. + if (antiCount == 0.0 && proCount == 0.0) continue; + // if only anti match, set probability to 0% + if (proCount == 0.0) prob = 0.0; + // if only pro match, set probability to 100% + else if (antiCount == 0.0) + prob = 1.0; + // not really needed, but just to be sure check the denom as well + else if ((denom = proCount * numAntiMessages[traitIndex] + + antiCount * numProMessages[traitIndex]) == 0.0) + continue; + else + prob = (proCount * numAntiMessages[traitIndex]) / denom; + + double n = proCount + antiCount; + prob = (0.225 + n * prob) / (.45 + n); + double distance = std::abs(prob - 0.5); + if (distance >= .1) { + mozilla::DebugOnly<nsresult> rv = + setAnalysis(token, traitIndex, distance, prob); + NS_ASSERTION(NS_SUCCEEDED(rv), "Problem in setAnalysis"); + } + } + } + + for (uint32_t traitIndex = 0; traitIndex < traitCount; traitIndex++) { + AutoTArray<TraitAnalysis, 1024> traitAnalyses; + // copy valid tokens into an array to sort + for (uint32_t tokenIndex = 0; tokenIndex < tokenCount; tokenIndex++) { + uint32_t storeIndex = getAnalysisIndex(tokens[tokenIndex], traitIndex); + if (storeIndex) { + TraitAnalysis ta = {tokenIndex, mAnalysisStore[storeIndex].mDistance, + mAnalysisStore[storeIndex].mProbability}; + traitAnalyses.AppendElement(ta); + } + } + + // sort the array by the distances + traitAnalyses.Sort(compareTraitAnalysis()); + uint32_t count = traitAnalyses.Length(); + uint32_t first, last = count; + const uint32_t kMaxTokens = 150; + first = (count > kMaxTokens) ? count - kMaxTokens : 0; + + // Setup the arrays to save details if needed + nsTArray<double> sArray; + nsTArray<double> hArray; + uint32_t usedTokenCount = (count > kMaxTokens) ? kMaxTokens : count; + if (aDetailListener) { + sArray.SetCapacity(usedTokenCount); + hArray.SetCapacity(usedTokenCount); + } + + double H = 1.0, S = 1.0; + int32_t Hexp = 0, Sexp = 0; + uint32_t goodclues = 0; + int e; + + // index from end to analyze most significant first + for (uint32_t ip1 = last; ip1 != first; --ip1) { + TraitAnalysis& ta = traitAnalyses[ip1 - 1]; + if (ta.mDistance > 0.0) { + goodclues++; + double value = ta.mProbability; + S *= (1.0 - value); + H *= value; + if (S < 1e-200) { + S = frexp(S, &e); + Sexp += e; + } + if (H < 1e-200) { + H = frexp(H, &e); + Hexp += e; + } + MOZ_LOG(BayesianFilterLogModule, LogLevel::Warning, + ("token probability (%s) is %f", tokens[ta.mTokenIndex].mWord, + ta.mProbability)); + } + if (aDetailListener) { + sArray.AppendElement(log(S) + Sexp * M_LN2); + hArray.AppendElement(log(H) + Hexp * M_LN2); + } + } + + S = log(S) + Sexp * M_LN2; + H = log(H) + Hexp * M_LN2; + + double prob; + if (goodclues > 0) { + int32_t chi_error; + S = chi2P(-2.0 * S, 2.0 * goodclues, &chi_error); + if (!chi_error) H = chi2P(-2.0 * H, 2.0 * goodclues, &chi_error); + // if any error toss the entire calculation + if (!chi_error) + prob = (S - H + 1.0) / 2.0; + else + prob = 0.5; + } else + prob = 0.5; + + if (aDetailListener) { + // Prepare output arrays + nsTArray<uint32_t> tokenPercents(usedTokenCount); + nsTArray<uint32_t> runningPercents(usedTokenCount); + nsTArray<nsString> tokenStrings(usedTokenCount); + + double clueCount = 1.0; + for (uint32_t tokenIndex = 0; tokenIndex < usedTokenCount; tokenIndex++) { + TraitAnalysis& ta = traitAnalyses[last - 1 - tokenIndex]; + int32_t chi_error; + S = chi2P(-2.0 * sArray[tokenIndex], 2.0 * clueCount, &chi_error); + if (!chi_error) + H = chi2P(-2.0 * hArray[tokenIndex], 2.0 * clueCount, &chi_error); + clueCount += 1.0; + double runningProb; + if (!chi_error) + runningProb = (S - H + 1.0) / 2.0; + else + runningProb = 0.5; + runningPercents.AppendElement( + static_cast<uint32_t>(runningProb * 100. + .5)); + tokenPercents.AppendElement( + static_cast<uint32_t>(ta.mProbability * 100. + .5)); + tokenStrings.AppendElement( + NS_ConvertUTF8toUTF16(tokens[ta.mTokenIndex].mWord)); + } + + aDetailListener->OnMessageTraitDetails(messageURI, aProTraits[traitIndex], + tokenStrings, tokenPercents, + runningPercents); + } + + uint32_t proPercent = static_cast<uint32_t>(prob * 100. + .5); + + // directly classify junk to maintain backwards compatibility + if (aProTraits[traitIndex] == kJunkTrait) { + bool isJunk = (prob >= mJunkProbabilityThreshold); + MOZ_LOG(BayesianFilterLogModule, LogLevel::Info, + ("%s is junk probability = (%f) HAM SCORE:%f SPAM SCORE:%f", + PromiseFlatCString(messageURI).get(), prob, H, S)); + + // the algorithm in "A Plan For Spam" assumes that you have a large good + // corpus and a large junk corpus. + // that won't be the case with users who first use the junk mail trait + // so, we do certain things to encourage them to train. + // + // if there are no good tokens, assume the message is junk + // this will "encourage" the user to train + // and if there are no bad tokens, assume the message is not junk + // this will also "encourage" the user to train + // see bug #194238 + + if (listener && !mCorpus.getMessageCount(kGoodTrait)) + isJunk = true; + else if (listener && !mCorpus.getMessageCount(kJunkTrait)) + isJunk = false; + + if (listener) + listener->OnMessageClassified( + messageURI, + isJunk ? nsMsgJunkStatus(nsIJunkMailPlugin::JUNK) + : nsMsgJunkStatus(nsIJunkMailPlugin::GOOD), + proPercent); + } + + if (aTraitListener) { + traits.AppendElement(aProTraits[traitIndex]); + percents.AppendElement(proPercent); + } + } + + if (aTraitListener) + aTraitListener->OnMessageTraitsClassified(messageURI, traits, percents); + + delete[] tokens; + // reuse mAnalysisStore without clearing memory + mNextAnalysisIndex = 1; + // but shrink it back to the default size + if (mAnalysisStore.Length() > kAnalysisStoreCapacity) + mAnalysisStore.RemoveElementsAt( + kAnalysisStoreCapacity, + mAnalysisStore.Length() - kAnalysisStoreCapacity); + mAnalysisStore.Compact(); +} + +void nsBayesianFilter::classifyMessage( + Tokenizer& tokens, const nsACString& messageURI, + nsIJunkMailClassificationListener* aJunkListener) { + AutoTArray<uint32_t, 1> proTraits; + AutoTArray<uint32_t, 1> antiTraits; + proTraits.AppendElement(kJunkTrait); + antiTraits.AppendElement(kGoodTrait); + classifyMessage(tokens, messageURI, proTraits, antiTraits, aJunkListener, + nullptr, nullptr); +} + +NS_IMETHODIMP +nsBayesianFilter::Observe(nsISupports* aSubject, const char* aTopic, + const char16_t* someData) { + if (!strcmp(aTopic, "profile-before-change")) Shutdown(); + return NS_OK; +} + +/* void shutdown (); */ +NS_IMETHODIMP nsBayesianFilter::Shutdown() { + if (mTrainingDataDirty) mCorpus.writeTrainingData(mMaximumTokenCount); + mTrainingDataDirty = false; + + return NS_OK; +} + +/* readonly attribute boolean shouldDownloadAllHeaders; */ +NS_IMETHODIMP nsBayesianFilter::GetShouldDownloadAllHeaders( + bool* aShouldDownloadAllHeaders) { + // bayesian filters work on the whole msg body currently. + *aShouldDownloadAllHeaders = false; + return NS_OK; +} + +/* void classifyMessage (in string aMsgURL, in nsIJunkMailClassificationListener + * aListener); */ +NS_IMETHODIMP nsBayesianFilter::ClassifyMessage( + const nsACString& aMessageURL, nsIMsgWindow* aMsgWindow, + nsIJunkMailClassificationListener* aListener) { + AutoTArray<nsCString, 1> urls = {PromiseFlatCString(aMessageURL)}; + MessageClassifier* analyzer = + new MessageClassifier(this, aListener, aMsgWindow, urls); + NS_ENSURE_TRUE(analyzer, NS_ERROR_OUT_OF_MEMORY); + TokenStreamListener* tokenListener = new TokenStreamListener(analyzer); + NS_ENSURE_TRUE(tokenListener, NS_ERROR_OUT_OF_MEMORY); + analyzer->setTokenListener(tokenListener); + return tokenizeMessage(aMessageURL, aMsgWindow, analyzer); +} + +/* void classifyMessages(in Array<ACString> aMsgURIs, + * in nsIMsgWindow aMsgWindow, + * in nsIJunkMailClassificationListener aListener); */ +NS_IMETHODIMP nsBayesianFilter::ClassifyMessages( + const nsTArray<nsCString>& aMsgURLs, nsIMsgWindow* aMsgWindow, + nsIJunkMailClassificationListener* aListener) { + TokenAnalyzer* analyzer = + new MessageClassifier(this, aListener, aMsgWindow, aMsgURLs); + NS_ENSURE_TRUE(analyzer, NS_ERROR_OUT_OF_MEMORY); + TokenStreamListener* tokenListener = new TokenStreamListener(analyzer); + NS_ENSURE_TRUE(tokenListener, NS_ERROR_OUT_OF_MEMORY); + analyzer->setTokenListener(tokenListener); + return tokenizeMessage(aMsgURLs[0], aMsgWindow, analyzer); +} + +nsresult nsBayesianFilter::setAnalysis(Token& token, uint32_t aTraitIndex, + double aDistance, double aProbability) { + uint32_t nextLink = token.mAnalysisLink; + uint32_t lastLink = 0; + uint32_t linkCount = 0, maxLinks = 100; + + // try to find an existing element. Limit the search to maxLinks + // as a precaution + for (linkCount = 0; nextLink && linkCount < maxLinks; linkCount++) { + AnalysisPerToken& rAnalysis = mAnalysisStore[nextLink]; + if (rAnalysis.mTraitIndex == aTraitIndex) { + rAnalysis.mDistance = aDistance; + rAnalysis.mProbability = aProbability; + return NS_OK; + } + lastLink = nextLink; + nextLink = rAnalysis.mNextLink; + } + if (linkCount >= maxLinks) return NS_ERROR_FAILURE; + + // trait does not exist, so add it + + AnalysisPerToken analysis(aTraitIndex, aDistance, aProbability); + if (mAnalysisStore.Length() == mNextAnalysisIndex) + mAnalysisStore.InsertElementAt(mNextAnalysisIndex, analysis); + else if (mAnalysisStore.Length() > mNextAnalysisIndex) + mAnalysisStore.ReplaceElementsAt(mNextAnalysisIndex, 1, analysis); + else // we can only insert at the end of the array + return NS_ERROR_FAILURE; + + if (lastLink) + // the token had at least one link, so update the last link to point to + // the new item + mAnalysisStore[lastLink].mNextLink = mNextAnalysisIndex; + else + // need to update the token's first link + token.mAnalysisLink = mNextAnalysisIndex; + mNextAnalysisIndex++; + return NS_OK; +} + +uint32_t nsBayesianFilter::getAnalysisIndex(Token& token, + uint32_t aTraitIndex) { + uint32_t nextLink; + uint32_t linkCount = 0, maxLinks = 100; + for (nextLink = token.mAnalysisLink; nextLink && linkCount < maxLinks; + linkCount++) { + AnalysisPerToken& rAnalysis = mAnalysisStore[nextLink]; + if (rAnalysis.mTraitIndex == aTraitIndex) return nextLink; + nextLink = rAnalysis.mNextLink; + } + NS_ASSERTION(linkCount < maxLinks, "corrupt analysis store"); + + // Trait not found, indicate by zero + return 0; +} + +NS_IMETHODIMP nsBayesianFilter::ClassifyTraitsInMessage( + const nsACString& aMsgURI, const nsTArray<uint32_t>& aProTraits, + const nsTArray<uint32_t>& aAntiTraits, + nsIMsgTraitClassificationListener* aTraitListener, nsIMsgWindow* aMsgWindow, + nsIJunkMailClassificationListener* aJunkListener) { + AutoTArray<nsCString, 1> uris = {PromiseFlatCString(aMsgURI)}; + return ClassifyTraitsInMessages(uris, aProTraits, aAntiTraits, aTraitListener, + aMsgWindow, aJunkListener); +} + +NS_IMETHODIMP nsBayesianFilter::ClassifyTraitsInMessages( + const nsTArray<nsCString>& aMsgURIs, const nsTArray<uint32_t>& aProTraits, + const nsTArray<uint32_t>& aAntiTraits, + nsIMsgTraitClassificationListener* aTraitListener, nsIMsgWindow* aMsgWindow, + nsIJunkMailClassificationListener* aJunkListener) { + MOZ_ASSERT(aProTraits.Length() == aAntiTraits.Length()); + MessageClassifier* analyzer = + new MessageClassifier(this, aJunkListener, aTraitListener, nullptr, + aProTraits, aAntiTraits, aMsgWindow, aMsgURIs); + + TokenStreamListener* tokenListener = new TokenStreamListener(analyzer); + + analyzer->setTokenListener(tokenListener); + return tokenizeMessage(aMsgURIs[0], aMsgWindow, analyzer); +} + +class MessageObserver : public TokenAnalyzer { + public: + MessageObserver(nsBayesianFilter* filter, + const nsTArray<uint32_t>& aOldClassifications, + const nsTArray<uint32_t>& aNewClassifications, + nsIJunkMailClassificationListener* aJunkListener, + nsIMsgTraitClassificationListener* aTraitListener) + : mFilter(filter), + mJunkMailPlugin(filter), + mJunkListener(aJunkListener), + mTraitListener(aTraitListener), + mOldClassifications(aOldClassifications.Clone()), + mNewClassifications(aNewClassifications.Clone()) {} + + virtual void analyzeTokens(Tokenizer& tokenizer) { + mFilter->observeMessage(tokenizer, mTokenSource, mOldClassifications, + mNewClassifications, mJunkListener, mTraitListener); + // release reference to listener, which will allow us to go away as well. + mTokenListener = nullptr; + } + + private: + nsBayesianFilter* mFilter; + nsCOMPtr<nsIJunkMailPlugin> mJunkMailPlugin; + nsCOMPtr<nsIJunkMailClassificationListener> mJunkListener; + nsCOMPtr<nsIMsgTraitClassificationListener> mTraitListener; + nsTArray<uint32_t> mOldClassifications; + nsTArray<uint32_t> mNewClassifications; +}; + +NS_IMETHODIMP nsBayesianFilter::SetMsgTraitClassification( + const nsACString& aMsgURI, const nsTArray<uint32_t>& aOldTraits, + const nsTArray<uint32_t>& aNewTraits, + nsIMsgTraitClassificationListener* aTraitListener, nsIMsgWindow* aMsgWindow, + nsIJunkMailClassificationListener* aJunkListener) { + MessageObserver* analyzer = new MessageObserver( + this, aOldTraits, aNewTraits, aJunkListener, aTraitListener); + NS_ENSURE_TRUE(analyzer, NS_ERROR_OUT_OF_MEMORY); + + TokenStreamListener* tokenListener = new TokenStreamListener(analyzer); + NS_ENSURE_TRUE(tokenListener, NS_ERROR_OUT_OF_MEMORY); + + analyzer->setTokenListener(tokenListener); + return tokenizeMessage(aMsgURI, aMsgWindow, analyzer); +} + +// set new message classifications for a message +void nsBayesianFilter::observeMessage( + Tokenizer& tokenizer, const nsACString& messageURL, + nsTArray<uint32_t>& oldClassifications, + nsTArray<uint32_t>& newClassifications, + nsIJunkMailClassificationListener* aJunkListener, + nsIMsgTraitClassificationListener* aTraitListener) { + bool trainingDataWasDirty = mTrainingDataDirty; + + // Uhoh...if the user is re-training then the message may already be + // classified and we are classifying it again with the same classification. + // the old code would have removed the tokens for this message then added them + // back. But this really hurts the message occurrence count for tokens if you + // just removed training.dat and are re-training. See Bug #237095 for more + // details. What can we do here? Well we can skip the token removal step if + // the classifications are the same and assume the user is just re-training. + // But this then allows users to re-classify the same message on the same + // training set over and over again leading to data skew. But that's all I can + // think to do right now to address this..... + uint32_t oldLength = oldClassifications.Length(); + for (uint32_t index = 0; index < oldLength; index++) { + uint32_t trait = oldClassifications.ElementAt(index); + // skip removing if trait is also in the new set + if (newClassifications.Contains(trait)) continue; + // remove the tokens from the token set it is currently in + uint32_t messageCount; + messageCount = mCorpus.getMessageCount(trait); + if (messageCount > 0) { + mCorpus.setMessageCount(trait, messageCount - 1); + mCorpus.forgetTokens(tokenizer, trait, 1); + mTrainingDataDirty = true; + } + } + + nsMsgJunkStatus newClassification = nsIJunkMailPlugin::UNCLASSIFIED; + uint32_t junkPercent = + 0; // 0 here is no possibility of meeting the classification + uint32_t newLength = newClassifications.Length(); + for (uint32_t index = 0; index < newLength; index++) { + uint32_t trait = newClassifications.ElementAt(index); + mCorpus.setMessageCount(trait, mCorpus.getMessageCount(trait) + 1); + mCorpus.rememberTokens(tokenizer, trait, 1); + mTrainingDataDirty = true; + + if (aJunkListener) { + if (trait == kJunkTrait) { + junkPercent = nsIJunkMailPlugin::IS_SPAM_SCORE; + newClassification = nsIJunkMailPlugin::JUNK; + } else if (trait == kGoodTrait) { + junkPercent = nsIJunkMailPlugin::IS_HAM_SCORE; + newClassification = nsIJunkMailPlugin::GOOD; + } + } + } + + if (aJunkListener) + aJunkListener->OnMessageClassified(messageURL, newClassification, + junkPercent); + + if (aTraitListener) { + // construct the outgoing listener arrays + AutoTArray<uint32_t, kTraitAutoCapacity> traits; + AutoTArray<uint32_t, kTraitAutoCapacity> percents; + uint32_t newLength = newClassifications.Length(); + if (newLength > kTraitAutoCapacity) { + traits.SetCapacity(newLength); + percents.SetCapacity(newLength); + } + traits.AppendElements(newClassifications); + for (uint32_t index = 0; index < newLength; index++) + percents.AppendElement(100); // This is 100 percent, or certainty + aTraitListener->OnMessageTraitsClassified(messageURL, traits, percents); + } + + if (mTrainingDataDirty && !trainingDataWasDirty) { + // if training data became dirty just now, schedule flush + // mMinFlushInterval msec from now + MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, + ("starting training data flush timer %i msec", mMinFlushInterval)); + + nsresult rv = NS_NewTimerWithFuncCallback( + getter_AddRefs(mTimer), nsBayesianFilter::TimerCallback, (void*)this, + mMinFlushInterval, nsITimer::TYPE_ONE_SHOT, + "nsBayesianFilter::TimerCallback", nullptr); + if (NS_FAILED(rv)) { + NS_WARNING("Could not start nsBayesianFilter timer"); + } + } +} + +NS_IMETHODIMP nsBayesianFilter::GetUserHasClassified(bool* aResult) { + *aResult = ((mCorpus.getMessageCount(kGoodTrait) + + mCorpus.getMessageCount(kJunkTrait)) && + mCorpus.countTokens()); + return NS_OK; +} + +// Set message classification (only allows junk and good) +NS_IMETHODIMP nsBayesianFilter::SetMessageClassification( + const nsACString& aMsgURL, nsMsgJunkStatus aOldClassification, + nsMsgJunkStatus aNewClassification, nsIMsgWindow* aMsgWindow, + nsIJunkMailClassificationListener* aListener) { + AutoTArray<uint32_t, 1> oldClassifications; + AutoTArray<uint32_t, 1> newClassifications; + + // convert between classifications and trait + if (aOldClassification == nsIJunkMailPlugin::JUNK) + oldClassifications.AppendElement(kJunkTrait); + else if (aOldClassification == nsIJunkMailPlugin::GOOD) + oldClassifications.AppendElement(kGoodTrait); + if (aNewClassification == nsIJunkMailPlugin::JUNK) + newClassifications.AppendElement(kJunkTrait); + else if (aNewClassification == nsIJunkMailPlugin::GOOD) + newClassifications.AppendElement(kGoodTrait); + + MessageObserver* analyzer = new MessageObserver( + this, oldClassifications, newClassifications, aListener, nullptr); + NS_ENSURE_TRUE(analyzer, NS_ERROR_OUT_OF_MEMORY); + + TokenStreamListener* tokenListener = new TokenStreamListener(analyzer); + NS_ENSURE_TRUE(tokenListener, NS_ERROR_OUT_OF_MEMORY); + + analyzer->setTokenListener(tokenListener); + return tokenizeMessage(aMsgURL, aMsgWindow, analyzer); +} + +NS_IMETHODIMP nsBayesianFilter::ResetTrainingData() { + return mCorpus.resetTrainingData(); +} + +NS_IMETHODIMP nsBayesianFilter::DetailMessage( + const nsACString& aMsgURI, uint32_t aProTrait, uint32_t aAntiTrait, + nsIMsgTraitDetailListener* aDetailListener, nsIMsgWindow* aMsgWindow) { + AutoTArray<uint32_t, 1> proTraits = {aProTrait}; + AutoTArray<uint32_t, 1> antiTraits = {aAntiTrait}; + AutoTArray<nsCString, 1> uris = {PromiseFlatCString(aMsgURI)}; + + MessageClassifier* analyzer = + new MessageClassifier(this, nullptr, nullptr, aDetailListener, proTraits, + antiTraits, aMsgWindow, uris); + NS_ENSURE_TRUE(analyzer, NS_ERROR_OUT_OF_MEMORY); + + TokenStreamListener* tokenListener = new TokenStreamListener(analyzer); + NS_ENSURE_TRUE(tokenListener, NS_ERROR_OUT_OF_MEMORY); + + analyzer->setTokenListener(tokenListener); + return tokenizeMessage(aMsgURI, aMsgWindow, analyzer); +} + +// nsIMsgCorpus implementation + +NS_IMETHODIMP nsBayesianFilter::CorpusCounts(uint32_t aTrait, + uint32_t* aMessageCount, + uint32_t* aTokenCount) { + NS_ENSURE_ARG_POINTER(aTokenCount); + *aTokenCount = mCorpus.countTokens(); + if (aTrait && aMessageCount) *aMessageCount = mCorpus.getMessageCount(aTrait); + return NS_OK; +} + +NS_IMETHODIMP nsBayesianFilter::ClearTrait(uint32_t aTrait) { + return mCorpus.ClearTrait(aTrait); +} + +NS_IMETHODIMP +nsBayesianFilter::UpdateData(nsIFile* aFile, bool aIsAdd, + const nsTArray<uint32_t>& aFromTraits, + const nsTArray<uint32_t>& aToTraits) { + MOZ_ASSERT(aFromTraits.Length() == aToTraits.Length()); + return mCorpus.UpdateData(aFile, aIsAdd, aFromTraits, aToTraits); +} + +NS_IMETHODIMP +nsBayesianFilter::GetTokenCount(const nsACString& aWord, uint32_t aTrait, + uint32_t* aCount) { + NS_ENSURE_ARG_POINTER(aCount); + CorpusToken* t = mCorpus.get(PromiseFlatCString(aWord).get()); + uint32_t count = mCorpus.getTraitCount(t, aTrait); + *aCount = count; + return NS_OK; +} + +/* Corpus Store */ + +/* + Format of the training file for version 1: + [0xFEEDFACE] + [number good messages][number bad messages] + [number good tokens] + [count][length of word]word + ... + [number bad tokens] + [count][length of word]word + ... + + Format of the trait file for version 1: + [0xFCA93601] (the 01 is the version) + for each trait to write + [id of trait to write] (0 means end of list) + [number of messages per trait] + for each token with non-zero count + [count] + [length of word]word +*/ + +CorpusStore::CorpusStore() + : TokenHash(sizeof(CorpusToken)), + mNextTraitIndex(1) // skip 0 since index=0 will mean end of linked list +{ + getTrainingFile(getter_AddRefs(mTrainingFile)); + mTraitStore.SetCapacity(kTraitStoreCapacity); + TraitPerToken traitPT(0, 0); + mTraitStore.AppendElement(traitPT); // dummy 0th element +} + +CorpusStore::~CorpusStore() {} + +inline int writeUInt32(FILE* stream, uint32_t value) { + value = PR_htonl(value); + return fwrite(&value, sizeof(uint32_t), 1, stream); +} + +inline int readUInt32(FILE* stream, uint32_t* value) { + int n = fread(value, sizeof(uint32_t), 1, stream); + if (n == 1) { + *value = PR_ntohl(*value); + } + return n; +} + +void CorpusStore::forgetTokens(Tokenizer& aTokenizer, uint32_t aTraitId, + uint32_t aCount) { + // if we are forgetting the tokens for a message, should only + // subtract 1 from the occurrence count for that token in the training set + // because we assume we only bumped the training set count once per messages + // containing the token. + TokenEnumeration tokens = aTokenizer.getTokens(); + while (tokens.hasMoreTokens()) { + CorpusToken* token = static_cast<CorpusToken*>(tokens.nextToken()); + remove(token->mWord, aTraitId, aCount); + } +} + +void CorpusStore::rememberTokens(Tokenizer& aTokenizer, uint32_t aTraitId, + uint32_t aCount) { + TokenEnumeration tokens = aTokenizer.getTokens(); + while (tokens.hasMoreTokens()) { + CorpusToken* token = static_cast<CorpusToken*>(tokens.nextToken()); + if (!token) { + NS_ERROR("null token"); + continue; + } + add(token->mWord, aTraitId, aCount); + } +} + +bool CorpusStore::writeTokens(FILE* stream, bool shrink, uint32_t aTraitId) { + uint32_t tokenCount = countTokens(); + uint32_t newTokenCount = 0; + + // calculate the tokens for this trait to write + + TokenEnumeration tokens = getTokens(); + for (uint32_t i = 0; i < tokenCount; ++i) { + CorpusToken* token = static_cast<CorpusToken*>(tokens.nextToken()); + uint32_t count = getTraitCount(token, aTraitId); + // Shrinking the token database is accomplished by dividing all token counts + // by 2. If shrinking, we'll ignore counts < 2, otherwise only ignore counts + // of < 1 + if ((shrink && count > 1) || (!shrink && count)) newTokenCount++; + } + + if (writeUInt32(stream, newTokenCount) != 1) return false; + + if (newTokenCount > 0) { + TokenEnumeration tokens = getTokens(); + for (uint32_t i = 0; i < tokenCount; ++i) { + CorpusToken* token = static_cast<CorpusToken*>(tokens.nextToken()); + uint32_t wordCount = getTraitCount(token, aTraitId); + if (shrink) wordCount /= 2; + if (!wordCount) continue; // Don't output zero count words + if (writeUInt32(stream, wordCount) != 1) return false; + uint32_t tokenLength = strlen(token->mWord); + if (writeUInt32(stream, tokenLength) != 1) return false; + if (fwrite(token->mWord, tokenLength, 1, stream) != 1) return false; + } + } + return true; +} + +bool CorpusStore::readTokens(FILE* stream, int64_t fileSize, uint32_t aTraitId, + bool aIsAdd) { + uint32_t tokenCount; + if (readUInt32(stream, &tokenCount) != 1) return false; + + int64_t fpos = ftell(stream); + if (fpos < 0) return false; + + uint32_t bufferSize = 4096; + char* buffer = new char[bufferSize]; + if (!buffer) return false; + + for (uint32_t i = 0; i < tokenCount; ++i) { + uint32_t count; + if (readUInt32(stream, &count) != 1) break; + uint32_t size; + if (readUInt32(stream, &size) != 1) break; + fpos += 8; + if (fpos + size > fileSize) { + delete[] buffer; + return false; + } + if (size >= bufferSize) { + delete[] buffer; + while (size >= bufferSize) { + bufferSize *= 2; + if (bufferSize == 0) return false; + } + buffer = new char[bufferSize]; + if (!buffer) return false; + } + if (fread(buffer, size, 1, stream) != 1) break; + fpos += size; + buffer[size] = '\0'; + if (aIsAdd) + add(buffer, aTraitId, count); + else + remove(buffer, aTraitId, count); + } + + delete[] buffer; + + return true; +} + +nsresult CorpusStore::getTrainingFile(nsIFile** aTrainingFile) { + // should we cache the profile manager's directory? + nsCOMPtr<nsIFile> profileDir; + + nsresult rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, + getter_AddRefs(profileDir)); + NS_ENSURE_SUCCESS(rv, rv); + rv = profileDir->Append(u"training.dat"_ns); + NS_ENSURE_SUCCESS(rv, rv); + + return profileDir->QueryInterface(NS_GET_IID(nsIFile), (void**)aTrainingFile); +} + +nsresult CorpusStore::getTraitFile(nsIFile** aTraitFile) { + // should we cache the profile manager's directory? + nsCOMPtr<nsIFile> profileDir; + + nsresult rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, + getter_AddRefs(profileDir)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = profileDir->Append(u"traits.dat"_ns); + NS_ENSURE_SUCCESS(rv, rv); + + return profileDir->QueryInterface(NS_GET_IID(nsIFile), (void**)aTraitFile); +} + +static const char kMagicCookie[] = {'\xFE', '\xED', '\xFA', '\xCE'}; + +// random string used to identify trait file and version (last byte is version) +static const char kTraitCookie[] = {'\xFC', '\xA9', '\x36', '\x01'}; + +void CorpusStore::writeTrainingData(uint32_t aMaximumTokenCount) { + MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, + ("writeTrainingData() entered")); + if (!mTrainingFile) return; + + /* + * For backwards compatibility, write the good and junk tokens to + * training.dat; additional traits are added to a different file + */ + + // open the file, and write out training data + FILE* stream; + nsresult rv = mTrainingFile->OpenANSIFileDesc("wb", &stream); + if (NS_FAILED(rv)) return; + + // If the number of tokens exceeds our limit, set the shrink flag + bool shrink = false; + if ((aMaximumTokenCount > 0) && // if 0, do not limit tokens + (countTokens() > aMaximumTokenCount)) { + shrink = true; + MOZ_LOG(BayesianFilterLogModule, LogLevel::Warning, + ("shrinking token data file")); + } + + // We implement shrink by dividing counts by two + uint32_t shrinkFactor = shrink ? 2 : 1; + + if (!((fwrite(kMagicCookie, sizeof(kMagicCookie), 1, stream) == 1) && + (writeUInt32(stream, getMessageCount(kGoodTrait) / shrinkFactor)) && + (writeUInt32(stream, getMessageCount(kJunkTrait) / shrinkFactor)) && + writeTokens(stream, shrink, kGoodTrait) && + writeTokens(stream, shrink, kJunkTrait))) { + NS_WARNING("failed to write training data."); + fclose(stream); + // delete the training data file, since it is potentially corrupt. + mTrainingFile->Remove(false); + } else { + fclose(stream); + } + + /* + * Write the remaining data to a second file traits.dat + */ + + if (!mTraitFile) { + getTraitFile(getter_AddRefs(mTraitFile)); + if (!mTraitFile) return; + } + + // open the file, and write out training data + rv = mTraitFile->OpenANSIFileDesc("wb", &stream); + if (NS_FAILED(rv)) return; + + uint32_t numberOfTraits = mMessageCounts.Length(); + bool error; + while (1) // break on error or done + { + if ((error = (fwrite(kTraitCookie, sizeof(kTraitCookie), 1, stream) != 1))) + break; + + for (uint32_t index = 0; index < numberOfTraits; index++) { + uint32_t trait = mMessageCountsId[index]; + if (trait == 1 || trait == 2) + continue; // junk traits are stored in training.dat + if ((error = (writeUInt32(stream, trait) != 1))) break; + if ((error = (writeUInt32(stream, mMessageCounts[index] / shrinkFactor) != + 1))) + break; + if ((error = !writeTokens(stream, shrink, trait))) break; + } + break; + } + // we add a 0 at the end to represent end of trait list + error = writeUInt32(stream, 0) != 1; + + fclose(stream); + if (error) { + NS_WARNING("failed to write trait data."); + // delete the trait data file, since it is probably corrupt. + mTraitFile->Remove(false); + } + + if (shrink) { + // We'll clear the tokens, and read them back in from the file. + // Yes this is slower than in place, but this is a rare event. + + if (countTokens()) { + clearTokens(); + for (uint32_t index = 0; index < numberOfTraits; index++) + mMessageCounts[index] = 0; + } + + readTrainingData(); + } +} + +void CorpusStore::readTrainingData() { + /* + * To maintain backwards compatibility, good and junk traits + * are stored in a file "training.dat" + */ + if (!mTrainingFile) return; + + bool exists; + nsresult rv = mTrainingFile->Exists(&exists); + if (NS_FAILED(rv) || !exists) return; + + FILE* stream; + rv = mTrainingFile->OpenANSIFileDesc("rb", &stream); + if (NS_FAILED(rv)) return; + + int64_t fileSize; + rv = mTrainingFile->GetFileSize(&fileSize); + if (NS_FAILED(rv)) return; + + // FIXME: should make sure that the tokenizers are empty. + char cookie[4]; + uint32_t goodMessageCount = 0, junkMessageCount = 0; + if (!((fread(cookie, sizeof(cookie), 1, stream) == 1) && + (memcmp(cookie, kMagicCookie, sizeof(cookie)) == 0) && + (readUInt32(stream, &goodMessageCount) == 1) && + (readUInt32(stream, &junkMessageCount) == 1) && + readTokens(stream, fileSize, kGoodTrait, true) && + readTokens(stream, fileSize, kJunkTrait, true))) { + NS_WARNING("failed to read training data."); + MOZ_LOG(BayesianFilterLogModule, LogLevel::Error, + ("failed to read training data.")); + } + setMessageCount(kGoodTrait, goodMessageCount); + setMessageCount(kJunkTrait, junkMessageCount); + + fclose(stream); + + /* + * Additional traits are stored in traits.dat + */ + + if (!mTraitFile) { + getTraitFile(getter_AddRefs(mTraitFile)); + if (!mTraitFile) return; + } + + rv = mTraitFile->Exists(&exists); + if (NS_FAILED(rv) || !exists) return; + + nsTArray<uint32_t> empty; + rv = UpdateData(mTraitFile, true, empty, empty); + + if (NS_FAILED(rv)) { + NS_WARNING("failed to read training data."); + MOZ_LOG(BayesianFilterLogModule, LogLevel::Error, + ("failed to read training data.")); + } + return; +} + +nsresult CorpusStore::resetTrainingData() { + // clear out our in memory training tokens... + if (countTokens()) clearTokens(); + + uint32_t length = mMessageCounts.Length(); + for (uint32_t index = 0; index < length; index++) mMessageCounts[index] = 0; + + if (mTrainingFile) mTrainingFile->Remove(false); + if (mTraitFile) mTraitFile->Remove(false); + return NS_OK; +} + +inline CorpusToken* CorpusStore::get(const char* word) { + return static_cast<CorpusToken*>(TokenHash::get(word)); +} + +nsresult CorpusStore::updateTrait(CorpusToken* token, uint32_t aTraitId, + int32_t aCountChange) { + NS_ENSURE_ARG_POINTER(token); + uint32_t nextLink = token->mTraitLink; + uint32_t lastLink = 0; + + uint32_t linkCount, maxLinks = 100; // sanity check + for (linkCount = 0; nextLink && linkCount < maxLinks; linkCount++) { + TraitPerToken& traitPT = mTraitStore[nextLink]; + if (traitPT.mId == aTraitId) { + // be careful with signed versus unsigned issues here + if (static_cast<int32_t>(traitPT.mCount) + aCountChange > 0) + traitPT.mCount += aCountChange; + else + traitPT.mCount = 0; + // we could delete zero count traits here, but let's not. It's rare + // anyway. + return NS_OK; + } + lastLink = nextLink; + nextLink = traitPT.mNextLink; + } + if (linkCount >= maxLinks) return NS_ERROR_FAILURE; + + // trait does not exist, so add it + + if (aCountChange > 0) // don't set a negative count + { + TraitPerToken traitPT(aTraitId, aCountChange); + if (mTraitStore.Length() == mNextTraitIndex) + mTraitStore.InsertElementAt(mNextTraitIndex, traitPT); + else if (mTraitStore.Length() > mNextTraitIndex) + mTraitStore.ReplaceElementsAt(mNextTraitIndex, 1, traitPT); + else + return NS_ERROR_FAILURE; + if (lastLink) + // the token had a parent, so update it + mTraitStore[lastLink].mNextLink = mNextTraitIndex; + else + // need to update the token's root link + token->mTraitLink = mNextTraitIndex; + mNextTraitIndex++; + } + return NS_OK; +} + +uint32_t CorpusStore::getTraitCount(CorpusToken* token, uint32_t aTraitId) { + uint32_t nextLink; + if (!token || !(nextLink = token->mTraitLink)) return 0; + + uint32_t linkCount, maxLinks = 100; // sanity check + for (linkCount = 0; nextLink && linkCount < maxLinks; linkCount++) { + TraitPerToken& traitPT = mTraitStore[nextLink]; + if (traitPT.mId == aTraitId) return traitPT.mCount; + nextLink = traitPT.mNextLink; + } + NS_ASSERTION(linkCount < maxLinks, "Corrupt trait count store"); + + // trait not found (or error), so count is zero + return 0; +} + +CorpusToken* CorpusStore::add(const char* word, uint32_t aTraitId, + uint32_t aCount) { + CorpusToken* token = static_cast<CorpusToken*>(TokenHash::add(word)); + if (token) { + MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, + ("adding word to corpus store: %s (Trait=%d) (deltaCount=%d)", word, + aTraitId, aCount)); + updateTrait(token, aTraitId, aCount); + } + return token; +} + +void CorpusStore::remove(const char* word, uint32_t aTraitId, uint32_t aCount) { + MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, + ("remove word: %s (TraitId=%d) (Count=%d)", word, aTraitId, aCount)); + CorpusToken* token = get(word); + if (token) updateTrait(token, aTraitId, -static_cast<int32_t>(aCount)); +} + +uint32_t CorpusStore::getMessageCount(uint32_t aTraitId) { + size_t index = mMessageCountsId.IndexOf(aTraitId); + if (index == mMessageCountsId.NoIndex) return 0; + return mMessageCounts.ElementAt(index); +} + +void CorpusStore::setMessageCount(uint32_t aTraitId, uint32_t aCount) { + size_t index = mMessageCountsId.IndexOf(aTraitId); + if (index == mMessageCountsId.NoIndex) { + mMessageCounts.AppendElement(aCount); + mMessageCountsId.AppendElement(aTraitId); + } else { + mMessageCounts[index] = aCount; + } +} + +nsresult CorpusStore::UpdateData(nsIFile* aFile, bool aIsAdd, + const nsTArray<uint32_t>& aFromTraits, + const nsTArray<uint32_t>& aToTraits) { + NS_ENSURE_ARG_POINTER(aFile); + MOZ_ASSERT(aFromTraits.Length() == aToTraits.Length()); + + int64_t fileSize; + nsresult rv = aFile->GetFileSize(&fileSize); + NS_ENSURE_SUCCESS(rv, rv); + + FILE* stream; + rv = aFile->OpenANSIFileDesc("rb", &stream); + NS_ENSURE_SUCCESS(rv, rv); + + bool error; + do // break on error or done + { + char cookie[4]; + if ((error = (fread(cookie, sizeof(cookie), 1, stream) != 1))) break; + + if ((error = memcmp(cookie, kTraitCookie, sizeof(cookie)))) break; + + uint32_t fileTrait; + while (!(error = (readUInt32(stream, &fileTrait) != 1)) && fileTrait) { + uint32_t count; + if ((error = (readUInt32(stream, &count) != 1))) break; + + uint32_t localTrait = fileTrait; + // remap the trait + for (uint32_t i = 0; i < aFromTraits.Length(); i++) { + if (aFromTraits[i] == fileTrait) localTrait = aToTraits[i]; + } + + uint32_t messageCount = getMessageCount(localTrait); + if (aIsAdd) + messageCount += count; + else if (count > messageCount) + messageCount = 0; + else + messageCount -= count; + setMessageCount(localTrait, messageCount); + + if ((error = !readTokens(stream, fileSize, localTrait, aIsAdd))) break; + } + break; + } while (0); + + fclose(stream); + + if (error) return NS_ERROR_FAILURE; + return NS_OK; +} + +nsresult CorpusStore::ClearTrait(uint32_t aTrait) { + // clear message counts + setMessageCount(aTrait, 0); + + TokenEnumeration tokens = getTokens(); + while (tokens.hasMoreTokens()) { + CorpusToken* token = static_cast<CorpusToken*>(tokens.nextToken()); + int32_t wordCount = static_cast<int32_t>(getTraitCount(token, aTrait)); + updateTrait(token, aTrait, -wordCount); + } + return NS_OK; +} diff --git a/comm/mailnews/extensions/bayesian-spam-filter/nsBayesianFilter.h b/comm/mailnews/extensions/bayesian-spam-filter/nsBayesianFilter.h new file mode 100644 index 0000000000..70d0a1a02b --- /dev/null +++ b/comm/mailnews/extensions/bayesian-spam-filter/nsBayesianFilter.h @@ -0,0 +1,397 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef nsBayesianFilter_h__ +#define nsBayesianFilter_h__ + +#include <stdio.h> +#include "nsCOMPtr.h" +#include "nsIMsgFilterPlugin.h" +#include "PLDHashTable.h" +#include "nsITimer.h" +#include "nsTArray.h" +#include "nsString.h" +#include "nsWeakReference.h" +#include "nsIObserver.h" +#include "nsHashPropertyBag.h" +#include "mozilla/intl/WordBreaker.h" + +#include "mozilla/ArenaAllocator.h" + +#define DEFAULT_MIN_INTERVAL_BETWEEN_WRITES 15 * 60 * 1000 + +struct Token; +class TokenEnumeration; +class TokenAnalyzer; +class nsIMsgWindow; +class nsIUTF8StringEnumerator; +struct BaseToken; +struct CorpusToken; + +/** + * Helper class to enumerate Token objects in a PLDHashTable + * safely and without copying (see bugzilla #174859). The + * enumeration is safe to use until an Add() + * or Remove() is performed on the table. + */ +class TokenEnumeration { + public: + explicit TokenEnumeration(PLDHashTable* table); + bool hasMoreTokens(); + BaseToken* nextToken(); + + private: + PLDHashTable::Iterator mIterator; +}; + +// A trait is some aspect of a message, like being junk or tagged as +// Personal, that the statistical classifier should track. The Trait +// structure is a per-token representation of information pertaining to +// a message trait. +// +// Traits per token are maintained as a linked list. +// +struct TraitPerToken { + uint32_t mId; // identifying number for a trait + uint32_t mCount; // count of messages with this token and trait + uint32_t mNextLink; // index in mTraitStore for the next trait, or 0 + // for none + TraitPerToken(uint32_t aId, uint32_t aCount); // inititializer +}; + +// An Analysis is the statistical results for a particular message, a +// particular token, and for a particular pair of trait/antitrait, that +// is then used in subsequent analysis to score the message. +// +// Analyses per token are maintained as a linked list. +// +struct AnalysisPerToken { + uint32_t mTraitIndex; // index representing a protrait/antitrait pair. + // So if we are analyzing 3 different traits, then + // the first trait is 0, the second 1, etc. + double mDistance; // absolute value of mProbability - 0.5 + double mProbability; // relative indicator of match of trait to token + uint32_t mNextLink; // index in mAnalysisStore for the Analysis object + // for the next trait index, or 0 for none. + // initializer + AnalysisPerToken(uint32_t aTraitIndex, double aDistance, double aProbability); +}; + +class TokenHash { + public: + virtual ~TokenHash(); + /** + * Clears out the previous message tokens. + */ + nsresult clearTokens(); + uint32_t countTokens(); + TokenEnumeration getTokens(); + BaseToken* add(const char* word); + + protected: + explicit TokenHash(uint32_t entrySize); + mozilla::ArenaAllocator<16384, 2> mWordPool; + uint32_t mEntrySize; + PLDHashTable mTokenTable; + char* copyWord(const char* word, uint32_t len); + BaseToken* get(const char* word); +}; + +class Tokenizer : public TokenHash { + public: + Tokenizer(); + ~Tokenizer(); + + Token* get(const char* word); + + // The training set keeps an occurrence count on each word. This count + // is supposed to count the # of messages it occurs in. + // When add/remove is called while tokenizing a message and NOT the training + // set, + // + Token* add(const char* word, uint32_t count = 1); + + Token* copyTokens(); + + void tokenize(const char* text); + + /** + * Creates specific tokens based on the mime headers for the message being + * tokenized + */ + void tokenizeHeaders(nsTArray<nsCString>& aHeaderNames, + nsTArray<nsCString>& aHeaderValues); + + void tokenizeAttachments(nsTArray<RefPtr<nsIPropertyBag2>>& attachments); + + nsCString mBodyDelimiters; // delimiters for body tokenization + nsCString mHeaderDelimiters; // delimiters for header tokenization + + // arrays of extra headers to tokenize / to not tokenize + nsTArray<nsCString> mEnabledHeaders; + nsTArray<nsCString> mDisabledHeaders; + // Delimiters used in tokenizing a particular header. + // Parallel array to mEnabledHeaders + nsTArray<nsCString> mEnabledHeadersDelimiters; + bool mCustomHeaderTokenization; // Are there any preference-set tokenization + // customizations? + uint32_t mMaxLengthForToken; // maximum length of a token + // should we convert iframe to div during tokenization? + bool mIframeToDiv; + + private: + void tokenize_ascii_word(char* word); + void tokenize_japanese_word(char* chunk); + inline void addTokenForHeader(const char* aTokenPrefix, nsACString& aValue, + bool aTokenizeValue = false, + const char* aDelimiters = nullptr); + nsresult stripHTML(const nsAString& inString, nsAString& outString); + // helper function to escape \n, \t, etc from a CString + void UnescapeCString(nsCString& aCString); + nsresult ScannerNext(const char16_t* text, int32_t length, int32_t pos, + bool isLastBuffer, int32_t* begin, int32_t* end, + bool* _retval); +}; + +/** + * Implements storage of a collection of message tokens and counts for + * a corpus of classified messages + */ + +class CorpusStore : public TokenHash { + public: + CorpusStore(); + ~CorpusStore(); + + /** + * retrieve the token structure for a particular string + * + * @param word the character representation of the token + * + * @return token structure containing counts, null if not found + */ + CorpusToken* get(const char* word); + + /** + * add tokens to the storage, or increment counts if already exists. + * + * @param aTokenizer tokenizer for the list of tokens to remember + * @param aTraitId id for the trait whose counts will be remembered + * @param aCount number of new messages represented by the token list + */ + void rememberTokens(Tokenizer& aTokenizer, uint32_t aTraitId, + uint32_t aCount); + + /** + * decrement counts for tokens in the storage, removing if all counts + * are zero + * + * @param aTokenizer tokenizer for the list of tokens to forget + * @param aTraitId id for the trait whose counts will be removed + * @param aCount number of messages represented by the token list + */ + void forgetTokens(Tokenizer& aTokenizer, uint32_t aTraitId, uint32_t aCount); + + /** + * write the corpus information to file storage + * + * @param aMaximumTokenCount prune tokens if number of tokens exceeds + * this value. == 0 for no pruning + */ + void writeTrainingData(uint32_t aMaximumTokenCount); + + /** + * read the corpus information from file storage + */ + void readTrainingData(); + + /** + * delete the local corpus storage file and data + */ + nsresult resetTrainingData(); + + /** + * get the count of messages whose tokens are stored that are associated + * with a trait + * + * @param aTraitId identifier for the trait + * @return number of messages for that trait + */ + uint32_t getMessageCount(uint32_t aTraitId); + + /** + * set the count of messages whose tokens are stored that are associated + * with a trait + * + * @param aTraitId identifier for the trait + * @param aCount number of messages for that trait + */ + void setMessageCount(uint32_t aTraitId, uint32_t aCount); + + /** + * get the count of messages associated with a particular token and trait + * + * @param token the token string and associated counts + * @param aTraitId identifier for the trait + */ + uint32_t getTraitCount(CorpusToken* token, uint32_t aTraitId); + + /** + * Add (or remove) data from a particular file to the corpus data. + * + * @param aFile the file with the data, in the format: + * + * Format of the trait file for version 1: + * [0xFCA93601] (the 01 is the version) + * for each trait to write: + * [id of trait to write] (0 means end of list) + * [number of messages per trait] + * for each token with non-zero count + * [count] + * [length of word]word + * + * @param aIsAdd should the data be added, or removed? true if adding, + * else removing. + * + * @param aFromTraits array of trait ids used in aFile. If aFile contains + * trait ids that are not in this array, they are not + * remapped, but assumed to be local trait ids. + * + * @param aToTraits array of trait ids, corresponding to elements of + * aFromTraits, that represent the local trait ids to be + * used in storing data from aFile into the local corpus. + * + */ + nsresult UpdateData(nsIFile* aFile, bool aIsAdd, + const nsTArray<uint32_t>& aFromTraits, + const nsTArray<uint32_t>& aToTraits); + + /** + * remove all counts (message and tokens) for a trait id + * + * @param aTrait trait id for the trait to remove + */ + nsresult ClearTrait(uint32_t aTrait); + + protected: + /** + * return the local corpus storage file for junk traits + */ + nsresult getTrainingFile(nsIFile** aFile); + + /** + * return the local corpus storage file for non-junk traits + */ + nsresult getTraitFile(nsIFile** aFile); + + /** + * read token strings from the data file + * + * @param stream file stream with token data + * @param fileSize file size + * @param aTraitId id for the trait whose counts will be read + * @param aIsAdd true to add the counts, false to remove them + * + * @return true if successful, false if error + */ + bool readTokens(FILE* stream, int64_t fileSize, uint32_t aTraitId, + bool aIsAdd); + + /** + * write token strings to the data file + */ + bool writeTokens(FILE* stream, bool shrink, uint32_t aTraitId); + + /** + * remove counts for a token string + */ + void remove(const char* word, uint32_t aTraitId, uint32_t aCount); + + /** + * add counts for a token string, adding the token string if new + */ + CorpusToken* add(const char* word, uint32_t aTraitId, uint32_t aCount); + + /** + * change counts in a trait in the traits array, adding the trait if needed + */ + nsresult updateTrait(CorpusToken* token, uint32_t aTraitId, + int32_t aCountChange); + nsCOMPtr<nsIFile> mTrainingFile; // file used to store junk training data + nsCOMPtr<nsIFile> mTraitFile; // file used to store non-junk + // training data + nsTArray<TraitPerToken> mTraitStore; // memory for linked-list of counts + uint32_t mNextTraitIndex; // index in mTraitStore to first empty + // TraitPerToken + nsTArray<uint32_t> mMessageCounts; // count of messages per trait + // represented in the store + nsTArray<uint32_t> mMessageCountsId; // Parallel array to mMessageCounts, + // with the corresponding trait ID +}; + +class nsBayesianFilter : public nsIJunkMailPlugin, + nsIMsgCorpus, + nsIObserver, + nsSupportsWeakReference { + public: + NS_DECL_ISUPPORTS + NS_DECL_NSIMSGFILTERPLUGIN + NS_DECL_NSIJUNKMAILPLUGIN + NS_DECL_NSIMSGCORPUS + NS_DECL_NSIOBSERVER + + nsBayesianFilter(); + + nsresult Init(); + + nsresult tokenizeMessage(const nsACString& messageURI, + nsIMsgWindow* aMsgWindow, TokenAnalyzer* analyzer); + void classifyMessage(Tokenizer& tokens, const nsACString& messageURI, + nsIJunkMailClassificationListener* listener); + + void classifyMessage(Tokenizer& tokenizer, const nsACString& messageURI, + nsTArray<uint32_t>& aProTraits, + nsTArray<uint32_t>& aAntiTraits, + nsIJunkMailClassificationListener* listener, + nsIMsgTraitClassificationListener* aTraitListener, + nsIMsgTraitDetailListener* aDetailListener); + + void observeMessage(Tokenizer& tokens, const nsACString& messageURI, + nsTArray<uint32_t>& oldClassifications, + nsTArray<uint32_t>& newClassifications, + nsIJunkMailClassificationListener* listener, + nsIMsgTraitClassificationListener* aTraitListener); + + protected: + virtual ~nsBayesianFilter(); + + static void TimerCallback(nsITimer* aTimer, void* aClosure); + + CorpusStore mCorpus; + double mJunkProbabilityThreshold; + int32_t mMaximumTokenCount; + bool mTrainingDataDirty; + int32_t mMinFlushInterval; // in milliseconds, must be positive + // and not too close to 0 + nsCOMPtr<nsITimer> mTimer; + + // index in mAnalysisStore for first empty AnalysisPerToken + uint32_t mNextAnalysisIndex; + // memory for linked list of AnalysisPerToken objects + nsTArray<AnalysisPerToken> mAnalysisStore; + /** + * Determine the location in mAnalysisStore where the AnalysisPerToken + * object for a particular token and trait is stored + */ + uint32_t getAnalysisIndex(Token& token, uint32_t aTraitIndex); + /** + * Set the value of the AnalysisPerToken object for a particular + * token and trait + */ + nsresult setAnalysis(Token& token, uint32_t aTraitIndex, double aDistance, + double aProbability); +}; + +#endif // _nsBayesianFilter_h__ diff --git a/comm/mailnews/extensions/bayesian-spam-filter/nsIncompleteGamma.h b/comm/mailnews/extensions/bayesian-spam-filter/nsIncompleteGamma.h new file mode 100644 index 0000000000..9b96459c7c --- /dev/null +++ b/comm/mailnews/extensions/bayesian-spam-filter/nsIncompleteGamma.h @@ -0,0 +1,239 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef nsIncompleteGamma_h__ +#define nsIncompleteGamma_h__ + +/* An implementation of the incomplete gamma functions for real + arguments. P is defined as + + x + / + 1 [ a - 1 - t + P(a, x) = -------- I t e dt + Gamma(a) ] + / + 0 + + and + + infinity + / + 1 [ a - 1 - t + Q(a, x) = -------- I t e dt + Gamma(a) ] + / + x + + so that P(a,x) + Q(a,x) = 1. + + Both a series expansion and a continued fraction exist. This + implementation uses the more efficient method based on the arguments. + + Either case involves calculating a multiplicative term: + e^(-x)*x^a/Gamma(a). + Here we calculate the log of this term. Most math libraries have a + "lgamma" function but it is not re-entrant. Some libraries have a + "lgamma_r" which is re-entrant. Use it if possible. I have included a + simple replacement but it is certainly not as accurate. + + Relative errors are almost always < 1e-10 and usually < 1e-14. Very + small and very large arguments cause trouble. + + The region where a < 0.5 and x < 0.5 has poor error properties and is + not too stable. Get a better routine if you need results in this + region. + + The error argument will be set negative if there is a domain error or + positive for an internal calculation error, currently lack of + convergence. A value is always returned, though. + + */ + +#include <math.h> +#include <float.h> + +// the main routine +static double nsIncompleteGammaP(double a, double x, int* error); + +// nsLnGamma(z): either a wrapper around lgamma_r or the internal function. +// C_m = B[2*m]/(2*m*(2*m-1)) where B is a Bernoulli number +static const double C_1 = 1.0 / 12.0; +static const double C_2 = -1.0 / 360.0; +static const double C_3 = 1.0 / 1260.0; +static const double C_4 = -1.0 / 1680.0; +static const double C_5 = 1.0 / 1188.0; +static const double C_6 = -691.0 / 360360.0; +static const double C_7 = 1.0 / 156.0; +static const double C_8 = -3617.0 / 122400.0; +static const double C_9 = 43867.0 / 244188.0; +static const double C_10 = -174611.0 / 125400.0; +static const double C_11 = 77683.0 / 5796.0; + +// truncated asymptotic series in 1/z +static inline double lngamma_asymp(double z) { + double w, w2, sum; + w = 1.0 / z; + w2 = w * w; + sum = + w * + (w2 * (w2 * (w2 * (w2 * (w2 * (w2 * (w2 * (w2 * (w2 * (C_11 * w2 + C_10) + + C_9) + + C_8) + + C_7) + + C_6) + + C_5) + + C_4) + + C_3) + + C_2) + + C_1); + + return sum; +} + +struct fact_table_s { + double fact; + double lnfact; +}; + +// for speed and accuracy +static const struct fact_table_s FactTable[] = { + {1.000000000000000, 0.0000000000000000000000e+00}, + {1.000000000000000, 0.0000000000000000000000e+00}, + {2.000000000000000, 6.9314718055994530942869e-01}, + {6.000000000000000, 1.7917594692280550007892e+00}, + {24.00000000000000, 3.1780538303479456197550e+00}, + {120.0000000000000, 4.7874917427820459941458e+00}, + {720.0000000000000, 6.5792512120101009952602e+00}, + {5040.000000000000, 8.5251613610654142999881e+00}, + {40320.00000000000, 1.0604602902745250228925e+01}, + {362880.0000000000, 1.2801827480081469610995e+01}, + {3628800.000000000, 1.5104412573075515295248e+01}, + {39916800.00000000, 1.7502307845873885839769e+01}, + {479001600.0000000, 1.9987214495661886149228e+01}, + {6227020800.000000, 2.2552163853123422886104e+01}, + {87178291200.00000, 2.5191221182738681499610e+01}, + {1307674368000.000, 2.7899271383840891566988e+01}, + {20922789888000.00, 3.0671860106080672803835e+01}, + {355687428096000.0, 3.3505073450136888885825e+01}, + {6402373705728000., 3.6395445208033053576674e+01}}; +#define FactTableLength (int)(sizeof(FactTable) / sizeof(FactTable[0])) + +// for speed +static const double ln_2pi_2 = 0.918938533204672741803; // log(2*PI)/2 + +/* A simple lgamma function, not very robust. + + Valid for z_in > 0 ONLY. + + For z_in > 8 precision is quite good, relative errors < 1e-14 and + usually better. For z_in < 8 relative errors increase but are usually + < 1e-10. In two small regions, 1 +/- .001 and 2 +/- .001 errors + increase quickly. +*/ +static double nsLnGamma(double z_in, int* gsign) { + double scale, z, sum, result; + *gsign = 1; + + int zi = (int)z_in; + if (z_in == (double)zi) { + if (0 < zi && zi <= FactTableLength) + return FactTable[zi - 1].lnfact; // gamma(z) = (z-1)! + } + + for (scale = 1.0, z = z_in; z < 8.0; ++z) scale *= z; + + sum = lngamma_asymp(z); + result = (z - 0.5) * log(z) - z + ln_2pi_2 - log(scale); + result += sum; + return result; +} + +// log( e^(-x)*x^a/Gamma(a) ) +static inline double lnPQfactor(double a, double x) { + int gsign; // ignored because a > 0 + return a * log(x) - x - nsLnGamma(a, &gsign); +} + +static double Pseries(double a, double x, int* error) { + double sum, term; + const double eps = 2.0 * DBL_EPSILON; + const int imax = 5000; + int i; + + sum = term = 1.0 / a; + for (i = 1; i < imax; ++i) { + term *= x / (a + i); + sum += term; + if (fabs(term) < eps * fabs(sum)) break; + } + + if (i >= imax) *error = 1; + + return sum; +} + +static double Qcontfrac(double a, double x, int* error) { + double result, D, C, e, f, term; + const double eps = 2.0 * DBL_EPSILON; + const double small = DBL_EPSILON * DBL_EPSILON * DBL_EPSILON * DBL_EPSILON; + const int imax = 5000; + int i; + + // modified Lentz method + f = x - a + 1.0; + if (fabs(f) < small) f = small; + C = f + 1.0 / small; + D = 1.0 / f; + result = D; + for (i = 1; i < imax; ++i) { + e = i * (a - i); + f += 2.0; + D = f + e * D; + if (fabs(D) < small) D = small; + D = 1.0 / D; + C = f + e / C; + if (fabs(C) < small) C = small; + term = C * D; + result *= term; + if (fabs(term - 1.0) < eps) break; + } + + if (i >= imax) *error = 1; + return result; +} + +static double nsIncompleteGammaP(double a, double x, int* error) { + double result, dom, ldom; + // domain errors. the return values are meaningless but have + // to return something. + *error = -1; + if (a <= 0.0) return 1.0; + if (x < 0.0) return 0.0; + *error = 0; + if (x == 0.0) return 0.0; + + ldom = lnPQfactor(a, x); + dom = exp(ldom); + // might need to adjust the crossover point + if (a <= 0.5) { + if (x < a + 1.0) + result = dom * Pseries(a, x, error); + else + result = 1.0 - dom * Qcontfrac(a, x, error); + } else { + if (x < a) + result = dom * Pseries(a, x, error); + else + result = 1.0 - dom * Qcontfrac(a, x, error); + } + + // not clear if this can ever happen + if (result > 1.0) result = 1.0; + if (result < 0.0) result = 0.0; + return result; +} + +#endif diff --git a/comm/mailnews/extensions/bayesian-spam-filter/test/unit/head_bayes.js b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/head_bayes.js new file mode 100644 index 0000000000..b502dcc2e5 --- /dev/null +++ b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/head_bayes.js @@ -0,0 +1,28 @@ +var { MailServices } = ChromeUtils.import( + "resource:///modules/MailServices.jsm" +); +var { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); +var { mailTestUtils } = ChromeUtils.import( + "resource://testing-common/mailnews/MailTestUtils.jsm" +); +var { localAccountUtils } = ChromeUtils.import( + "resource://testing-common/mailnews/LocalAccountUtils.jsm" +); + +var CC = Components.Constructor; + +// Ensure the profile directory is set up +do_get_profile(); + +function getSpec(aFileName) { + var file = do_get_file("resources/" + aFileName); + var uri = Services.io.newFileURI(file).QueryInterface(Ci.nsIURL); + uri = uri.mutate().setQuery("type=application/x-message-display").finalize(); + return uri.spec; +} + +registerCleanupFunction(function () { + load("../../../../resources/mailShutdown.js"); +}); diff --git a/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/aliases.dat b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/aliases.dat Binary files differnew file mode 100644 index 0000000000..31162459e4 --- /dev/null +++ b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/aliases.dat diff --git a/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/aliases1.eml b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/aliases1.eml new file mode 100644 index 0000000000..4720467fe6 --- /dev/null +++ b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/aliases1.eml @@ -0,0 +1,6 @@ +From - Sat Jan 26 08:43:42 2008 +Subject: test1 +Content-Type: text/plain; charset=iso-8859-1 + +important + diff --git a/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/aliases2.eml b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/aliases2.eml new file mode 100644 index 0000000000..9a251486a9 --- /dev/null +++ b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/aliases2.eml @@ -0,0 +1,6 @@ +From - Sat Jan 26 08:43:42 2008 +Subject: test2 +Content-Type: text/plain; charset=iso-8859-1 + +work + diff --git a/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/aliases3.eml b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/aliases3.eml new file mode 100644 index 0000000000..de31992ac5 --- /dev/null +++ b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/aliases3.eml @@ -0,0 +1,6 @@ +From - Sat Jan 26 08:43:42 2008 +Subject: test3 +Content-Type: text/plain; charset=iso-8859-1 + +very important work + diff --git a/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/ham1.eml b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/ham1.eml new file mode 100644 index 0000000000..6a63f587b8 --- /dev/null +++ b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/ham1.eml @@ -0,0 +1,7 @@ +Date: Tue, 30 Apr 2008 00:12:17 -0700 +From: Mom <mother@example.com> +To: Careful Reader <reader@example.org> +Subject: eat your vegetables +MIME-Version: 1.0 + +vegetables are very important for your health and wealth. diff --git a/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/ham2.eml b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/ham2.eml new file mode 100644 index 0000000000..cd6691b921 --- /dev/null +++ b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/ham2.eml @@ -0,0 +1,8 @@ +Date: Tue, 27 Apr 2006 00:13:23 -0700 +From: Evil Despot <boss@example.com> +To: Careful Reader <reader@example.org> +Subject: finish your report +MIME-Version: 1.0 + +If you want to keep your sorry job and health, finish that +important report before the close of business today. diff --git a/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/msgCorpus.dat b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/msgCorpus.dat Binary files differnew file mode 100644 index 0000000000..f273a4f10c --- /dev/null +++ b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/msgCorpus.dat diff --git a/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/spam1.eml b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/spam1.eml new file mode 100644 index 0000000000..ea629213cc --- /dev/null +++ b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/spam1.eml @@ -0,0 +1,7 @@ +Date: Tue, 29 Apr 2008 00:10:07 -0700 +From: Spam King <spammer@example.com> +To: Careful Reader <reader@example.org> +Subject: viagra is your nigerian xxx dream +MIME-Version: 1.0 + +click here to make lots of money and wealth diff --git a/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/spam2.eml b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/spam2.eml new file mode 100644 index 0000000000..817d328cf2 --- /dev/null +++ b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/spam2.eml @@ -0,0 +1,8 @@ +Date: Mon, 27 Apr 2008 01:02:03 -0700 +From: Stock Pusher <broker@example.net> +To: Careful Reader <reader@example.org> +Subject: ABCD Corporation will soar tomorrow! +MIME-Version: 1.0 + +Make lots of money! Put all of your money into ACBD Corporation +Stock! diff --git a/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/spam3.eml b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/spam3.eml new file mode 100644 index 0000000000..0a524e604b --- /dev/null +++ b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/spam3.eml @@ -0,0 +1,7 @@ +Date: Wed, 30 Apr 2008 01:11:17 -0700 +From: Spam King <spammer@example.com> +To: Careful Reader <reader@example.org> +Subject: we have your nigerian xxx dream +MIME-Version: 1.0 + +Not making lots of money and wealth? Call me! diff --git a/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/spam4.eml b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/spam4.eml new file mode 100644 index 0000000000..775d3b41fa --- /dev/null +++ b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/spam4.eml @@ -0,0 +1,8 @@ +Date: Tue, 28 Apr 2008 01:02:04 -0700 +From: Stock Pusher <broker@example.net> +To: Careful Reader <reader@example.org> +Subject: ABCD Corporation will really soar this time! +MIME-Version: 1.0 + +Make lots of money! Put all of your money into ABCD Corporation +Stock! (We really mean it this time!) diff --git a/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/tokenTest.eml b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/tokenTest.eml new file mode 100644 index 0000000000..d6e7e0ae3d --- /dev/null +++ b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/tokenTest.eml @@ -0,0 +1,14 @@ +Date: Tue, 30 Apr 2008 00:12:17 -0700 +From: Mom <mother@example.com> +To: Careful Reader <reader@example.org> +Subject: eat your vegetables to live long +Received: from c-1-2-3-4.hsd1.wa.example.net ([1.2.3.4] helo=theComputer) + by host301.example.com with esmtpa (Exim 4.69) + (envelope-from <someone@example.com>) + id 1LeEgH-0003GN-Rr + for reader@example.org; Mon, 02 Mar 2009 13:24:06 -0700 +MIME-Version: 1.0 +Message-Id: 14159 +Sender: Bugzilla Test Setup <noreply@example.org> + +This is a sentence. Important URL is http://www.example.org Check it out! diff --git a/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/trainingfile.js b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/trainingfile.js new file mode 100644 index 0000000000..b6d37e879b --- /dev/null +++ b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/resources/trainingfile.js @@ -0,0 +1,108 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// service class to manipulate the junk training.dat file +// code is adapted from Mnehy Thunderbird Extension + +/* exported TrainingData */ +function TrainingData() { + // local constants + + const CC = Components.Constructor; + + // public methods + + this.read = read; + + // public variables + + this.mGoodTokens = 0; + this.mJunkTokens = 0; + this.mGoodMessages = 0; + this.mJunkMessages = 0; + this.mGoodCounts = {}; + this.mJunkCounts = {}; + + // helper functions + + function getJunkStatFile() { + var sBaseDir = Services.dirsvc.get("ProfD", Ci.nsIFile); + var CFileByFile = new CC( + "@mozilla.org/file/local;1", + "nsIFile", + "initWithFile" + ); + var oFile = new CFileByFile(sBaseDir); + oFile.append("training.dat"); + return oFile; + } + + function getBinStream(oFile) { + if (oFile && oFile.exists()) { + var oUri = Services.io.newFileURI(oFile); + // open stream (channel) + let channel = Services.io.newChannelFromURI( + oUri, + null, + Services.scriptSecurityManager.getSystemPrincipal(), + null, + Ci.nsILoadInfo.SEC_ALLOW_CROSS_ORIGIN_SEC_CONTEXT_IS_NULL, + Ci.nsIContentPolicy.TYPE_OTHER + ); + var oStream = channel.open(); + // buffer it + var oBufStream = Cc[ + "@mozilla.org/network/buffered-input-stream;1" + ].createInstance(Ci.nsIBufferedInputStream); + oBufStream.init(oStream, oFile.fileSize); + // read as binary + var oBinStream = Cc["@mozilla.org/binaryinputstream;1"].createInstance( + Ci.nsIBinaryInputStream + ); + oBinStream.setInputStream(oBufStream); + // return it + return oBinStream; + } + return null; + } + + // method specifications + + function read() { + var file = getJunkStatFile(); + + // does the file exist? + Assert.ok(file.exists()); + + var fileStream = getBinStream(file); + + // check magic number + var iMagicNumber = fileStream.read32(); + Assert.equal(iMagicNumber, 0xfeedface); + + // get ham'n'spam numbers + this.mGoodMessages = fileStream.read32(); + this.mJunkMessages = fileStream.read32(); + + // Read good tokens + this.mGoodTokens = fileStream.read32(); + var iRefCount, iTokenLen, sToken; + for (let i = 0; i < this.mGoodTokens; ++i) { + iRefCount = fileStream.read32(); + iTokenLen = fileStream.read32(); + sToken = fileStream.readBytes(iTokenLen); + this.mGoodCounts[sToken] = iRefCount; + } + + // we have no further good tokens, so read junk tokens + this.mJunkTokens = fileStream.read32(); + for (let i = 0; i < this.mJunkTokens; i++) { + // read token data + iRefCount = fileStream.read32(); + iTokenLen = fileStream.read32(); + sToken = fileStream.readBytes(iTokenLen); + this.mJunkCounts[sToken] = iRefCount; + } + } +} diff --git a/comm/mailnews/extensions/bayesian-spam-filter/test/unit/test_bug228675.js b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/test_bug228675.js new file mode 100644 index 0000000000..40180006d7 --- /dev/null +++ b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/test_bug228675.js @@ -0,0 +1,136 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// tests reduction in size of training.dat + +// main setup + +/* import-globals-from resources/trainingfile.js */ +load("resources/trainingfile.js"); + +var { MailServices } = ChromeUtils.import( + "resource:///modules/MailServices.jsm" +); + +// before shrink, the trained messages have 76 tokens. Force shrink. +Services.prefs.setIntPref("mailnews.bayesian_spam_filter.junk_maxtokens", 75); + +// local constants +var kUnclassified = MailServices.junk.UNCLASSIFIED; +var kJunk = MailServices.junk.JUNK; +var kGood = MailServices.junk.GOOD; + +var emails = [ + "ham1.eml", + "ham2.eml", + "spam1.eml", + "spam2.eml", + "spam3.eml", + "spam4.eml", +]; +var classifications = [kGood, kGood, kJunk, kJunk, kJunk, kJunk]; +var trainingData; + +// main test +function run_test() { + localAccountUtils.loadLocalMailAccount(); + MailServices.junk.resetTrainingData(); + + do_test_pending(); + + var email = emails.shift(); + var classification = classifications.shift(); + // additional calls to setMessageClassifiaction are done in the callback + MailServices.junk.setMessageClassification( + getSpec(email), + kUnclassified, + classification, + null, + doTestingListener + ); +} + +var doTestingListener = { + onMessageClassified(aMsgURI, aClassification, aJunkPercent) { + if (!aMsgURI) { + // Ignore end-of-batch signal. + return; + } + var email = emails.shift(); + var classification = classifications.shift(); + if (email) { + MailServices.junk.setMessageClassification( + getSpec(email), + kUnclassified, + classification, + null, + doTestingListener + ); + return; + } + + // all done classifying, time to test + MailServices.junk.shutdown(); // just flushes training.dat + trainingData = new TrainingData(); + trainingData.read(); + + /* + // List training.dat information for debug + dump("training.data results: goodMessages=" + trainingData.mGoodMessages + + " junkMessages = " + trainingData.mJunkMessages + + " goodTokens = " + trainingData.mGoodTokens + + " junkTokens = " + trainingData.mJunkTokens + + "\n"); + print("Good counts"); + for (var token in trainingData.mGoodCounts) + dump("count: " + trainingData.mGoodCounts[token] + " token: " + token + "\n"); + print("Junk Counts"); + for (var token in trainingData.mJunkCounts) + dump("count: " + trainingData.mJunkCounts[token] + " token: " + token + "\n"); + */ + + /* Selected pre-shrink counts after training + training.data results: goodMessages=2 junkMessages = 4 tokens = 78 + Good counts + count: 1 token: subject:report + count: 2 token: important + count: 2 token: to:careful reader <reader@example.org> + + Junk Counts + count: 3 token: make + count: 4 token: money + count: 4 token: to:careful reader <reader@example.org> + count: 2 token: money! + */ + + // Shrinking divides all counts by two. In comments, I show the + // calculation for each test, (pre-shrink count)/2. + + Assert.equal(trainingData.mGoodMessages, 1); // 2/2 + Assert.equal(trainingData.mJunkMessages, 2); // 4/2 + checkToken("money", 0, 2); // (0/2, 4/2) + checkToken("subject:report", 0, 0); // (1/2, 0/2) + checkToken("to:careful reader <reader@example.org>", 1, 2); // (2/2, 4/2) + checkToken("make", 0, 1); // (0/2, 3/2) + checkToken("important", 1, 0); // (2/2, 0/2) + + do_test_finished(); + }, +}; + +// helper functions + +function checkToken(aToken, aGoodCount, aJunkCount) { + print(" checking " + aToken); + var goodCount = trainingData.mGoodCounts[aToken]; + var junkCount = trainingData.mJunkCounts[aToken]; + if (!goodCount) { + goodCount = 0; + } + if (!junkCount) { + junkCount = 0; + } + Assert.equal(goodCount, aGoodCount); + Assert.equal(junkCount, aJunkCount); +} diff --git a/comm/mailnews/extensions/bayesian-spam-filter/test/unit/test_customTokenization.js b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/test_customTokenization.js new file mode 100644 index 0000000000..222a9557d8 --- /dev/null +++ b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/test_customTokenization.js @@ -0,0 +1,197 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// Tests use of custom tokenization, originally introduced in bug 476389 + +var { MailServices } = ChromeUtils.import( + "resource:///modules/MailServices.jsm" +); + +// command functions for test data +var kTrain = 0; // train a file +var kTest = 1; // test headers returned from detail +var kSetup = 2; // run a setup function + +// trait ids +var kProArray = [3]; +var kAntiArray = [4]; + +var gTest; // currently active test + +// The tests array defines the tests to attempt. + +var tests = [ + // test a few tokens using defaults + { + command: kTrain, + fileName: "tokenTest.eml", + }, + { + command: kTest, + fileName: "tokenTest.eml", + tokens: ["important", "subject:eat", "message-id:14159", "http://www"], + nottokens: ["idonotexist", "subject:to"], + }, + + // enable received, disable message-id + // switch tokenization of body to catch full urls (no "." delimiter) + // enable sender, keeping full value + { + command: kSetup, + operation() { + Services.prefs.setCharPref( + "mailnews.bayesian_spam_filter.tokenizeheader.received", + "standard" + ); + Services.prefs.setCharPref( + "mailnews.bayesian_spam_filter.tokenizeheader.message-id", + "false" + ); + Services.prefs.setCharPref( + "mailnews.bayesian_spam_filter.body_delimiters", + " \t\r\n\v" + ); + Services.prefs.setCharPref( + "mailnews.bayesian_spam_filter.tokenizeheader.sender", + "full" + ); + }, + }, + { + command: kTrain, + fileName: "tokenTest.eml", + }, + { + command: kTest, + fileName: "tokenTest.eml", + tokens: [ + "important", + "subject:eat", + "received:reader@example", + "skip:h 20", + "sender:bugzilla test setup <noreply@example.org>", + "received:<someone@example", + ], + nottokens: ["message-id:14159", "http://www"], + }, + + // increase the length of the maximum token to catch full URLs in the body + // add <>;, remove . from standard header delimiters to better capture emails + // use custom delimiters on sender, without "." or "<>" + { + command: kSetup, + operation() { + Services.prefs.setIntPref( + "mailnews.bayesian_spam_filter.maxlengthfortoken", + 50 + ); + Services.prefs.setCharPref( + "mailnews.bayesian_spam_filter.header_delimiters", + " ;<>\t\r\n\v" + ); + Services.prefs.setCharPref( + "mailnews.bayesian_spam_filter.tokenizeheader.sender", + " \t\r\n\v" + ); + }, + }, + { + command: kTrain, + fileName: "tokenTest.eml", + }, + { + command: kTest, + fileName: "tokenTest.eml", + tokens: [ + "received:someone@example.com", + "http://www.example.org", + "received:reader@example.org", + "sender:<noreply@example.org>", + ], + nottokens: ["skip:h 20", "received:<someone@example"], + }, +]; + +// main test +function run_test() { + localAccountUtils.loadLocalMailAccount(); + do_test_pending(); + + startCommand(); +} + +var listener = { + // nsIMsgTraitClassificationListener implementation + onMessageTraitsClassified(aMsgURI, aTraits, aPercents) { + startCommand(); + }, + + onMessageTraitDetails( + aMsgURI, + aProTrait, + aTokenString, + aTokenPercents, + aRunningPercents + ) { + print("Details for " + aMsgURI); + for (let i = 0; i < aTokenString.length; i++) { + print("Token " + aTokenString[i]); + } + + // we should have these tokens + for (let value of gTest.tokens) { + print("We should have '" + value + "'? "); + Assert.ok(aTokenString.includes(value)); + } + + // should not have these tokens + for (let value of gTest.nottokens) { + print("We should not have '" + value + "'? "); + Assert.ok(!aTokenString.includes(value)); + } + startCommand(); + }, +}; + +// start the next test command +function startCommand() { + if (!tests.length) { + // Do we have more commands? + // no, all done + do_test_finished(); + return; + } + + gTest = tests.shift(); + // print("StartCommand command = " + gTest.command + ", remaining tests " + tests.length); + switch (gTest.command) { + case kTrain: + // train message + + MailServices.junk.setMsgTraitClassification( + getSpec(gTest.fileName), // aMsgURI + [], // aOldTraits + kProArray, // aNewTraits + listener + ); // [optional] in nsIMsgTraitClassificationListener aTraitListener + // null, // [optional] in nsIMsgWindow aMsgWindow + // null, // [optional] in nsIJunkMailClassificationListener aJunkListener + break; + + case kTest: + // test headers from detail message + MailServices.junk.detailMessage( + getSpec(gTest.fileName), // in string aMsgURI + kProArray[0], // proTrait + kAntiArray[0], // antiTrait + listener + ); // in nsIMsgTraitDetailListener aDetailListener + break; + + case kSetup: + gTest.operation(); + startCommand(); + break; + } +} diff --git a/comm/mailnews/extensions/bayesian-spam-filter/test/unit/test_junkAsTraits.js b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/test_junkAsTraits.js new file mode 100644 index 0000000000..a1800b93e7 --- /dev/null +++ b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/test_junkAsTraits.js @@ -0,0 +1,574 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// tests calls to the bayesian filter plugin to train, classify, and forget +// messages using both the older junk-oriented calls, as well as the newer +// trait-oriented calls. Only a single trait is tested. The main intent of +// these tests is to demonstrate that both the old junk-oriented calls and the +// new trait-oriented calls give the same results on junk processing. + +var { MailServices } = ChromeUtils.import( + "resource:///modules/MailServices.jsm" +); + +// local constants +var kUnclassified = MailServices.junk.UNCLASSIFIED; +var kJunk = MailServices.junk.JUNK; +var kGood = MailServices.junk.GOOD; +var kJunkTrait = MailServices.junk.JUNK_TRAIT; +var kGoodTrait = MailServices.junk.GOOD_TRAIT; +var kIsHamScore = MailServices.junk.IS_HAM_SCORE; +var kIsSpamScore = MailServices.junk.IS_SPAM_SCORE; + +// command functions for test data +var kTrainJ = 0; // train using junk method +var kTrainT = 1; // train using trait method +var kClassJ = 2; // classify using junk method +var kClassT = 3; // classify using trait method +var kForgetJ = 4; // forget training using junk method +var kForgetT = 5; // forget training using trait method +var kCounts = 6; // test token and message counts + +var gProArray = [], + gAntiArray = []; // traits arrays, pro is junk, anti is good +var gTest; // currently active test + +// The tests array defines the tests to attempt. Format of +// an element "test" of this array (except for kCounts): +// +// test.command: function to perform, see definitions above +// test.fileName: file containing message to test +// test.junkPercent: sets the classification (for Class or Forget commands) +// tests the classification (for Class commands) +// As a special case for the no-training tests, if +// junkPercent is negative, test its absolute value +// for percents, but reverse the junk/good classification +// test.traitListener: should we use the trait listener call? +// test.junkListener: should we use the junk listener call? + +var tests = [ + // test the trait-based calls. We mix trait listeners, junk listeners, + // and both + + { + // with no training, percents is 50 - but classifies as junk + command: kClassT, + fileName: "ham1.eml", + junkPercent: -50, // negative means classifies as junk + traitListener: false, + junkListener: true, + }, + { + // train 1 ham message + command: kTrainT, + fileName: "ham1.eml", + junkPercent: 0, + traitListener: false, + junkListener: true, + }, + { + // with ham but no spam training, percents are 0 and classifies as ham + command: kClassT, + fileName: "ham1.eml", + junkPercent: 0, + traitListener: false, + junkListener: true, + }, + { + // train 1 spam message + command: kTrainT, + fileName: "spam1.eml", + junkPercent: 100, + traitListener: true, + junkListener: false, + }, + { + // the trained messages will classify at 0 and 100 + command: kClassT, + fileName: "ham1.eml", + junkPercent: 0, + traitListener: false, + junkListener: true, + }, + { + command: kClassT, + fileName: "spam1.eml", + junkPercent: 100, + traitListener: true, + junkListener: false, + }, + { + // ham2, spam2, spam4 give partial percents, but still ham + command: kClassT, + fileName: "ham2.eml", + junkPercent: 8, + traitListener: true, + junkListener: true, + }, + { + command: kClassT, + fileName: "spam2.eml", + junkPercent: 81, + traitListener: false, + junkListener: true, + }, + { + command: kClassT, + fileName: "spam4.eml", + junkPercent: 81, + traitListener: true, + junkListener: false, + }, + { + // spam3 evaluates to spam + command: kClassT, + fileName: "spam3.eml", + junkPercent: 98, + traitListener: true, + junkListener: true, + }, + { + // train ham2, then test percents of 0 (clearly good) + command: kTrainT, + fileName: "ham2.eml", + junkPercent: 0, + traitListener: true, + junkListener: true, + }, + { + command: kClassT, + fileName: "ham2.eml", + junkPercent: 0, + traitListener: true, + junkListener: true, + }, + { + // forget ham2, percents should return to partial value + command: kForgetT, + fileName: "ham2.eml", + junkPercent: 0, + traitListener: false, + junkListener: true, + }, + { + command: kClassT, + fileName: "ham2.eml", + junkPercent: 8, + traitListener: true, + junkListener: true, + }, + { + // train, classify, forget, reclassify spam4 + command: kTrainT, + fileName: "spam4.eml", + junkPercent: 100, + traitListener: true, + junkListener: true, + }, + { + command: kClassT, + fileName: "spam4.eml", + junkPercent: 100, + traitListener: true, + junkListener: true, + }, + { + command: kCounts, + tokenCount: 66, // count of tokens in the corpus + junkCount: 2, // count of junk messages in the corpus + goodCount: 1, // count of good messages in the corpus + }, + { + command: kForgetT, + fileName: "spam4.eml", + junkPercent: 100, + traitListener: true, + junkListener: false, + }, + { + command: kClassT, + fileName: "spam4.eml", + junkPercent: 81, + traitListener: true, + junkListener: true, + }, + { + // forget ham1 and spam1 to empty training + command: kForgetT, + fileName: "ham1.eml", + junkPercent: 0, + traitListener: true, + junkListener: true, + }, + { + command: kForgetT, + fileName: "spam1.eml", + junkPercent: 100, + traitListener: true, + junkListener: true, + }, + // repeat the whole sequence using the junk calls + { + // train 1 ham and 1 spam message + command: kTrainJ, + fileName: "ham1.eml", + junkPercent: 0, + traitListener: false, + junkListener: true, + }, + { + command: kTrainJ, + fileName: "spam1.eml", + junkPercent: 100, + traitListener: false, + junkListener: true, + }, + { + // the trained messages will classify at 0 and 100 + command: kClassJ, + fileName: "ham1.eml", + junkPercent: 0, + traitListener: false, + junkListener: true, + }, + { + command: kClassJ, + fileName: "spam1.eml", + junkPercent: 100, + traitListener: false, + junkListener: true, + }, + { + // ham2, spam2, spam4 give partial percents, but still ham + command: kClassJ, + fileName: "ham2.eml", + junkPercent: 8, + traitListener: false, + junkListener: true, + }, + { + command: kClassJ, + fileName: "spam2.eml", + junkPercent: 81, + traitListener: false, + junkListener: true, + }, + { + command: kClassJ, + fileName: "spam4.eml", + junkPercent: 81, + traitListener: false, + junkListener: true, + }, + { + // spam3 evaluates to spam + command: kClassJ, + fileName: "spam3.eml", + junkPercent: 98, + traitListener: false, + junkListener: true, + }, + { + // train ham2, then test percents of 0 (clearly good) + command: kTrainJ, + fileName: "ham2.eml", + junkPercent: 0, + traitListener: false, + junkListener: true, + }, + { + command: kClassJ, + fileName: "ham2.eml", + junkPercent: 0, + traitListener: false, + junkListener: true, + }, + { + // forget ham2, percents should return to partial value + command: kForgetJ, + fileName: "ham2.eml", + junkPercent: 0, + traitListener: false, + junkListener: true, + }, + { + command: kClassJ, + fileName: "ham2.eml", + junkPercent: 8, + traitListener: false, + junkListener: true, + }, + { + // train, classify, forget, reclassify spam4 + command: kTrainJ, + fileName: "spam4.eml", + junkPercent: 100, + traitListener: false, + junkListener: true, + }, + { + command: kClassJ, + fileName: "spam4.eml", + junkPercent: 100, + traitListener: false, + junkListener: true, + }, + { + command: kForgetJ, + fileName: "spam4.eml", + junkPercent: 100, + traitListener: false, + junkListener: true, + }, + { + command: kClassJ, + fileName: "spam4.eml", + junkPercent: 81, + traitListener: false, + junkListener: true, + }, + { + // forget ham1 and spam1 to be empty + command: kForgetJ, + fileName: "ham1.eml", + junkPercent: 0, + traitListener: false, + junkListener: true, + }, + { + command: kForgetJ, + fileName: "spam1.eml", + junkPercent: 100, + traitListener: false, + junkListener: true, + }, +]; + +// main test +function run_test() { + localAccountUtils.loadLocalMailAccount(); + do_test_pending(); + + // setup pro/anti arrays as junk/good + gProArray.push(kJunkTrait); + gAntiArray.push(kGoodTrait); + + startCommand(); +} + +var junkListener = { + // nsIJunkMailClassificationListener implementation + onMessageClassified(aMsgURI, aClassification, aJunkPercent) { + if (!aMsgURI) { + // Ignore end-of-batch signal. + return; + } + // print("Message URI is " + aMsgURI); + // print("Junk percent is " + aJunkPercent); + // print("Classification is " + aClassification); + var command = gTest.command; + var junkPercent = gTest.junkPercent; + // file returned correctly + Assert.equal(getSpec(gTest.fileName), aMsgURI); + + // checks of aClassification + + // forget returns unclassified + if (command == kForgetJ || command == kForgetT) { + Assert.equal(aClassification, kUnclassified); + } else { + // classification or train should return an actual classification + // check junk classification set by default cutoff of 90 + var isGood = Math.abs(junkPercent) < 90; + if (junkPercent < 0) { + isGood = !isGood; + } + Assert.equal(aClassification, isGood ? kGood : kJunk); + } + + // checks of aJunkPercent + + if (command == kClassJ || command == kClassT) { + // classify returns the actual junk percents + Assert.equal(Math.abs(junkPercent), aJunkPercent); + } else if (command == kTrainJ || command == kTrainT) { + // train returns the ham and spam limits + Assert.equal(aJunkPercent, junkPercent < 90 ? kIsHamScore : kIsSpamScore); + } else { + // Forget always returns 0. + Assert.equal(aJunkPercent, 0); + } + + // if the current test includes a trait listener, it will + // run next, so we defer to it for starting the next command + if (gTest.traitListener) { + return; + } + startCommand(); + }, +}; + +var traitListener = { + // nsIMsgTraitClassificationListener implementation + onMessageTraitsClassified(aMsgURI, aTraits, aPercents) { + if (!aMsgURI) { + // Ignore end-of-batch signal. + return; + } + // print("(Trait Listener)Message URI is " + aMsgURI); + // print("(Trait Listener)Junk percent is " + aPercents); + var command = gTest.command; + var junkPercent = gTest.junkPercent; + // print("command, junkPercent is " + command + " , " + junkPercent); + + Assert.equal(getSpec(gTest.fileName), aMsgURI); + + // checks of aPercents + + if (command == kForgetJ || command == kForgetT) { + // "forgets" with null newClassifications does not return a percent + Assert.equal(aPercents.length, 0); + } else { + var percent = aPercents[0]; + // print("Percent is " + percent); + if (command == kClassJ || command == kClassT) { + // Classify returns actual percents + Assert.equal(percent, junkPercent); + } else { + // Train simply returns 100. + Assert.equal(percent, 100); + } + } + + // checks of aTraits + + if (command == kForgetJ || command == kForgetT) { + // "forgets" with null newClassifications does not return a + // classification + Assert.equal(aTraits.length, 0); + } else if (command == kClassJ || command == kClassT) { + // classification just returns the tested "Pro" trait (junk) + let trait = aTraits[0]; + Assert.equal(trait, kJunkTrait); + } else { + // training returns the actual trait trained + let trait = aTraits[0]; + Assert.equal(trait, junkPercent < 90 ? kGoodTrait : kJunkTrait); + } + + // All done, start the next test + startCommand(); + }, +}; + +// start the next test command +function startCommand() { + if (!tests.length) { + // Do we have more commands? + // no, all done + do_test_finished(); + return; + } + + gTest = tests.shift(); + print( + "StartCommand command = " + + gTest.command + + ", remaining tests " + + tests.length + ); + var command = gTest.command; + var junkPercent = gTest.junkPercent; + var fileName = gTest.fileName; + var tListener = gTest.traitListener; + var jListener = gTest.junkListener; + switch (command) { + case kTrainJ: + // train message using junk call + MailServices.junk.setMessageClassification( + getSpec(fileName), // in string aMsgURI + null, // in nsMsgJunkStatus aOldUserClassification + junkPercent == kIsHamScore ? kGood : kJunk, // in nsMsgJunkStatus aNewClassification + null, // in nsIMsgWindow aMsgWindow + junkListener + ); // in nsIJunkMailClassificationListener aListener); + break; + + case kTrainT: + // train message using trait call + MailServices.junk.setMsgTraitClassification( + getSpec(fileName), // aMsgURI + [], // aOldTraits + junkPercent == kIsSpamScore ? gProArray : gAntiArray, // aNewTraits + tListener ? traitListener : null, // aTraitListener + null, // aMsgWindow + jListener ? junkListener : null + ); + break; + + case kClassJ: + // classify message using junk call + MailServices.junk.classifyMessage( + getSpec(fileName), // in string aMsgURI + null, // in nsIMsgWindow aMsgWindow + junkListener + ); // in nsIJunkMailClassificationListener aListener + break; + + case kClassT: + // classify message using trait call + MailServices.junk.classifyTraitsInMessage( + getSpec(fileName), // in string aMsgURI + gProArray, // in array aProTraits, + gAntiArray, // in array aAntiTraits + tListener ? traitListener : null, // in nsIMsgTraitClassificationListener aTraitListener + null, // in nsIMsgWindow aMsgWindow + jListener ? junkListener : null + ); // in nsIJunkMailClassificationListener aJunkListener + break; + + case kForgetJ: + // forget message using junk call + MailServices.junk.setMessageClassification( + getSpec(fileName), // in string aMsgURI + junkPercent == kIsHamScore ? kGood : kJunk, // in nsMsgJunkStatus aOldUserClassification + null, // in nsMsgJunkStatus aNewClassification, + null, // in nsIMsgWindow aMsgWindow, + junkListener + ); // in nsIJunkMailClassificationListener aListener + break; + + case kForgetT: + // forget message using trait call + MailServices.junk.setMsgTraitClassification( + getSpec(fileName), // in string aMsgURI + junkPercent == kIsSpamScore ? gProArray : gAntiArray, // in array aOldTraits + [], // in array aNewTraits + tListener ? traitListener : null, // in nsIMsgTraitClassificationListener aTraitListener + null, // in nsIMsgWindow aMsgWindow + jListener ? junkListener : null + ); // in nsIJunkMailClassificationListener aJunkListener + break; + + case kCounts: + // test counts + let msgCount = {}; + let nsIMsgCorpus = MailServices.junk.QueryInterface(Ci.nsIMsgCorpus); + let tokenCount = nsIMsgCorpus.corpusCounts(null, {}); + nsIMsgCorpus.corpusCounts(kJunkTrait, msgCount); + let junkCount = msgCount.value; + nsIMsgCorpus.corpusCounts(kGoodTrait, msgCount); + let goodCount = msgCount.value; + print( + "tokenCount, junkCount, goodCount is " + tokenCount, + junkCount, + goodCount + ); + Assert.equal(tokenCount, gTest.tokenCount); + Assert.equal(junkCount, gTest.junkCount); + Assert.equal(goodCount, gTest.goodCount); + do_timeout(0, startCommand); + break; + } +} diff --git a/comm/mailnews/extensions/bayesian-spam-filter/test/unit/test_msgCorpus.js b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/test_msgCorpus.js new file mode 100644 index 0000000000..0c39215fcb --- /dev/null +++ b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/test_msgCorpus.js @@ -0,0 +1,144 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// Tests corpus management functions using nsIMsgCorpus + +var { MailServices } = ChromeUtils.import( + "resource:///modules/MailServices.jsm" +); + +var msgCorpus = MailServices.junk.QueryInterface(Ci.nsIMsgCorpus); + +// tokens found in the test corpus file. trait 1001 was trained with +// 2 messages, and trait 1003 with 1. + +var tokenData = [ + // [traitid, count, token] + [1001, 0, "iDoNotExist"], + [1001, 1, "linecount"], + [1001, 2, "envelope-to:kenttest@caspia.com"], + [1003, 0, "iAlsoDoNotExist"], + [1003, 0, "isjunk"], // in 1001 but not 1003 + [1003, 1, "linecount"], + [1003, 1, "subject:test"], + [1003, 1, "envelope-to:kenttest@caspia.com"], +]; + +// list of tests + +var gTests = [ + // train two different combinations of messages + function checkLoadOnce() { + let fileName = "msgCorpus.dat"; + let file = do_get_file("resources/" + fileName); + msgCorpus.updateData(file, true); + + // check message counts + let messageCount = {}; + msgCorpus.corpusCounts(1001, messageCount); + Assert.equal(2, messageCount.value); + msgCorpus.corpusCounts(1003, messageCount); + Assert.equal(1, messageCount.value); + + for (let i = 0; i < tokenData.length; i++) { + let id = tokenData[i][0]; + let count = tokenData[i][1]; + let word = tokenData[i][2]; + Assert.equal(count, msgCorpus.getTokenCount(word, id)); + } + }, + function checkLoadTwice() { + let fileName = "msgCorpus.dat"; + let file = do_get_file("resources/" + fileName); + msgCorpus.updateData(file, true); + + // check message counts + let messageCount = {}; + msgCorpus.corpusCounts(1001, messageCount); + Assert.equal(4, messageCount.value); + msgCorpus.corpusCounts(1003, messageCount); + Assert.equal(2, messageCount.value); + + for (let i = 0; i < tokenData.length; i++) { + let id = tokenData[i][0]; + let count = 2 * tokenData[i][1]; + let word = tokenData[i][2]; + Assert.equal(count, msgCorpus.getTokenCount(word, id)); + } + }, + // remap the ids in the file to different local ids + function loadWithRemap() { + let fileName = "msgCorpus.dat"; + let file = do_get_file("resources/" + fileName); + msgCorpus.updateData(file, true, [1001, 1003], [1, 3]); + + for (let i = 0; i < tokenData.length; i++) { + let id = tokenData[i][0] - 1000; + let count = tokenData[i][1]; + let word = tokenData[i][2]; + Assert.equal(count, msgCorpus.getTokenCount(word, id)); + } + }, + // test removing data + function checkRemove() { + let fileName = "msgCorpus.dat"; + let file = do_get_file("resources/" + fileName); + msgCorpus.updateData(file, false); + + // check message counts + let messageCount = {}; + msgCorpus.corpusCounts(1001, messageCount); + Assert.equal(2, messageCount.value); + msgCorpus.corpusCounts(1003, messageCount); + Assert.equal(1, messageCount.value); + + for (let i = 0; i < tokenData.length; i++) { + let id = tokenData[i][0]; + let count = tokenData[i][1]; + let word = tokenData[i][2]; + Assert.equal(count, msgCorpus.getTokenCount(word, id)); + } + }, + // test clearing a trait + function checkClear() { + let messageCountObject = {}; + /* + msgCorpus.corpusCounts(1001, messageCountObject); + let v1001 = messageCountObject.value; + msgCorpus.corpusCounts(1003, messageCountObject); + let v1003 = messageCountObject.value; + dump("pre-clear value " + v1001 + " " + v1003 + "\n"); + /**/ + msgCorpus.clearTrait(1001); + // check that the message count is zero + msgCorpus.corpusCounts(1001, messageCountObject); + Assert.equal(0, messageCountObject.value); + // but the other trait should still have counts + msgCorpus.corpusCounts(1003, messageCountObject); + Assert.equal(1, messageCountObject.value); + // check that token count was cleared + for (let i = 0; i < tokenData.length; i++) { + let id = tokenData[i][0]; + let count = tokenData[i][1]; + let word = tokenData[i][2]; + Assert.equal(id == 1001 ? 0 : count, msgCorpus.getTokenCount(word, id)); + } + }, +]; + +// main test +function run_test() { + do_test_pending(); + while (true) { + if (!gTests.length) { + // Do we have more commands? + // no, all done + do_test_finished(); + return; + } + + let test = gTests.shift(); + test(); + } +} diff --git a/comm/mailnews/extensions/bayesian-spam-filter/test/unit/test_traitAliases.js b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/test_traitAliases.js new file mode 100644 index 0000000000..41a9f22a9b --- /dev/null +++ b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/test_traitAliases.js @@ -0,0 +1,172 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// Tests bayes trait analysis with aliases. Adapted from test_traits.js + +/* + * These tests rely on data stored in a file, with the same format as traits.dat, + * that was trained in the following manner. There are two training messages, + * included here as files aliases1.eml and aliases2.eml Aliases.dat was trained on + * each of these messages, for different trait indices, as follows, with + * columns showing the training count for each trait index: + * + * file count(1001) count(1005) count(1007) count(1009) + * + * aliases1.eml 1 0 2 0 + * aliases2.eml 0 1 0 1 + * + * There is also a third email file, aliases3.eml, which combines tokens + * from aliases1.eml and aliases2.eml + * + * The goal here is to demonstrate that traits 1001 and 1007, and traits + * 1005 and 1009, can be combined using aliases. We classify messages with + * trait 1001 as the PRO trait, and 1005 as the ANTI trait. + * + * With these characteristics, I've run a trait analysis without aliases, and + * determined that the following is the correct percentage results from the + * analysis for each message. "Train11" means that the training was 1 pro count + * from aliases1.eml, and 1 anti count from alias2.eml. "Train32" is 3 pro counts, + * and 2 anti counts. + * + * percentage + * file Train11 Train32 + * + * alias1.eml 92 98 + * alias2.eml 8 3 + * alias3.eml 50 53 + */ + +var { MailServices } = ChromeUtils.import( + "resource:///modules/MailServices.jsm" +); + +var traitService = Cc["@mozilla.org/msg-trait-service;1"].getService( + Ci.nsIMsgTraitService +); +var kProTrait = 1001; +var kAntiTrait = 1005; +var kProAlias = 1007; +var kAntiAlias = 1009; + +var gTest; // currently active test + +// The tests array defines the tests to attempt. Format of +// an element "test" of this array: +// +// test.fileName: file containing message to test +// test.proAliases: array of aliases for the pro trait +// test.antiAliases: array of aliases for the anti trait +// test.percent: expected results from the classifier + +var tests = [ + { + fileName: "aliases1.eml", + proAliases: [], + antiAliases: [], + percent: 92, + }, + { + fileName: "aliases2.eml", + proAliases: [], + antiAliases: [], + percent: 8, + }, + { + fileName: "aliases3.eml", + proAliases: [], + antiAliases: [], + percent: 50, + }, + { + fileName: "aliases1.eml", + proAliases: [kProAlias], + antiAliases: [kAntiAlias], + percent: 98, + }, + { + fileName: "aliases2.eml", + proAliases: [kProAlias], + antiAliases: [kAntiAlias], + percent: 3, + }, + { + fileName: "aliases3.eml", + proAliases: [kProAlias], + antiAliases: [kAntiAlias], + percent: 53, + }, +]; + +// main test +function run_test() { + localAccountUtils.loadLocalMailAccount(); + + // load in the aliases trait testing file + MailServices.junk + .QueryInterface(Ci.nsIMsgCorpus) + .updateData(do_get_file("resources/aliases.dat"), true); + do_test_pending(); + + startCommand(); +} + +var listener = { + // nsIMsgTraitClassificationListener implementation + onMessageTraitsClassified(aMsgURI, aTraits, aPercents) { + // print("Message URI is " + aMsgURI); + if (!aMsgURI) { + // Ignore end-of-batch signal. + return; + } + + Assert.equal(aPercents[0], gTest.percent); + // All done, start the next test + startCommand(); + }, +}; + +// start the next test command +function startCommand() { + if (!tests.length) { + // Do we have more commands? + // no, all done + do_test_finished(); + return; + } + + gTest = tests.shift(); + + // classify message + var antiArray = [kAntiTrait]; + var proArray = [kProTrait]; + + // remove any existing aliases + let proAliases = traitService.getAliases(kProTrait); + let antiAliases = traitService.getAliases(kAntiTrait); + let proAlias; + let antiAlias; + while ((proAlias = proAliases.pop())) { + traitService.removeAlias(kProTrait, proAlias); + } + while ((antiAlias = antiAliases.pop())) { + traitService.removeAlias(kAntiTrait, antiAlias); + } + + // add new aliases + while ((proAlias = gTest.proAliases.pop())) { + traitService.addAlias(kProTrait, proAlias); + } + while ((antiAlias = gTest.antiAliases.pop())) { + traitService.addAlias(kAntiTrait, antiAlias); + } + + MailServices.junk.classifyTraitsInMessage( + getSpec(gTest.fileName), // in string aMsgURI + proArray, // in array aProTraits, + antiArray, // in array aAntiTraits + listener + ); // in nsIMsgTraitClassificationListener aTraitListener + // null, // [optional] in nsIMsgWindow aMsgWindow + // null, // [optional] in nsIJunkMailClassificationListener aJunkListener +} diff --git a/comm/mailnews/extensions/bayesian-spam-filter/test/unit/test_traits.js b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/test_traits.js new file mode 100644 index 0000000000..b005db72cc --- /dev/null +++ b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/test_traits.js @@ -0,0 +1,287 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// Tests bayes trait analysis + +// I make this an instance so that I know I can reset and get +// a completely new component. Should be getService in production code. +var nsIJunkMailPlugin = Cc[ + "@mozilla.org/messenger/filter-plugin;1?name=bayesianfilter" +].createInstance(Ci.nsIJunkMailPlugin); + +// command functions for test data +var kTrain = 0; // train a file as a trait +var kClass = 1; // classify files with traits +var kReset = 2; // reload plugin, reading in data from disk +var kDetail = 3; // test details + +var gTest; // currently active test + +// The tests array defines the tests to attempt. Format of +// an element "test" of this array: +// +// test.command: function to perform, see definitions above +// test.fileName: file(s) containing message(s) to test +// test.traitIds: Array of traits to train (kTrain) or pro trait (kClass) +// test.traitAntiIds: Array of anti traits to classify +// test.percents: array of arrays (1 per message, 1 per trait) of +// expected results from the classifier + +var tests = [ + // train two different combinations of messages + { + command: kTrain, + fileName: "ham1.eml", + traitIds: [3, 6], + }, + { + command: kTrain, + fileName: "spam1.eml", + traitIds: [4], + }, + { + command: kTrain, + fileName: "spam4.eml", + traitIds: [5], + }, + // test the message classifications using both singular and plural classifier + { + command: kClass, + fileName: "ham1.eml", + traitIds: [4, 6], + traitAntiIds: [3, 5], + // ham1 is trained "anti" for first test, "pro" for second + percents: [[0, 100]], + }, + { + command: kClass, + fileName: "ham2.eml", + traitIds: [4, 6], + traitAntiIds: [3, 5], + // these are partial percents for an untrained message. ham2 is similar to ham1 + percents: [[8, 95]], + }, + { + command: kDetail, + fileName: "spam2.eml", + traitIds: [4], + traitAntiIds: [3], + percents: { + lots: 84, + money: 84, + make: 84, + your: 16, + }, + runnings: [84, 92, 95, 81], + }, + { + command: kClass, + fileName: "spam1.eml,spam2.eml,spam3.eml,spam4.eml", + traitIds: [4, 6], + traitAntiIds: [3, 5], + // spam1 trained as "pro" for first pro/anti pair + // spam4 trained as "anti" for second pro/anti pair + // others are partials + percents: [ + [100, 50], + [81, 0], + [98, 50], + [81, 0], + ], + }, + // reset the plugin, read in data, and retest the classification + // this tests the trait file writing + { + command: kReset, + }, + { + command: kClass, + fileName: "ham1.eml", + traitIds: [4, 6], + traitAntiIds: [3, 5], + percents: [[0, 100]], + }, + { + command: kClass, + fileName: "ham2.eml", + traitIds: [4, 6], + traitAntiIds: [3, 5], + percents: [[8, 95]], + }, + { + command: kClass, + fileName: "spam1.eml,spam2.eml,spam3.eml,spam4.eml", + traitIds: [4, 6], + traitAntiIds: [3, 5], + percents: [ + [100, 50], + [81, 0], + [98, 50], + [81, 0], + ], + }, +]; + +// main test +function run_test() { + localAccountUtils.loadLocalMailAccount(); + do_test_pending(); + + startCommand(); +} + +var listener = { + // nsIMsgTraitClassificationListener implementation + onMessageTraitsClassified(aMsgURI, aTraits, aPercents) { + // print("Message URI is " + aMsgURI); + if (!aMsgURI) { + // Ignore end-of-batch signal. + return; + } + + switch (gTest.command) { + case kClass: + Assert.equal(gTest.files[gTest.currentIndex], aMsgURI); + var currentPercents = gTest.percents[gTest.currentIndex]; + for (let i = 0; i < currentPercents.length; i++) { + // print("expecting score " + currentPercents[i] + + // " got score " + aPercents[i]); + Assert.equal(currentPercents[i], aPercents[i]); + } + gTest.currentIndex++; + break; + + case kTrain: // We tested this some in test_junkAsTraits.js, so let's not bother + default: + break; + } + if (!--gTest.callbacks) { + // All done, start the next test + startCommand(); + } + }, + onMessageTraitDetails( + aMsgURI, + aProTrait, + aTokenString, + aTokenPercents, + aRunningPercents + ) { + print("Details for " + aMsgURI); + for (let i = 0; i < aTokenString.length; i++) { + print( + "Percent " + + aTokenPercents[i] + + " Running " + + aRunningPercents[i] + + " Token " + + aTokenString[i] + ); + Assert.ok(aTokenString[i] in gTest.percents); + + Assert.equal(gTest.percents[aTokenString[i]], aTokenPercents[i]); + Assert.equal(gTest.runnings[i], aRunningPercents[i]); + delete gTest.percents[aTokenString[i]]; + } + Assert.equal(Object.keys(gTest.percents).length, 0); + if (gTest.command == kClass) { + gTest.currentIndex++; + } + startCommand(); + }, +}; + +// start the next test command +function startCommand() { + if (!tests.length) { + // Do we have more commands? + // no, all done + do_test_finished(); + return; + } + + gTest = tests.shift(); + print( + "StartCommand command = " + + gTest.command + + ", remaining tests " + + tests.length + ); + switch (gTest.command) { + case kTrain: { + // train message + let proArray = []; + for (let i = 0; i < gTest.traitIds.length; i++) { + proArray.push(gTest.traitIds[i]); + } + gTest.callbacks = 1; + + nsIJunkMailPlugin.setMsgTraitClassification( + getSpec(gTest.fileName), // aMsgURI + [], // aOldTraits + proArray, // aNewTraits + listener + ); // [optional] in nsIMsgTraitClassificationListener aTraitListener + // null, // [optional] in nsIMsgWindow aMsgWindow + // null, // [optional] in nsIJunkMailClassificationListener aJunkListener + break; + } + case kClass: { + // classify message + var antiArray = []; + let proArray = []; + for (let i = 0; i < gTest.traitIds.length; i++) { + antiArray.push(gTest.traitAntiIds[i]); + proArray.push(gTest.traitIds[i]); + } + gTest.files = gTest.fileName.split(","); + gTest.callbacks = gTest.files.length; + gTest.currentIndex = 0; + for (let i = 0; i < gTest.files.length; i++) { + gTest.files[i] = getSpec(gTest.files[i]); + } + if (gTest.files.length == 1) { + // use the singular classifier + nsIJunkMailPlugin.classifyTraitsInMessage( + getSpec(gTest.fileName), // in string aMsgURI + proArray, // in array aProTraits, + antiArray, // in array aAntiTraits + listener + ); // in nsIMsgTraitClassificationListener aTraitListener + // null, // [optional] in nsIMsgWindow aMsgWindow + // null, // [optional] in nsIJunkMailClassificationListener aJunkListener + } else { + // use the plural classifier + nsIJunkMailPlugin.classifyTraitsInMessages( + gTest.files, // in Array<ACString> aMsgURIs, + proArray, // in array aProTraits, + antiArray, // in array aAntiTraits + listener + ); // in nsIMsgTraitClassificationListener aTraitListener + // null, // [optional] in nsIMsgWindow aMsgWindow + // null, // [optional] in nsIJunkMailClassificationListener aJunkListener + } + break; + } + case kDetail: + // detail message + nsIJunkMailPlugin.detailMessage( + getSpec(gTest.fileName), // in string aMsgURI + gTest.traitIds[0], // proTrait + gTest.traitAntiIds[0], // antiTrait + listener + ); // in nsIMsgTraitDetailListener aDetailListener + break; + case kReset: + // reload a new nsIJunkMailPlugin, reading file in the process + nsIJunkMailPlugin.shutdown(); // writes files + nsIJunkMailPlugin = null; + nsIJunkMailPlugin = Cc[ + "@mozilla.org/messenger/filter-plugin;1?name=bayesianfilter" + ].createInstance(Ci.nsIJunkMailPlugin); + // does not do a callback, so we must restart next command + startCommand(); + break; + } +} diff --git a/comm/mailnews/extensions/bayesian-spam-filter/test/unit/xpcshell.ini b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/xpcshell.ini new file mode 100644 index 0000000000..86776834ba --- /dev/null +++ b/comm/mailnews/extensions/bayesian-spam-filter/test/unit/xpcshell.ini @@ -0,0 +1,11 @@ +[DEFAULT] +head = head_bayes.js +tail = +support-files = resources/* + +[test_bug228675.js] +[test_customTokenization.js] +[test_junkAsTraits.js] +[test_msgCorpus.js] +[test_traitAliases.js] +[test_traits.js] diff --git a/comm/mailnews/extensions/fts3/Normalize.c b/comm/mailnews/extensions/fts3/Normalize.c new file mode 100644 index 0000000000..984a03bd08 --- /dev/null +++ b/comm/mailnews/extensions/fts3/Normalize.c @@ -0,0 +1,2694 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* THIS FILE IS GENERATED BY generate_table.py. DON'T EDIT THIS */ + +static const unsigned short gNormalizeTable0040[] = { + /* U+0040 */ + 0x0040, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, +}; + +static const unsigned short gNormalizeTable0080[] = { + /* U+0080 */ + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x0020, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, + 0x0020, 0x00a9, 0x0061, 0x00ab, 0x00ac, 0x0020, 0x00ae, 0x0020, + 0x00b0, 0x00b1, 0x0032, 0x0033, 0x0020, 0x03bc, 0x00b6, 0x00b7, + 0x0020, 0x0031, 0x006f, 0x00bb, 0x0031, 0x0031, 0x0033, 0x00bf, +}; + +static const unsigned short gNormalizeTable00c0[] = { + /* U+00c0 */ + 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x00e6, 0x0063, + 0x0065, 0x0065, 0x0065, 0x0065, 0x0069, 0x0069, 0x0069, 0x0069, + 0x00f0, 0x006e, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x00d7, + 0x00f8, 0x0075, 0x0075, 0x0075, 0x0075, 0x0079, 0x00fe, 0x0073, + 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x00e6, 0x0063, + 0x0065, 0x0065, 0x0065, 0x0065, 0x0069, 0x0069, 0x0069, 0x0069, + 0x00f0, 0x006e, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x00f7, + 0x00f8, 0x0075, 0x0075, 0x0075, 0x0075, 0x0079, 0x00fe, 0x0079, +}; + +static const unsigned short gNormalizeTable0100[] = { + /* U+0100 */ + 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0063, 0x0063, + 0x0063, 0x0063, 0x0063, 0x0063, 0x0063, 0x0063, 0x0064, 0x0064, + 0x0111, 0x0111, 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, + 0x0065, 0x0065, 0x0065, 0x0065, 0x0067, 0x0067, 0x0067, 0x0067, + 0x0067, 0x0067, 0x0067, 0x0067, 0x0068, 0x0068, 0x0127, 0x0127, + 0x0069, 0x0069, 0x0069, 0x0069, 0x0069, 0x0069, 0x0069, 0x0069, + 0x0069, 0x0131, 0x0069, 0x0069, 0x006a, 0x006a, 0x006b, 0x006b, + 0x0138, 0x006c, 0x006c, 0x006c, 0x006c, 0x006c, 0x006c, 0x006c, +}; + +static const unsigned short gNormalizeTable0140[] = { + /* U+0140 */ + 0x006c, 0x0142, 0x0142, 0x006e, 0x006e, 0x006e, 0x006e, 0x006e, + 0x006e, 0x02bc, 0x014b, 0x014b, 0x006f, 0x006f, 0x006f, 0x006f, + 0x006f, 0x006f, 0x0153, 0x0153, 0x0072, 0x0072, 0x0072, 0x0072, + 0x0072, 0x0072, 0x0073, 0x0073, 0x0073, 0x0073, 0x0073, 0x0073, + 0x0073, 0x0073, 0x0074, 0x0074, 0x0074, 0x0074, 0x0167, 0x0167, + 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, + 0x0075, 0x0075, 0x0075, 0x0075, 0x0077, 0x0077, 0x0079, 0x0079, + 0x0079, 0x007a, 0x007a, 0x007a, 0x007a, 0x007a, 0x007a, 0x0073, +}; + +static const unsigned short gNormalizeTable0180[] = { + /* U+0180 */ + 0x0180, 0x0253, 0x0183, 0x0183, 0x0185, 0x0185, 0x0254, 0x0188, + 0x0188, 0x0256, 0x0257, 0x018c, 0x018c, 0x018d, 0x01dd, 0x0259, + 0x025b, 0x0192, 0x0192, 0x0260, 0x0263, 0x0195, 0x0269, 0x0268, + 0x0199, 0x0199, 0x019a, 0x019b, 0x026f, 0x0272, 0x019e, 0x0275, + 0x006f, 0x006f, 0x01a3, 0x01a3, 0x01a5, 0x01a5, 0x0280, 0x01a8, + 0x01a8, 0x0283, 0x01aa, 0x01ab, 0x01ad, 0x01ad, 0x0288, 0x0075, + 0x0075, 0x028a, 0x028b, 0x01b4, 0x01b4, 0x01b6, 0x01b6, 0x0292, + 0x01b9, 0x01b9, 0x01ba, 0x01bb, 0x01bd, 0x01bd, 0x01be, 0x01bf, +}; + +static const unsigned short gNormalizeTable01c0[] = { + /* U+01c0 */ + 0x01c0, 0x01c1, 0x01c2, 0x01c3, 0x0064, 0x0064, 0x0064, 0x006c, + 0x006c, 0x006c, 0x006e, 0x006e, 0x006e, 0x0061, 0x0061, 0x0069, + 0x0069, 0x006f, 0x006f, 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, + 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, 0x01dd, 0x0061, 0x0061, + 0x0061, 0x0061, 0x00e6, 0x00e6, 0x01e5, 0x01e5, 0x0067, 0x0067, + 0x006b, 0x006b, 0x006f, 0x006f, 0x006f, 0x006f, 0x0292, 0x0292, + 0x006a, 0x0064, 0x0064, 0x0064, 0x0067, 0x0067, 0x0195, 0x01bf, + 0x006e, 0x006e, 0x0061, 0x0061, 0x00e6, 0x00e6, 0x00f8, 0x00f8, +}; + +static const unsigned short gNormalizeTable0200[] = { + /* U+0200 */ + 0x0061, 0x0061, 0x0061, 0x0061, 0x0065, 0x0065, 0x0065, 0x0065, + 0x0069, 0x0069, 0x0069, 0x0069, 0x006f, 0x006f, 0x006f, 0x006f, + 0x0072, 0x0072, 0x0072, 0x0072, 0x0075, 0x0075, 0x0075, 0x0075, + 0x0073, 0x0073, 0x0074, 0x0074, 0x021d, 0x021d, 0x0068, 0x0068, + 0x019e, 0x0221, 0x0223, 0x0223, 0x0225, 0x0225, 0x0061, 0x0061, + 0x0065, 0x0065, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, + 0x006f, 0x006f, 0x0079, 0x0079, 0x0234, 0x0235, 0x0236, 0x0237, + 0x0238, 0x0239, 0x2c65, 0x023c, 0x023c, 0x019a, 0x2c66, 0x023f, +}; + +static const unsigned short gNormalizeTable0240[] = { + /* U+0240 */ + 0x0240, 0x0242, 0x0242, 0x0180, 0x0289, 0x028c, 0x0247, 0x0247, + 0x0249, 0x0249, 0x024b, 0x024b, 0x024d, 0x024d, 0x024f, 0x024f, + 0x0250, 0x0251, 0x0252, 0x0253, 0x0254, 0x0255, 0x0256, 0x0257, + 0x0258, 0x0259, 0x025a, 0x025b, 0x025c, 0x025d, 0x025e, 0x025f, + 0x0260, 0x0261, 0x0262, 0x0263, 0x0264, 0x0265, 0x0266, 0x0267, + 0x0268, 0x0269, 0x026a, 0x026b, 0x026c, 0x026d, 0x026e, 0x026f, + 0x0270, 0x0271, 0x0272, 0x0273, 0x0274, 0x0275, 0x0276, 0x0277, + 0x0278, 0x0279, 0x027a, 0x027b, 0x027c, 0x027d, 0x027e, 0x027f, +}; + +static const unsigned short gNormalizeTable0280[] = { + /* U+0280 */ + 0x0280, 0x0281, 0x0282, 0x0283, 0x0284, 0x0285, 0x0286, 0x0287, + 0x0288, 0x0289, 0x028a, 0x028b, 0x028c, 0x028d, 0x028e, 0x028f, + 0x0290, 0x0291, 0x0292, 0x0293, 0x0294, 0x0295, 0x0296, 0x0297, + 0x0298, 0x0299, 0x029a, 0x029b, 0x029c, 0x029d, 0x029e, 0x029f, + 0x02a0, 0x02a1, 0x02a2, 0x02a3, 0x02a4, 0x02a5, 0x02a6, 0x02a7, + 0x02a8, 0x02a9, 0x02aa, 0x02ab, 0x02ac, 0x02ad, 0x02ae, 0x02af, + 0x0068, 0x0266, 0x006a, 0x0072, 0x0279, 0x027b, 0x0281, 0x0077, + 0x0079, 0x02b9, 0x02ba, 0x02bb, 0x02bc, 0x02bd, 0x02be, 0x02bf, +}; + +static const unsigned short gNormalizeTable02c0[] = { + /* U+02c0 */ + 0x02c0, 0x02c1, 0x02c2, 0x02c3, 0x02c4, 0x02c5, 0x02c6, 0x02c7, + 0x02c8, 0x02c9, 0x02ca, 0x02cb, 0x02cc, 0x02cd, 0x02ce, 0x02cf, + 0x02d0, 0x02d1, 0x02d2, 0x02d3, 0x02d4, 0x02d5, 0x02d6, 0x02d7, + 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x02de, 0x02df, + 0x0263, 0x006c, 0x0073, 0x0078, 0x0295, 0x02e5, 0x02e6, 0x02e7, + 0x02e8, 0x02e9, 0x02ea, 0x02eb, 0x02ec, 0x02ed, 0x02ee, 0x02ef, + 0x02f0, 0x02f1, 0x02f2, 0x02f3, 0x02f4, 0x02f5, 0x02f6, 0x02f7, + 0x02f8, 0x02f9, 0x02fa, 0x02fb, 0x02fc, 0x02fd, 0x02fe, 0x02ff, +}; + +static const unsigned short gNormalizeTable0340[] = { + /* U+0340 */ + 0x0300, 0x0301, 0x0342, 0x0313, 0x0308, 0x03b9, 0x0346, 0x0347, + 0x0348, 0x0349, 0x034a, 0x034b, 0x034c, 0x034d, 0x034e, 0x0020, + 0x0350, 0x0351, 0x0352, 0x0353, 0x0354, 0x0355, 0x0356, 0x0357, + 0x0358, 0x0359, 0x035a, 0x035b, 0x035c, 0x035d, 0x035e, 0x035f, + 0x0360, 0x0361, 0x0362, 0x0363, 0x0364, 0x0365, 0x0366, 0x0367, + 0x0368, 0x0369, 0x036a, 0x036b, 0x036c, 0x036d, 0x036e, 0x036f, + 0x0371, 0x0371, 0x0373, 0x0373, 0x02b9, 0x0375, 0x0377, 0x0377, + 0x0378, 0x0379, 0x0020, 0x037b, 0x037c, 0x037d, 0x003b, 0x037f, +}; + +static const unsigned short gNormalizeTable0380[] = { + /* U+0380 */ + 0x0380, 0x0381, 0x0382, 0x0383, 0x0020, 0x0020, 0x03b1, 0x00b7, + 0x03b5, 0x03b7, 0x03b9, 0x038b, 0x03bf, 0x038d, 0x03c5, 0x03c9, + 0x03b9, 0x03b1, 0x03b2, 0x03b3, 0x03b4, 0x03b5, 0x03b6, 0x03b7, + 0x03b8, 0x03b9, 0x03ba, 0x03bb, 0x03bc, 0x03bd, 0x03be, 0x03bf, + 0x03c0, 0x03c1, 0x03a2, 0x03c3, 0x03c4, 0x03c5, 0x03c6, 0x03c7, + 0x03c8, 0x03c9, 0x03b9, 0x03c5, 0x03b1, 0x03b5, 0x03b7, 0x03b9, + 0x03c5, 0x03b1, 0x03b2, 0x03b3, 0x03b4, 0x03b5, 0x03b6, 0x03b7, + 0x03b8, 0x03b9, 0x03ba, 0x03bb, 0x03bc, 0x03bd, 0x03be, 0x03bf, +}; + +static const unsigned short gNormalizeTable03c0[] = { + /* U+03c0 */ + 0x03c0, 0x03c1, 0x03c3, 0x03c3, 0x03c4, 0x03c5, 0x03c6, 0x03c7, + 0x03c8, 0x03c9, 0x03b9, 0x03c5, 0x03bf, 0x03c5, 0x03c9, 0x03d7, + 0x03b2, 0x03b8, 0x03c5, 0x03c5, 0x03c5, 0x03c6, 0x03c0, 0x03d7, + 0x03d9, 0x03d9, 0x03db, 0x03db, 0x03dd, 0x03dd, 0x03df, 0x03df, + 0x03e1, 0x03e1, 0x03e3, 0x03e3, 0x03e5, 0x03e5, 0x03e7, 0x03e7, + 0x03e9, 0x03e9, 0x03eb, 0x03eb, 0x03ed, 0x03ed, 0x03ef, 0x03ef, + 0x03ba, 0x03c1, 0x03c3, 0x03f3, 0x03b8, 0x03b5, 0x03f6, 0x03f8, + 0x03f8, 0x03c3, 0x03fb, 0x03fb, 0x03fc, 0x037b, 0x037c, 0x037d, +}; + +static const unsigned short gNormalizeTable0400[] = { + /* U+0400 */ + 0x0435, 0x0435, 0x0452, 0x0433, 0x0454, 0x0455, 0x0456, 0x0456, + 0x0458, 0x0459, 0x045a, 0x045b, 0x043a, 0x0438, 0x0443, 0x045f, + 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, + 0x0438, 0x0438, 0x043a, 0x043b, 0x043c, 0x043d, 0x043e, 0x043f, + 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, + 0x0448, 0x0449, 0x044a, 0x044b, 0x044c, 0x044d, 0x044e, 0x044f, + 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, + 0x0438, 0x0438, 0x043a, 0x043b, 0x043c, 0x043d, 0x043e, 0x043f, +}; + +static const unsigned short gNormalizeTable0440[] = { + /* U+0440 */ + 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, + 0x0448, 0x0449, 0x044a, 0x044b, 0x044c, 0x044d, 0x044e, 0x044f, + 0x0435, 0x0435, 0x0452, 0x0433, 0x0454, 0x0455, 0x0456, 0x0456, + 0x0458, 0x0459, 0x045a, 0x045b, 0x043a, 0x0438, 0x0443, 0x045f, + 0x0461, 0x0461, 0x0463, 0x0463, 0x0465, 0x0465, 0x0467, 0x0467, + 0x0469, 0x0469, 0x046b, 0x046b, 0x046d, 0x046d, 0x046f, 0x046f, + 0x0471, 0x0471, 0x0473, 0x0473, 0x0475, 0x0475, 0x0475, 0x0475, + 0x0479, 0x0479, 0x047b, 0x047b, 0x047d, 0x047d, 0x047f, 0x047f, +}; + +static const unsigned short gNormalizeTable0480[] = { + /* U+0480 */ + 0x0481, 0x0481, 0x0482, 0x0483, 0x0484, 0x0485, 0x0486, 0x0487, + 0x0488, 0x0489, 0x048b, 0x048b, 0x048d, 0x048d, 0x048f, 0x048f, + 0x0491, 0x0491, 0x0493, 0x0493, 0x0495, 0x0495, 0x0497, 0x0497, + 0x0499, 0x0499, 0x049b, 0x049b, 0x049d, 0x049d, 0x049f, 0x049f, + 0x04a1, 0x04a1, 0x04a3, 0x04a3, 0x04a5, 0x04a5, 0x04a7, 0x04a7, + 0x04a9, 0x04a9, 0x04ab, 0x04ab, 0x04ad, 0x04ad, 0x04af, 0x04af, + 0x04b1, 0x04b1, 0x04b3, 0x04b3, 0x04b5, 0x04b5, 0x04b7, 0x04b7, + 0x04b9, 0x04b9, 0x04bb, 0x04bb, 0x04bd, 0x04bd, 0x04bf, 0x04bf, +}; + +static const unsigned short gNormalizeTable04c0[] = { + /* U+04c0 */ + 0x04cf, 0x0436, 0x0436, 0x04c4, 0x04c4, 0x04c6, 0x04c6, 0x04c8, + 0x04c8, 0x04ca, 0x04ca, 0x04cc, 0x04cc, 0x04ce, 0x04ce, 0x04cf, + 0x0430, 0x0430, 0x0430, 0x0430, 0x04d5, 0x04d5, 0x0435, 0x0435, + 0x04d9, 0x04d9, 0x04d9, 0x04d9, 0x0436, 0x0436, 0x0437, 0x0437, + 0x04e1, 0x04e1, 0x0438, 0x0438, 0x0438, 0x0438, 0x043e, 0x043e, + 0x04e9, 0x04e9, 0x04e9, 0x04e9, 0x044d, 0x044d, 0x0443, 0x0443, + 0x0443, 0x0443, 0x0443, 0x0443, 0x0447, 0x0447, 0x04f7, 0x04f7, + 0x044b, 0x044b, 0x04fb, 0x04fb, 0x04fd, 0x04fd, 0x04ff, 0x04ff, +}; + +static const unsigned short gNormalizeTable0500[] = { + /* U+0500 */ + 0x0501, 0x0501, 0x0503, 0x0503, 0x0505, 0x0505, 0x0507, 0x0507, + 0x0509, 0x0509, 0x050b, 0x050b, 0x050d, 0x050d, 0x050f, 0x050f, + 0x0511, 0x0511, 0x0513, 0x0513, 0x0515, 0x0515, 0x0517, 0x0517, + 0x0519, 0x0519, 0x051b, 0x051b, 0x051d, 0x051d, 0x051f, 0x051f, + 0x0521, 0x0521, 0x0523, 0x0523, 0x0525, 0x0525, 0x0526, 0x0527, + 0x0528, 0x0529, 0x052a, 0x052b, 0x052c, 0x052d, 0x052e, 0x052f, + 0x0530, 0x0561, 0x0562, 0x0563, 0x0564, 0x0565, 0x0566, 0x0567, + 0x0568, 0x0569, 0x056a, 0x056b, 0x056c, 0x056d, 0x056e, 0x056f, +}; + +static const unsigned short gNormalizeTable0540[] = { + /* U+0540 */ + 0x0570, 0x0571, 0x0572, 0x0573, 0x0574, 0x0575, 0x0576, 0x0577, + 0x0578, 0x0579, 0x057a, 0x057b, 0x057c, 0x057d, 0x057e, 0x057f, + 0x0580, 0x0581, 0x0582, 0x0583, 0x0584, 0x0585, 0x0586, 0x0557, + 0x0558, 0x0559, 0x055a, 0x055b, 0x055c, 0x055d, 0x055e, 0x055f, + 0x0560, 0x0561, 0x0562, 0x0563, 0x0564, 0x0565, 0x0566, 0x0567, + 0x0568, 0x0569, 0x056a, 0x056b, 0x056c, 0x056d, 0x056e, 0x056f, + 0x0570, 0x0571, 0x0572, 0x0573, 0x0574, 0x0575, 0x0576, 0x0577, + 0x0578, 0x0579, 0x057a, 0x057b, 0x057c, 0x057d, 0x057e, 0x057f, +}; + +static const unsigned short gNormalizeTable0580[] = { + /* U+0580 */ + 0x0580, 0x0581, 0x0582, 0x0583, 0x0584, 0x0585, 0x0586, 0x0565, + 0x0588, 0x0589, 0x058a, 0x058b, 0x058c, 0x058d, 0x058e, 0x058f, + 0x0590, 0x0591, 0x0592, 0x0593, 0x0594, 0x0595, 0x0596, 0x0597, + 0x0598, 0x0599, 0x059a, 0x059b, 0x059c, 0x059d, 0x059e, 0x059f, + 0x05a0, 0x05a1, 0x05a2, 0x05a3, 0x05a4, 0x05a5, 0x05a6, 0x05a7, + 0x05a8, 0x05a9, 0x05aa, 0x05ab, 0x05ac, 0x05ad, 0x05ae, 0x05af, + 0x05b0, 0x05b1, 0x05b2, 0x05b3, 0x05b4, 0x05b5, 0x05b6, 0x05b7, + 0x05b8, 0x05b9, 0x05ba, 0x05bb, 0x05bc, 0x05bd, 0x05be, 0x05bf, +}; + +static const unsigned short gNormalizeTable0600[] = { + /* U+0600 */ + 0x0600, 0x0601, 0x0602, 0x0603, 0x0604, 0x0605, 0x0606, 0x0607, + 0x0608, 0x0609, 0x060a, 0x060b, 0x060c, 0x060d, 0x060e, 0x060f, + 0x0610, 0x0611, 0x0612, 0x0613, 0x0614, 0x0615, 0x0616, 0x0617, + 0x0618, 0x0619, 0x061a, 0x061b, 0x061c, 0x061d, 0x061e, 0x061f, + 0x0620, 0x0621, 0x0627, 0x0627, 0x0648, 0x0627, 0x064a, 0x0627, + 0x0628, 0x0629, 0x062a, 0x062b, 0x062c, 0x062d, 0x062e, 0x062f, + 0x0630, 0x0631, 0x0632, 0x0633, 0x0634, 0x0635, 0x0636, 0x0637, + 0x0638, 0x0639, 0x063a, 0x063b, 0x063c, 0x063d, 0x063e, 0x063f, +}; + +static const unsigned short gNormalizeTable0640[] = { + /* U+0640 */ + 0x0640, 0x0641, 0x0642, 0x0643, 0x0644, 0x0645, 0x0646, 0x0647, + 0x0648, 0x0649, 0x064a, 0x064b, 0x064c, 0x064d, 0x064e, 0x064f, + 0x0650, 0x0651, 0x0652, 0x0653, 0x0654, 0x0655, 0x0656, 0x0657, + 0x0658, 0x0659, 0x065a, 0x065b, 0x065c, 0x065d, 0x065e, 0x065f, + 0x0660, 0x0661, 0x0662, 0x0663, 0x0664, 0x0665, 0x0666, 0x0667, + 0x0668, 0x0669, 0x066a, 0x066b, 0x066c, 0x066d, 0x066e, 0x066f, + 0x0670, 0x0671, 0x0672, 0x0673, 0x0674, 0x0627, 0x0648, 0x06c7, + 0x064a, 0x0679, 0x067a, 0x067b, 0x067c, 0x067d, 0x067e, 0x067f, +}; + +static const unsigned short gNormalizeTable06c0[] = { + /* U+06c0 */ + 0x06d5, 0x06c1, 0x06c1, 0x06c3, 0x06c4, 0x06c5, 0x06c6, 0x06c7, + 0x06c8, 0x06c9, 0x06ca, 0x06cb, 0x06cc, 0x06cd, 0x06ce, 0x06cf, + 0x06d0, 0x06d1, 0x06d2, 0x06d2, 0x06d4, 0x06d5, 0x06d6, 0x06d7, + 0x06d8, 0x06d9, 0x06da, 0x06db, 0x06dc, 0x06dd, 0x06de, 0x06df, + 0x06e0, 0x06e1, 0x06e2, 0x06e3, 0x06e4, 0x06e5, 0x06e6, 0x06e7, + 0x06e8, 0x06e9, 0x06ea, 0x06eb, 0x06ec, 0x06ed, 0x06ee, 0x06ef, + 0x06f0, 0x06f1, 0x06f2, 0x06f3, 0x06f4, 0x06f5, 0x06f6, 0x06f7, + 0x06f8, 0x06f9, 0x06fa, 0x06fb, 0x06fc, 0x06fd, 0x06fe, 0x06ff, +}; + +static const unsigned short gNormalizeTable0900[] = { + /* U+0900 */ + 0x0900, 0x0901, 0x0902, 0x0903, 0x0904, 0x0905, 0x0906, 0x0907, + 0x0908, 0x0909, 0x090a, 0x090b, 0x090c, 0x090d, 0x090e, 0x090f, + 0x0910, 0x0911, 0x0912, 0x0913, 0x0914, 0x0915, 0x0916, 0x0917, + 0x0918, 0x0919, 0x091a, 0x091b, 0x091c, 0x091d, 0x091e, 0x091f, + 0x0920, 0x0921, 0x0922, 0x0923, 0x0924, 0x0925, 0x0926, 0x0927, + 0x0928, 0x0928, 0x092a, 0x092b, 0x092c, 0x092d, 0x092e, 0x092f, + 0x0930, 0x0930, 0x0932, 0x0933, 0x0933, 0x0935, 0x0936, 0x0937, + 0x0938, 0x0939, 0x093a, 0x093b, 0x093c, 0x093d, 0x093e, 0x093f, +}; + +static const unsigned short gNormalizeTable0940[] = { + /* U+0940 */ + 0x0940, 0x0941, 0x0942, 0x0943, 0x0944, 0x0945, 0x0946, 0x0947, + 0x0948, 0x0949, 0x094a, 0x094b, 0x094c, 0x094d, 0x094e, 0x094f, + 0x0950, 0x0951, 0x0952, 0x0953, 0x0954, 0x0955, 0x0956, 0x0957, + 0x0915, 0x0916, 0x0917, 0x091c, 0x0921, 0x0922, 0x092b, 0x092f, + 0x0960, 0x0961, 0x0962, 0x0963, 0x0964, 0x0965, 0x0966, 0x0967, + 0x0968, 0x0969, 0x096a, 0x096b, 0x096c, 0x096d, 0x096e, 0x096f, + 0x0970, 0x0971, 0x0972, 0x0973, 0x0974, 0x0975, 0x0976, 0x0977, + 0x0978, 0x0979, 0x097a, 0x097b, 0x097c, 0x097d, 0x097e, 0x097f, +}; + +static const unsigned short gNormalizeTable09c0[] = { + /* U+09c0 */ + 0x09c0, 0x09c1, 0x09c2, 0x09c3, 0x09c4, 0x09c5, 0x09c6, 0x09c7, + 0x09c8, 0x09c9, 0x09ca, 0x09c7, 0x09c7, 0x09cd, 0x09ce, 0x09cf, + 0x09d0, 0x09d1, 0x09d2, 0x09d3, 0x09d4, 0x09d5, 0x09d6, 0x09d7, + 0x09d8, 0x09d9, 0x09da, 0x09db, 0x09a1, 0x09a2, 0x09de, 0x09af, + 0x09e0, 0x09e1, 0x09e2, 0x09e3, 0x09e4, 0x09e5, 0x09e6, 0x09e7, + 0x09e8, 0x09e9, 0x09ea, 0x09eb, 0x09ec, 0x09ed, 0x09ee, 0x09ef, + 0x09f0, 0x09f1, 0x09f2, 0x09f3, 0x09f4, 0x09f5, 0x09f6, 0x09f7, + 0x09f8, 0x09f9, 0x09fa, 0x09fb, 0x09fc, 0x09fd, 0x09fe, 0x09ff, +}; + +static const unsigned short gNormalizeTable0a00[] = { + /* U+0a00 */ + 0x0a00, 0x0a01, 0x0a02, 0x0a03, 0x0a04, 0x0a05, 0x0a06, 0x0a07, + 0x0a08, 0x0a09, 0x0a0a, 0x0a0b, 0x0a0c, 0x0a0d, 0x0a0e, 0x0a0f, + 0x0a10, 0x0a11, 0x0a12, 0x0a13, 0x0a14, 0x0a15, 0x0a16, 0x0a17, + 0x0a18, 0x0a19, 0x0a1a, 0x0a1b, 0x0a1c, 0x0a1d, 0x0a1e, 0x0a1f, + 0x0a20, 0x0a21, 0x0a22, 0x0a23, 0x0a24, 0x0a25, 0x0a26, 0x0a27, + 0x0a28, 0x0a29, 0x0a2a, 0x0a2b, 0x0a2c, 0x0a2d, 0x0a2e, 0x0a2f, + 0x0a30, 0x0a31, 0x0a32, 0x0a32, 0x0a34, 0x0a35, 0x0a38, 0x0a37, + 0x0a38, 0x0a39, 0x0a3a, 0x0a3b, 0x0a3c, 0x0a3d, 0x0a3e, 0x0a3f, +}; + +static const unsigned short gNormalizeTable0a40[] = { + /* U+0a40 */ + 0x0a40, 0x0a41, 0x0a42, 0x0a43, 0x0a44, 0x0a45, 0x0a46, 0x0a47, + 0x0a48, 0x0a49, 0x0a4a, 0x0a4b, 0x0a4c, 0x0a4d, 0x0a4e, 0x0a4f, + 0x0a50, 0x0a51, 0x0a52, 0x0a53, 0x0a54, 0x0a55, 0x0a56, 0x0a57, + 0x0a58, 0x0a16, 0x0a17, 0x0a1c, 0x0a5c, 0x0a5d, 0x0a2b, 0x0a5f, + 0x0a60, 0x0a61, 0x0a62, 0x0a63, 0x0a64, 0x0a65, 0x0a66, 0x0a67, + 0x0a68, 0x0a69, 0x0a6a, 0x0a6b, 0x0a6c, 0x0a6d, 0x0a6e, 0x0a6f, + 0x0a70, 0x0a71, 0x0a72, 0x0a73, 0x0a74, 0x0a75, 0x0a76, 0x0a77, + 0x0a78, 0x0a79, 0x0a7a, 0x0a7b, 0x0a7c, 0x0a7d, 0x0a7e, 0x0a7f, +}; + +static const unsigned short gNormalizeTable0b40[] = { + /* U+0b40 */ + 0x0b40, 0x0b41, 0x0b42, 0x0b43, 0x0b44, 0x0b45, 0x0b46, 0x0b47, + 0x0b47, 0x0b49, 0x0b4a, 0x0b47, 0x0b47, 0x0b4d, 0x0b4e, 0x0b4f, + 0x0b50, 0x0b51, 0x0b52, 0x0b53, 0x0b54, 0x0b55, 0x0b56, 0x0b57, + 0x0b58, 0x0b59, 0x0b5a, 0x0b5b, 0x0b21, 0x0b22, 0x0b5e, 0x0b5f, + 0x0b60, 0x0b61, 0x0b62, 0x0b63, 0x0b64, 0x0b65, 0x0b66, 0x0b67, + 0x0b68, 0x0b69, 0x0b6a, 0x0b6b, 0x0b6c, 0x0b6d, 0x0b6e, 0x0b6f, + 0x0b70, 0x0b71, 0x0b72, 0x0b73, 0x0b74, 0x0b75, 0x0b76, 0x0b77, + 0x0b78, 0x0b79, 0x0b7a, 0x0b7b, 0x0b7c, 0x0b7d, 0x0b7e, 0x0b7f, +}; + +static const unsigned short gNormalizeTable0b80[] = { + /* U+0b80 */ + 0x0b80, 0x0b81, 0x0b82, 0x0b83, 0x0b84, 0x0b85, 0x0b86, 0x0b87, + 0x0b88, 0x0b89, 0x0b8a, 0x0b8b, 0x0b8c, 0x0b8d, 0x0b8e, 0x0b8f, + 0x0b90, 0x0b91, 0x0b92, 0x0b93, 0x0b92, 0x0b95, 0x0b96, 0x0b97, + 0x0b98, 0x0b99, 0x0b9a, 0x0b9b, 0x0b9c, 0x0b9d, 0x0b9e, 0x0b9f, + 0x0ba0, 0x0ba1, 0x0ba2, 0x0ba3, 0x0ba4, 0x0ba5, 0x0ba6, 0x0ba7, + 0x0ba8, 0x0ba9, 0x0baa, 0x0bab, 0x0bac, 0x0bad, 0x0bae, 0x0baf, + 0x0bb0, 0x0bb1, 0x0bb2, 0x0bb3, 0x0bb4, 0x0bb5, 0x0bb6, 0x0bb7, + 0x0bb8, 0x0bb9, 0x0bba, 0x0bbb, 0x0bbc, 0x0bbd, 0x0bbe, 0x0bbf, +}; + +static const unsigned short gNormalizeTable0bc0[] = { + /* U+0bc0 */ + 0x0bc0, 0x0bc1, 0x0bc2, 0x0bc3, 0x0bc4, 0x0bc5, 0x0bc6, 0x0bc7, + 0x0bc8, 0x0bc9, 0x0bc6, 0x0bc7, 0x0bc6, 0x0bcd, 0x0bce, 0x0bcf, + 0x0bd0, 0x0bd1, 0x0bd2, 0x0bd3, 0x0bd4, 0x0bd5, 0x0bd6, 0x0bd7, + 0x0bd8, 0x0bd9, 0x0bda, 0x0bdb, 0x0bdc, 0x0bdd, 0x0bde, 0x0bdf, + 0x0be0, 0x0be1, 0x0be2, 0x0be3, 0x0be4, 0x0be5, 0x0be6, 0x0be7, + 0x0be8, 0x0be9, 0x0bea, 0x0beb, 0x0bec, 0x0bed, 0x0bee, 0x0bef, + 0x0bf0, 0x0bf1, 0x0bf2, 0x0bf3, 0x0bf4, 0x0bf5, 0x0bf6, 0x0bf7, + 0x0bf8, 0x0bf9, 0x0bfa, 0x0bfb, 0x0bfc, 0x0bfd, 0x0bfe, 0x0bff, +}; + +static const unsigned short gNormalizeTable0c40[] = { + /* U+0c40 */ + 0x0c40, 0x0c41, 0x0c42, 0x0c43, 0x0c44, 0x0c45, 0x0c46, 0x0c47, + 0x0c46, 0x0c49, 0x0c4a, 0x0c4b, 0x0c4c, 0x0c4d, 0x0c4e, 0x0c4f, + 0x0c50, 0x0c51, 0x0c52, 0x0c53, 0x0c54, 0x0c55, 0x0c56, 0x0c57, + 0x0c58, 0x0c59, 0x0c5a, 0x0c5b, 0x0c5c, 0x0c5d, 0x0c5e, 0x0c5f, + 0x0c60, 0x0c61, 0x0c62, 0x0c63, 0x0c64, 0x0c65, 0x0c66, 0x0c67, + 0x0c68, 0x0c69, 0x0c6a, 0x0c6b, 0x0c6c, 0x0c6d, 0x0c6e, 0x0c6f, + 0x0c70, 0x0c71, 0x0c72, 0x0c73, 0x0c74, 0x0c75, 0x0c76, 0x0c77, + 0x0c78, 0x0c79, 0x0c7a, 0x0c7b, 0x0c7c, 0x0c7d, 0x0c7e, 0x0c7f, +}; + +static const unsigned short gNormalizeTable0cc0[] = { + /* U+0cc0 */ + 0x0cbf, 0x0cc1, 0x0cc2, 0x0cc3, 0x0cc4, 0x0cc5, 0x0cc6, 0x0cc6, + 0x0cc6, 0x0cc9, 0x0cc6, 0x0cc6, 0x0ccc, 0x0ccd, 0x0cce, 0x0ccf, + 0x0cd0, 0x0cd1, 0x0cd2, 0x0cd3, 0x0cd4, 0x0cd5, 0x0cd6, 0x0cd7, + 0x0cd8, 0x0cd9, 0x0cda, 0x0cdb, 0x0cdc, 0x0cdd, 0x0cde, 0x0cdf, + 0x0ce0, 0x0ce1, 0x0ce2, 0x0ce3, 0x0ce4, 0x0ce5, 0x0ce6, 0x0ce7, + 0x0ce8, 0x0ce9, 0x0cea, 0x0ceb, 0x0cec, 0x0ced, 0x0cee, 0x0cef, + 0x0cf0, 0x0cf1, 0x0cf2, 0x0cf3, 0x0cf4, 0x0cf5, 0x0cf6, 0x0cf7, + 0x0cf8, 0x0cf9, 0x0cfa, 0x0cfb, 0x0cfc, 0x0cfd, 0x0cfe, 0x0cff, +}; + +static const unsigned short gNormalizeTable0d40[] = { + /* U+0d40 */ + 0x0d40, 0x0d41, 0x0d42, 0x0d43, 0x0d44, 0x0d45, 0x0d46, 0x0d47, + 0x0d48, 0x0d49, 0x0d46, 0x0d47, 0x0d46, 0x0d4d, 0x0d4e, 0x0d4f, + 0x0d50, 0x0d51, 0x0d52, 0x0d53, 0x0d54, 0x0d55, 0x0d56, 0x0d57, + 0x0d58, 0x0d59, 0x0d5a, 0x0d5b, 0x0d5c, 0x0d5d, 0x0d5e, 0x0d5f, + 0x0d60, 0x0d61, 0x0d62, 0x0d63, 0x0d64, 0x0d65, 0x0d66, 0x0d67, + 0x0d68, 0x0d69, 0x0d6a, 0x0d6b, 0x0d6c, 0x0d6d, 0x0d6e, 0x0d6f, + 0x0d70, 0x0d71, 0x0d72, 0x0d73, 0x0d74, 0x0d75, 0x0d76, 0x0d77, + 0x0d78, 0x0d79, 0x0d7a, 0x0d7b, 0x0d7c, 0x0d7d, 0x0d7e, 0x0d7f, +}; + +static const unsigned short gNormalizeTable0dc0[] = { + /* U+0dc0 */ + 0x0dc0, 0x0dc1, 0x0dc2, 0x0dc3, 0x0dc4, 0x0dc5, 0x0dc6, 0x0dc7, + 0x0dc8, 0x0dc9, 0x0dca, 0x0dcb, 0x0dcc, 0x0dcd, 0x0dce, 0x0dcf, + 0x0dd0, 0x0dd1, 0x0dd2, 0x0dd3, 0x0dd4, 0x0dd5, 0x0dd6, 0x0dd7, + 0x0dd8, 0x0dd9, 0x0dd9, 0x0ddb, 0x0dd9, 0x0dd9, 0x0dd9, 0x0ddf, + 0x0de0, 0x0de1, 0x0de2, 0x0de3, 0x0de4, 0x0de5, 0x0de6, 0x0de7, + 0x0de8, 0x0de9, 0x0dea, 0x0deb, 0x0dec, 0x0ded, 0x0dee, 0x0def, + 0x0df0, 0x0df1, 0x0df2, 0x0df3, 0x0df4, 0x0df5, 0x0df6, 0x0df7, + 0x0df8, 0x0df9, 0x0dfa, 0x0dfb, 0x0dfc, 0x0dfd, 0x0dfe, 0x0dff, +}; + +static const unsigned short gNormalizeTable0e00[] = { + /* U+0e00 */ + 0x0e00, 0x0e01, 0x0e02, 0x0e03, 0x0e04, 0x0e05, 0x0e06, 0x0e07, + 0x0e08, 0x0e09, 0x0e0a, 0x0e0b, 0x0e0c, 0x0e0d, 0x0e0e, 0x0e0f, + 0x0e10, 0x0e11, 0x0e12, 0x0e13, 0x0e14, 0x0e15, 0x0e16, 0x0e17, + 0x0e18, 0x0e19, 0x0e1a, 0x0e1b, 0x0e1c, 0x0e1d, 0x0e1e, 0x0e1f, + 0x0e20, 0x0e21, 0x0e22, 0x0e23, 0x0e24, 0x0e25, 0x0e26, 0x0e27, + 0x0e28, 0x0e29, 0x0e2a, 0x0e2b, 0x0e2c, 0x0e2d, 0x0e2e, 0x0e2f, + 0x0e30, 0x0e31, 0x0e32, 0x0e4d, 0x0e34, 0x0e35, 0x0e36, 0x0e37, + 0x0e38, 0x0e39, 0x0e3a, 0x0e3b, 0x0e3c, 0x0e3d, 0x0e3e, 0x0e3f, +}; + +static const unsigned short gNormalizeTable0e80[] = { + /* U+0e80 */ + 0x0e80, 0x0e81, 0x0e82, 0x0e83, 0x0e84, 0x0e85, 0x0e86, 0x0e87, + 0x0e88, 0x0e89, 0x0e8a, 0x0e8b, 0x0e8c, 0x0e8d, 0x0e8e, 0x0e8f, + 0x0e90, 0x0e91, 0x0e92, 0x0e93, 0x0e94, 0x0e95, 0x0e96, 0x0e97, + 0x0e98, 0x0e99, 0x0e9a, 0x0e9b, 0x0e9c, 0x0e9d, 0x0e9e, 0x0e9f, + 0x0ea0, 0x0ea1, 0x0ea2, 0x0ea3, 0x0ea4, 0x0ea5, 0x0ea6, 0x0ea7, + 0x0ea8, 0x0ea9, 0x0eaa, 0x0eab, 0x0eac, 0x0ead, 0x0eae, 0x0eaf, + 0x0eb0, 0x0eb1, 0x0eb2, 0x0ecd, 0x0eb4, 0x0eb5, 0x0eb6, 0x0eb7, + 0x0eb8, 0x0eb9, 0x0eba, 0x0ebb, 0x0ebc, 0x0ebd, 0x0ebe, 0x0ebf, +}; + +static const unsigned short gNormalizeTable0ec0[] = { + /* U+0ec0 */ + 0x0ec0, 0x0ec1, 0x0ec2, 0x0ec3, 0x0ec4, 0x0ec5, 0x0ec6, 0x0ec7, + 0x0ec8, 0x0ec9, 0x0eca, 0x0ecb, 0x0ecc, 0x0ecd, 0x0ece, 0x0ecf, + 0x0ed0, 0x0ed1, 0x0ed2, 0x0ed3, 0x0ed4, 0x0ed5, 0x0ed6, 0x0ed7, + 0x0ed8, 0x0ed9, 0x0eda, 0x0edb, 0x0eab, 0x0eab, 0x0ede, 0x0edf, + 0x0ee0, 0x0ee1, 0x0ee2, 0x0ee3, 0x0ee4, 0x0ee5, 0x0ee6, 0x0ee7, + 0x0ee8, 0x0ee9, 0x0eea, 0x0eeb, 0x0eec, 0x0eed, 0x0eee, 0x0eef, + 0x0ef0, 0x0ef1, 0x0ef2, 0x0ef3, 0x0ef4, 0x0ef5, 0x0ef6, 0x0ef7, + 0x0ef8, 0x0ef9, 0x0efa, 0x0efb, 0x0efc, 0x0efd, 0x0efe, 0x0eff, +}; + +static const unsigned short gNormalizeTable0f00[] = { + /* U+0f00 */ + 0x0f00, 0x0f01, 0x0f02, 0x0f03, 0x0f04, 0x0f05, 0x0f06, 0x0f07, + 0x0f08, 0x0f09, 0x0f0a, 0x0f0b, 0x0f0b, 0x0f0d, 0x0f0e, 0x0f0f, + 0x0f10, 0x0f11, 0x0f12, 0x0f13, 0x0f14, 0x0f15, 0x0f16, 0x0f17, + 0x0f18, 0x0f19, 0x0f1a, 0x0f1b, 0x0f1c, 0x0f1d, 0x0f1e, 0x0f1f, + 0x0f20, 0x0f21, 0x0f22, 0x0f23, 0x0f24, 0x0f25, 0x0f26, 0x0f27, + 0x0f28, 0x0f29, 0x0f2a, 0x0f2b, 0x0f2c, 0x0f2d, 0x0f2e, 0x0f2f, + 0x0f30, 0x0f31, 0x0f32, 0x0f33, 0x0f34, 0x0f35, 0x0f36, 0x0f37, + 0x0f38, 0x0f39, 0x0f3a, 0x0f3b, 0x0f3c, 0x0f3d, 0x0f3e, 0x0f3f, +}; + +static const unsigned short gNormalizeTable0f40[] = { + /* U+0f40 */ + 0x0f40, 0x0f41, 0x0f42, 0x0f42, 0x0f44, 0x0f45, 0x0f46, 0x0f47, + 0x0f48, 0x0f49, 0x0f4a, 0x0f4b, 0x0f4c, 0x0f4c, 0x0f4e, 0x0f4f, + 0x0f50, 0x0f51, 0x0f51, 0x0f53, 0x0f54, 0x0f55, 0x0f56, 0x0f56, + 0x0f58, 0x0f59, 0x0f5a, 0x0f5b, 0x0f5b, 0x0f5d, 0x0f5e, 0x0f5f, + 0x0f60, 0x0f61, 0x0f62, 0x0f63, 0x0f64, 0x0f65, 0x0f66, 0x0f67, + 0x0f68, 0x0f40, 0x0f6a, 0x0f6b, 0x0f6c, 0x0f6d, 0x0f6e, 0x0f6f, + 0x0f70, 0x0f71, 0x0f72, 0x0f71, 0x0f74, 0x0f71, 0x0fb2, 0x0fb2, + 0x0fb3, 0x0fb3, 0x0f7a, 0x0f7b, 0x0f7c, 0x0f7d, 0x0f7e, 0x0f7f, +}; + +static const unsigned short gNormalizeTable0f80[] = { + /* U+0f80 */ + 0x0f80, 0x0f71, 0x0f82, 0x0f83, 0x0f84, 0x0f85, 0x0f86, 0x0f87, + 0x0f88, 0x0f89, 0x0f8a, 0x0f8b, 0x0f8c, 0x0f8d, 0x0f8e, 0x0f8f, + 0x0f90, 0x0f91, 0x0f92, 0x0f92, 0x0f94, 0x0f95, 0x0f96, 0x0f97, + 0x0f98, 0x0f99, 0x0f9a, 0x0f9b, 0x0f9c, 0x0f9c, 0x0f9e, 0x0f9f, + 0x0fa0, 0x0fa1, 0x0fa1, 0x0fa3, 0x0fa4, 0x0fa5, 0x0fa6, 0x0fa6, + 0x0fa8, 0x0fa9, 0x0faa, 0x0fab, 0x0fab, 0x0fad, 0x0fae, 0x0faf, + 0x0fb0, 0x0fb1, 0x0fb2, 0x0fb3, 0x0fb4, 0x0fb5, 0x0fb6, 0x0fb7, + 0x0fb8, 0x0f90, 0x0fba, 0x0fbb, 0x0fbc, 0x0fbd, 0x0fbe, 0x0fbf, +}; + +static const unsigned short gNormalizeTable1000[] = { + /* U+1000 */ + 0x1000, 0x1001, 0x1002, 0x1003, 0x1004, 0x1005, 0x1006, 0x1007, + 0x1008, 0x1009, 0x100a, 0x100b, 0x100c, 0x100d, 0x100e, 0x100f, + 0x1010, 0x1011, 0x1012, 0x1013, 0x1014, 0x1015, 0x1016, 0x1017, + 0x1018, 0x1019, 0x101a, 0x101b, 0x101c, 0x101d, 0x101e, 0x101f, + 0x1020, 0x1021, 0x1022, 0x1023, 0x1024, 0x1025, 0x1025, 0x1027, + 0x1028, 0x1029, 0x102a, 0x102b, 0x102c, 0x102d, 0x102e, 0x102f, + 0x1030, 0x1031, 0x1032, 0x1033, 0x1034, 0x1035, 0x1036, 0x1037, + 0x1038, 0x1039, 0x103a, 0x103b, 0x103c, 0x103d, 0x103e, 0x103f, +}; + +static const unsigned short gNormalizeTable1080[] = { + /* U+1080 */ + 0x1080, 0x1081, 0x1082, 0x1083, 0x1084, 0x1085, 0x1086, 0x1087, + 0x1088, 0x1089, 0x108a, 0x108b, 0x108c, 0x108d, 0x108e, 0x108f, + 0x1090, 0x1091, 0x1092, 0x1093, 0x1094, 0x1095, 0x1096, 0x1097, + 0x1098, 0x1099, 0x109a, 0x109b, 0x109c, 0x109d, 0x109e, 0x109f, + 0x2d00, 0x2d01, 0x2d02, 0x2d03, 0x2d04, 0x2d05, 0x2d06, 0x2d07, + 0x2d08, 0x2d09, 0x2d0a, 0x2d0b, 0x2d0c, 0x2d0d, 0x2d0e, 0x2d0f, + 0x2d10, 0x2d11, 0x2d12, 0x2d13, 0x2d14, 0x2d15, 0x2d16, 0x2d17, + 0x2d18, 0x2d19, 0x2d1a, 0x2d1b, 0x2d1c, 0x2d1d, 0x2d1e, 0x2d1f, +}; + +static const unsigned short gNormalizeTable10c0[] = { + /* U+10c0 */ + 0x2d20, 0x2d21, 0x2d22, 0x2d23, 0x2d24, 0x2d25, 0x10c6, 0x10c7, + 0x10c8, 0x10c9, 0x10ca, 0x10cb, 0x10cc, 0x10cd, 0x10ce, 0x10cf, + 0x10d0, 0x10d1, 0x10d2, 0x10d3, 0x10d4, 0x10d5, 0x10d6, 0x10d7, + 0x10d8, 0x10d9, 0x10da, 0x10db, 0x10dc, 0x10dd, 0x10de, 0x10df, + 0x10e0, 0x10e1, 0x10e2, 0x10e3, 0x10e4, 0x10e5, 0x10e6, 0x10e7, + 0x10e8, 0x10e9, 0x10ea, 0x10eb, 0x10ec, 0x10ed, 0x10ee, 0x10ef, + 0x10f0, 0x10f1, 0x10f2, 0x10f3, 0x10f4, 0x10f5, 0x10f6, 0x10f7, + 0x10f8, 0x10f9, 0x10fa, 0x10fb, 0x10dc, 0x10fd, 0x10fe, 0x10ff, +}; + +static const unsigned short gNormalizeTable1140[] = { + /* U+1140 */ + 0x1140, 0x1141, 0x1142, 0x1143, 0x1144, 0x1145, 0x1146, 0x1147, + 0x1148, 0x1149, 0x114a, 0x114b, 0x114c, 0x114d, 0x114e, 0x114f, + 0x1150, 0x1151, 0x1152, 0x1153, 0x1154, 0x1155, 0x1156, 0x1157, + 0x1158, 0x1159, 0x115a, 0x115b, 0x115c, 0x115d, 0x115e, 0x0020, + 0x0020, 0x1161, 0x1162, 0x1163, 0x1164, 0x1165, 0x1166, 0x1167, + 0x1168, 0x1169, 0x116a, 0x116b, 0x116c, 0x116d, 0x116e, 0x116f, + 0x1170, 0x1171, 0x1172, 0x1173, 0x1174, 0x1175, 0x1176, 0x1177, + 0x1178, 0x1179, 0x117a, 0x117b, 0x117c, 0x117d, 0x117e, 0x117f, +}; + +static const unsigned short gNormalizeTable1780[] = { + /* U+1780 */ + 0x1780, 0x1781, 0x1782, 0x1783, 0x1784, 0x1785, 0x1786, 0x1787, + 0x1788, 0x1789, 0x178a, 0x178b, 0x178c, 0x178d, 0x178e, 0x178f, + 0x1790, 0x1791, 0x1792, 0x1793, 0x1794, 0x1795, 0x1796, 0x1797, + 0x1798, 0x1799, 0x179a, 0x179b, 0x179c, 0x179d, 0x179e, 0x179f, + 0x17a0, 0x17a1, 0x17a2, 0x17a3, 0x17a4, 0x17a5, 0x17a6, 0x17a7, + 0x17a8, 0x17a9, 0x17aa, 0x17ab, 0x17ac, 0x17ad, 0x17ae, 0x17af, + 0x17b0, 0x17b1, 0x17b2, 0x17b3, 0x0020, 0x0020, 0x17b6, 0x17b7, + 0x17b8, 0x17b9, 0x17ba, 0x17bb, 0x17bc, 0x17bd, 0x17be, 0x17bf, +}; + +static const unsigned short gNormalizeTable1800[] = { + /* U+1800 */ + 0x1800, 0x1801, 0x1802, 0x1803, 0x1804, 0x1805, 0x1806, 0x1807, + 0x1808, 0x1809, 0x180a, 0x0020, 0x0020, 0x0020, 0x180e, 0x180f, + 0x1810, 0x1811, 0x1812, 0x1813, 0x1814, 0x1815, 0x1816, 0x1817, + 0x1818, 0x1819, 0x181a, 0x181b, 0x181c, 0x181d, 0x181e, 0x181f, + 0x1820, 0x1821, 0x1822, 0x1823, 0x1824, 0x1825, 0x1826, 0x1827, + 0x1828, 0x1829, 0x182a, 0x182b, 0x182c, 0x182d, 0x182e, 0x182f, + 0x1830, 0x1831, 0x1832, 0x1833, 0x1834, 0x1835, 0x1836, 0x1837, + 0x1838, 0x1839, 0x183a, 0x183b, 0x183c, 0x183d, 0x183e, 0x183f, +}; + +static const unsigned short gNormalizeTable1b00[] = { + /* U+1b00 */ + 0x1b00, 0x1b01, 0x1b02, 0x1b03, 0x1b04, 0x1b05, 0x1b05, 0x1b07, + 0x1b07, 0x1b09, 0x1b09, 0x1b0b, 0x1b0b, 0x1b0d, 0x1b0d, 0x1b0f, + 0x1b10, 0x1b11, 0x1b11, 0x1b13, 0x1b14, 0x1b15, 0x1b16, 0x1b17, + 0x1b18, 0x1b19, 0x1b1a, 0x1b1b, 0x1b1c, 0x1b1d, 0x1b1e, 0x1b1f, + 0x1b20, 0x1b21, 0x1b22, 0x1b23, 0x1b24, 0x1b25, 0x1b26, 0x1b27, + 0x1b28, 0x1b29, 0x1b2a, 0x1b2b, 0x1b2c, 0x1b2d, 0x1b2e, 0x1b2f, + 0x1b30, 0x1b31, 0x1b32, 0x1b33, 0x1b34, 0x1b35, 0x1b36, 0x1b37, + 0x1b38, 0x1b39, 0x1b3a, 0x1b3a, 0x1b3c, 0x1b3c, 0x1b3e, 0x1b3f, +}; + +static const unsigned short gNormalizeTable1b40[] = { + /* U+1b40 */ + 0x1b3e, 0x1b3f, 0x1b42, 0x1b42, 0x1b44, 0x1b45, 0x1b46, 0x1b47, + 0x1b48, 0x1b49, 0x1b4a, 0x1b4b, 0x1b4c, 0x1b4d, 0x1b4e, 0x1b4f, + 0x1b50, 0x1b51, 0x1b52, 0x1b53, 0x1b54, 0x1b55, 0x1b56, 0x1b57, + 0x1b58, 0x1b59, 0x1b5a, 0x1b5b, 0x1b5c, 0x1b5d, 0x1b5e, 0x1b5f, + 0x1b60, 0x1b61, 0x1b62, 0x1b63, 0x1b64, 0x1b65, 0x1b66, 0x1b67, + 0x1b68, 0x1b69, 0x1b6a, 0x1b6b, 0x1b6c, 0x1b6d, 0x1b6e, 0x1b6f, + 0x1b70, 0x1b71, 0x1b72, 0x1b73, 0x1b74, 0x1b75, 0x1b76, 0x1b77, + 0x1b78, 0x1b79, 0x1b7a, 0x1b7b, 0x1b7c, 0x1b7d, 0x1b7e, 0x1b7f, +}; + +static const unsigned short gNormalizeTable1d00[] = { + /* U+1d00 */ + 0x1d00, 0x1d01, 0x1d02, 0x1d03, 0x1d04, 0x1d05, 0x1d06, 0x1d07, + 0x1d08, 0x1d09, 0x1d0a, 0x1d0b, 0x1d0c, 0x1d0d, 0x1d0e, 0x1d0f, + 0x1d10, 0x1d11, 0x1d12, 0x1d13, 0x1d14, 0x1d15, 0x1d16, 0x1d17, + 0x1d18, 0x1d19, 0x1d1a, 0x1d1b, 0x1d1c, 0x1d1d, 0x1d1e, 0x1d1f, + 0x1d20, 0x1d21, 0x1d22, 0x1d23, 0x1d24, 0x1d25, 0x1d26, 0x1d27, + 0x1d28, 0x1d29, 0x1d2a, 0x1d2b, 0x0061, 0x00e6, 0x0062, 0x1d2f, + 0x0064, 0x0065, 0x01dd, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x1d3b, 0x006f, 0x0223, 0x0070, 0x0072, +}; + +static const unsigned short gNormalizeTable1d40[] = { + /* U+1d40 */ + 0x0074, 0x0075, 0x0077, 0x0061, 0x0250, 0x0251, 0x1d02, 0x0062, + 0x0064, 0x0065, 0x0259, 0x025b, 0x025c, 0x0067, 0x1d4e, 0x006b, + 0x006d, 0x014b, 0x006f, 0x0254, 0x1d16, 0x1d17, 0x0070, 0x0074, + 0x0075, 0x1d1d, 0x026f, 0x0076, 0x1d25, 0x03b2, 0x03b3, 0x03b4, + 0x03c6, 0x03c7, 0x0069, 0x0072, 0x0075, 0x0076, 0x03b2, 0x03b3, + 0x03c1, 0x03c6, 0x03c7, 0x1d6b, 0x1d6c, 0x1d6d, 0x1d6e, 0x1d6f, + 0x1d70, 0x1d71, 0x1d72, 0x1d73, 0x1d74, 0x1d75, 0x1d76, 0x1d77, + 0x043d, 0x1d79, 0x1d7a, 0x1d7b, 0x1d7c, 0x1d7d, 0x1d7e, 0x1d7f, +}; + +static const unsigned short gNormalizeTable1d80[] = { + /* U+1d80 */ + 0x1d80, 0x1d81, 0x1d82, 0x1d83, 0x1d84, 0x1d85, 0x1d86, 0x1d87, + 0x1d88, 0x1d89, 0x1d8a, 0x1d8b, 0x1d8c, 0x1d8d, 0x1d8e, 0x1d8f, + 0x1d90, 0x1d91, 0x1d92, 0x1d93, 0x1d94, 0x1d95, 0x1d96, 0x1d97, + 0x1d98, 0x1d99, 0x1d9a, 0x0252, 0x0063, 0x0255, 0x00f0, 0x025c, + 0x0066, 0x025f, 0x0261, 0x0265, 0x0268, 0x0269, 0x026a, 0x1d7b, + 0x029d, 0x026d, 0x1d85, 0x029f, 0x0271, 0x0270, 0x0272, 0x0273, + 0x0274, 0x0275, 0x0278, 0x0282, 0x0283, 0x01ab, 0x0289, 0x028a, + 0x1d1c, 0x028b, 0x028c, 0x007a, 0x0290, 0x0291, 0x0292, 0x03b8, +}; + +static const unsigned short gNormalizeTable1e00[] = { + /* U+1e00 */ + 0x0061, 0x0061, 0x0062, 0x0062, 0x0062, 0x0062, 0x0062, 0x0062, + 0x0063, 0x0063, 0x0064, 0x0064, 0x0064, 0x0064, 0x0064, 0x0064, + 0x0064, 0x0064, 0x0064, 0x0064, 0x0065, 0x0065, 0x0065, 0x0065, + 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, 0x0066, 0x0066, + 0x0067, 0x0067, 0x0068, 0x0068, 0x0068, 0x0068, 0x0068, 0x0068, + 0x0068, 0x0068, 0x0068, 0x0068, 0x0069, 0x0069, 0x0069, 0x0069, + 0x006b, 0x006b, 0x006b, 0x006b, 0x006b, 0x006b, 0x006c, 0x006c, + 0x006c, 0x006c, 0x006c, 0x006c, 0x006c, 0x006c, 0x006d, 0x006d, +}; + +static const unsigned short gNormalizeTable1e40[] = { + /* U+1e40 */ + 0x006d, 0x006d, 0x006d, 0x006d, 0x006e, 0x006e, 0x006e, 0x006e, + 0x006e, 0x006e, 0x006e, 0x006e, 0x006f, 0x006f, 0x006f, 0x006f, + 0x006f, 0x006f, 0x006f, 0x006f, 0x0070, 0x0070, 0x0070, 0x0070, + 0x0072, 0x0072, 0x0072, 0x0072, 0x0072, 0x0072, 0x0072, 0x0072, + 0x0073, 0x0073, 0x0073, 0x0073, 0x0073, 0x0073, 0x0073, 0x0073, + 0x0073, 0x0073, 0x0074, 0x0074, 0x0074, 0x0074, 0x0074, 0x0074, + 0x0074, 0x0074, 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, + 0x0075, 0x0075, 0x0075, 0x0075, 0x0076, 0x0076, 0x0076, 0x0076, +}; + +static const unsigned short gNormalizeTable1e80[] = { + /* U+1e80 */ + 0x0077, 0x0077, 0x0077, 0x0077, 0x0077, 0x0077, 0x0077, 0x0077, + 0x0077, 0x0077, 0x0078, 0x0078, 0x0078, 0x0078, 0x0079, 0x0079, + 0x007a, 0x007a, 0x007a, 0x007a, 0x007a, 0x007a, 0x0068, 0x0074, + 0x0077, 0x0079, 0x0061, 0x0073, 0x1e9c, 0x1e9d, 0x0073, 0x1e9f, + 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, + 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, + 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, + 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, +}; + +static const unsigned short gNormalizeTable1ec0[] = { + /* U+1ec0 */ + 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, + 0x0069, 0x0069, 0x0069, 0x0069, 0x006f, 0x006f, 0x006f, 0x006f, + 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, + 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, + 0x006f, 0x006f, 0x006f, 0x006f, 0x0075, 0x0075, 0x0075, 0x0075, + 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, + 0x0075, 0x0075, 0x0079, 0x0079, 0x0079, 0x0079, 0x0079, 0x0079, + 0x0079, 0x0079, 0x1efb, 0x1efb, 0x1efd, 0x1efd, 0x1eff, 0x1eff, +}; + +static const unsigned short gNormalizeTable1f00[] = { + /* U+1f00 */ + 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, + 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, + 0x03b5, 0x03b5, 0x03b5, 0x03b5, 0x03b5, 0x03b5, 0x1f16, 0x1f17, + 0x03b5, 0x03b5, 0x03b5, 0x03b5, 0x03b5, 0x03b5, 0x1f1e, 0x1f1f, + 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, + 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, + 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x03b9, + 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x03b9, +}; + +static const unsigned short gNormalizeTable1f40[] = { + /* U+1f40 */ + 0x03bf, 0x03bf, 0x03bf, 0x03bf, 0x03bf, 0x03bf, 0x1f46, 0x1f47, + 0x03bf, 0x03bf, 0x03bf, 0x03bf, 0x03bf, 0x03bf, 0x1f4e, 0x1f4f, + 0x03c5, 0x03c5, 0x03c5, 0x03c5, 0x03c5, 0x03c5, 0x03c5, 0x03c5, + 0x1f58, 0x03c5, 0x1f5a, 0x03c5, 0x1f5c, 0x03c5, 0x1f5e, 0x03c5, + 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, + 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, + 0x03b1, 0x03b1, 0x03b5, 0x03b5, 0x03b7, 0x03b7, 0x03b9, 0x03b9, + 0x03bf, 0x03bf, 0x03c5, 0x03c5, 0x03c9, 0x03c9, 0x1f7e, 0x1f7f, +}; + +static const unsigned short gNormalizeTable1f80[] = { + /* U+1f80 */ + 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, + 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, + 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, + 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, + 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, + 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, + 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x1fb5, 0x03b1, 0x03b1, + 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x0020, 0x03b9, 0x0020, +}; + +static const unsigned short gNormalizeTable1fc0[] = { + /* U+1fc0 */ + 0x0020, 0x0020, 0x03b7, 0x03b7, 0x03b7, 0x1fc5, 0x03b7, 0x03b7, + 0x03b5, 0x03b5, 0x03b7, 0x03b7, 0x03b7, 0x0020, 0x0020, 0x0020, + 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x1fd4, 0x1fd5, 0x03b9, 0x03b9, + 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x1fdc, 0x0020, 0x0020, 0x0020, + 0x03c5, 0x03c5, 0x03c5, 0x03c5, 0x03c1, 0x03c1, 0x03c5, 0x03c5, + 0x03c5, 0x03c5, 0x03c5, 0x03c5, 0x03c1, 0x0020, 0x0020, 0x0060, + 0x1ff0, 0x1ff1, 0x03c9, 0x03c9, 0x03c9, 0x1ff5, 0x03c9, 0x03c9, + 0x03bf, 0x03bf, 0x03c9, 0x03c9, 0x03c9, 0x0020, 0x0020, 0x1fff, +}; + +static const unsigned short gNormalizeTable2000[] = { + /* U+2000 */ + 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, + 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, + 0x2010, 0x2010, 0x2012, 0x2013, 0x2014, 0x2015, 0x2016, 0x0020, + 0x2018, 0x2019, 0x201a, 0x201b, 0x201c, 0x201d, 0x201e, 0x201f, + 0x2020, 0x2021, 0x2022, 0x2023, 0x002e, 0x002e, 0x002e, 0x2027, + 0x2028, 0x2029, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, + 0x2030, 0x2031, 0x2032, 0x2032, 0x2032, 0x2035, 0x2035, 0x2035, + 0x2038, 0x2039, 0x203a, 0x203b, 0x0021, 0x203d, 0x0020, 0x203f, +}; + +static const unsigned short gNormalizeTable2040[] = { + /* U+2040 */ + 0x2040, 0x2041, 0x2042, 0x2043, 0x2044, 0x2045, 0x2046, 0x003f, + 0x003f, 0x0021, 0x204a, 0x204b, 0x204c, 0x204d, 0x204e, 0x204f, + 0x2050, 0x2051, 0x2052, 0x2053, 0x2054, 0x2055, 0x2056, 0x2032, + 0x2058, 0x2059, 0x205a, 0x205b, 0x205c, 0x205d, 0x205e, 0x0020, + 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, + 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, + 0x0030, 0x0069, 0x2072, 0x2073, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x002b, 0x2212, 0x003d, 0x0028, 0x0029, 0x006e, +}; + +static const unsigned short gNormalizeTable2080[] = { + /* U+2080 */ + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x002b, 0x2212, 0x003d, 0x0028, 0x0029, 0x208f, + 0x0061, 0x0065, 0x006f, 0x0078, 0x0259, 0x2095, 0x2096, 0x2097, + 0x2098, 0x2099, 0x209a, 0x209b, 0x209c, 0x209d, 0x209e, 0x209f, + 0x20a0, 0x20a1, 0x20a2, 0x20a3, 0x20a4, 0x20a5, 0x20a6, 0x20a7, + 0x0072, 0x20a9, 0x20aa, 0x20ab, 0x20ac, 0x20ad, 0x20ae, 0x20af, + 0x20b0, 0x20b1, 0x20b2, 0x20b3, 0x20b4, 0x20b5, 0x20b6, 0x20b7, + 0x20b8, 0x20b9, 0x20ba, 0x20bb, 0x20bc, 0x20bd, 0x20be, 0x20bf, +}; + +static const unsigned short gNormalizeTable2100[] = { + /* U+2100 */ + 0x0061, 0x0061, 0x0063, 0x00b0, 0x2104, 0x0063, 0x0063, 0x025b, + 0x2108, 0x00b0, 0x0067, 0x0068, 0x0068, 0x0068, 0x0068, 0x0127, + 0x0069, 0x0069, 0x006c, 0x006c, 0x2114, 0x006e, 0x006e, 0x2117, + 0x2118, 0x0070, 0x0071, 0x0072, 0x0072, 0x0072, 0x211e, 0x211f, + 0x0073, 0x0074, 0x0074, 0x2123, 0x007a, 0x2125, 0x03c9, 0x2127, + 0x007a, 0x2129, 0x006b, 0x0061, 0x0062, 0x0063, 0x212e, 0x0065, + 0x0065, 0x0066, 0x214e, 0x006d, 0x006f, 0x05d0, 0x05d1, 0x05d2, + 0x05d3, 0x0069, 0x213a, 0x0066, 0x03c0, 0x03b3, 0x03b3, 0x03c0, +}; + +static const unsigned short gNormalizeTable2140[] = { + /* U+2140 */ + 0x2211, 0x2141, 0x2142, 0x2143, 0x2144, 0x0064, 0x0064, 0x0065, + 0x0069, 0x006a, 0x214a, 0x214b, 0x214c, 0x214d, 0x214e, 0x214f, + 0x0031, 0x0031, 0x0031, 0x0031, 0x0032, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0031, 0x0035, 0x0031, 0x0033, 0x0035, 0x0037, 0x0031, + 0x0069, 0x0069, 0x0069, 0x0069, 0x0076, 0x0076, 0x0076, 0x0076, + 0x0069, 0x0078, 0x0078, 0x0078, 0x006c, 0x0063, 0x0064, 0x006d, + 0x0069, 0x0069, 0x0069, 0x0069, 0x0076, 0x0076, 0x0076, 0x0076, + 0x0069, 0x0078, 0x0078, 0x0078, 0x006c, 0x0063, 0x0064, 0x006d, +}; + +static const unsigned short gNormalizeTable2180[] = { + /* U+2180 */ + 0x2180, 0x2181, 0x2182, 0x2184, 0x2184, 0x2185, 0x2186, 0x2187, + 0x2188, 0x0030, 0x218a, 0x218b, 0x218c, 0x218d, 0x218e, 0x218f, + 0x2190, 0x2191, 0x2192, 0x2193, 0x2194, 0x2195, 0x2196, 0x2197, + 0x2198, 0x2199, 0x2190, 0x2192, 0x219c, 0x219d, 0x219e, 0x219f, + 0x21a0, 0x21a1, 0x21a2, 0x21a3, 0x21a4, 0x21a5, 0x21a6, 0x21a7, + 0x21a8, 0x21a9, 0x21aa, 0x21ab, 0x21ac, 0x21ad, 0x2194, 0x21af, + 0x21b0, 0x21b1, 0x21b2, 0x21b3, 0x21b4, 0x21b5, 0x21b6, 0x21b7, + 0x21b8, 0x21b9, 0x21ba, 0x21bb, 0x21bc, 0x21bd, 0x21be, 0x21bf, +}; + +static const unsigned short gNormalizeTable21c0[] = { + /* U+21c0 */ + 0x21c0, 0x21c1, 0x21c2, 0x21c3, 0x21c4, 0x21c5, 0x21c6, 0x21c7, + 0x21c8, 0x21c9, 0x21ca, 0x21cb, 0x21cc, 0x21d0, 0x21d4, 0x21d2, + 0x21d0, 0x21d1, 0x21d2, 0x21d3, 0x21d4, 0x21d5, 0x21d6, 0x21d7, + 0x21d8, 0x21d9, 0x21da, 0x21db, 0x21dc, 0x21dd, 0x21de, 0x21df, + 0x21e0, 0x21e1, 0x21e2, 0x21e3, 0x21e4, 0x21e5, 0x21e6, 0x21e7, + 0x21e8, 0x21e9, 0x21ea, 0x21eb, 0x21ec, 0x21ed, 0x21ee, 0x21ef, + 0x21f0, 0x21f1, 0x21f2, 0x21f3, 0x21f4, 0x21f5, 0x21f6, 0x21f7, + 0x21f8, 0x21f9, 0x21fa, 0x21fb, 0x21fc, 0x21fd, 0x21fe, 0x21ff, +}; + +static const unsigned short gNormalizeTable2200[] = { + /* U+2200 */ + 0x2200, 0x2201, 0x2202, 0x2203, 0x2203, 0x2205, 0x2206, 0x2207, + 0x2208, 0x2208, 0x220a, 0x220b, 0x220b, 0x220d, 0x220e, 0x220f, + 0x2210, 0x2211, 0x2212, 0x2213, 0x2214, 0x2215, 0x2216, 0x2217, + 0x2218, 0x2219, 0x221a, 0x221b, 0x221c, 0x221d, 0x221e, 0x221f, + 0x2220, 0x2221, 0x2222, 0x2223, 0x2223, 0x2225, 0x2225, 0x2227, + 0x2228, 0x2229, 0x222a, 0x222b, 0x222b, 0x222b, 0x222e, 0x222e, + 0x222e, 0x2231, 0x2232, 0x2233, 0x2234, 0x2235, 0x2236, 0x2237, + 0x2238, 0x2239, 0x223a, 0x223b, 0x223c, 0x223d, 0x223e, 0x223f, +}; + +static const unsigned short gNormalizeTable2240[] = { + /* U+2240 */ + 0x2240, 0x223c, 0x2242, 0x2243, 0x2243, 0x2245, 0x2246, 0x2245, + 0x2248, 0x2248, 0x224a, 0x224b, 0x224c, 0x224d, 0x224e, 0x224f, + 0x2250, 0x2251, 0x2252, 0x2253, 0x2254, 0x2255, 0x2256, 0x2257, + 0x2258, 0x2259, 0x225a, 0x225b, 0x225c, 0x225d, 0x225e, 0x225f, + 0x003d, 0x2261, 0x2261, 0x2263, 0x2264, 0x2265, 0x2266, 0x2267, + 0x2268, 0x2269, 0x226a, 0x226b, 0x226c, 0x224d, 0x003c, 0x003e, + 0x2264, 0x2265, 0x2272, 0x2273, 0x2272, 0x2273, 0x2276, 0x2277, + 0x2276, 0x2277, 0x227a, 0x227b, 0x227c, 0x227d, 0x227e, 0x227f, +}; + +static const unsigned short gNormalizeTable2280[] = { + /* U+2280 */ + 0x227a, 0x227b, 0x2282, 0x2283, 0x2282, 0x2283, 0x2286, 0x2287, + 0x2286, 0x2287, 0x228a, 0x228b, 0x228c, 0x228d, 0x228e, 0x228f, + 0x2290, 0x2291, 0x2292, 0x2293, 0x2294, 0x2295, 0x2296, 0x2297, + 0x2298, 0x2299, 0x229a, 0x229b, 0x229c, 0x229d, 0x229e, 0x229f, + 0x22a0, 0x22a1, 0x22a2, 0x22a3, 0x22a4, 0x22a5, 0x22a6, 0x22a7, + 0x22a8, 0x22a9, 0x22aa, 0x22ab, 0x22a2, 0x22a8, 0x22a9, 0x22ab, + 0x22b0, 0x22b1, 0x22b2, 0x22b3, 0x22b4, 0x22b5, 0x22b6, 0x22b7, + 0x22b8, 0x22b9, 0x22ba, 0x22bb, 0x22bc, 0x22bd, 0x22be, 0x22bf, +}; + +static const unsigned short gNormalizeTable22c0[] = { + /* U+22c0 */ + 0x22c0, 0x22c1, 0x22c2, 0x22c3, 0x22c4, 0x22c5, 0x22c6, 0x22c7, + 0x22c8, 0x22c9, 0x22ca, 0x22cb, 0x22cc, 0x22cd, 0x22ce, 0x22cf, + 0x22d0, 0x22d1, 0x22d2, 0x22d3, 0x22d4, 0x22d5, 0x22d6, 0x22d7, + 0x22d8, 0x22d9, 0x22da, 0x22db, 0x22dc, 0x22dd, 0x22de, 0x22df, + 0x227c, 0x227d, 0x2291, 0x2292, 0x22e4, 0x22e5, 0x22e6, 0x22e7, + 0x22e8, 0x22e9, 0x22b2, 0x22b3, 0x22b4, 0x22b5, 0x22ee, 0x22ef, + 0x22f0, 0x22f1, 0x22f2, 0x22f3, 0x22f4, 0x22f5, 0x22f6, 0x22f7, + 0x22f8, 0x22f9, 0x22fa, 0x22fb, 0x22fc, 0x22fd, 0x22fe, 0x22ff, +}; + +static const unsigned short gNormalizeTable2300[] = { + /* U+2300 */ + 0x2300, 0x2301, 0x2302, 0x2303, 0x2304, 0x2305, 0x2306, 0x2307, + 0x2308, 0x2309, 0x230a, 0x230b, 0x230c, 0x230d, 0x230e, 0x230f, + 0x2310, 0x2311, 0x2312, 0x2313, 0x2314, 0x2315, 0x2316, 0x2317, + 0x2318, 0x2319, 0x231a, 0x231b, 0x231c, 0x231d, 0x231e, 0x231f, + 0x2320, 0x2321, 0x2322, 0x2323, 0x2324, 0x2325, 0x2326, 0x2327, + 0x2328, 0x3008, 0x3009, 0x232b, 0x232c, 0x232d, 0x232e, 0x232f, + 0x2330, 0x2331, 0x2332, 0x2333, 0x2334, 0x2335, 0x2336, 0x2337, + 0x2338, 0x2339, 0x233a, 0x233b, 0x233c, 0x233d, 0x233e, 0x233f, +}; + +static const unsigned short gNormalizeTable2440[] = { + /* U+2440 */ + 0x2440, 0x2441, 0x2442, 0x2443, 0x2444, 0x2445, 0x2446, 0x2447, + 0x2448, 0x2449, 0x244a, 0x244b, 0x244c, 0x244d, 0x244e, 0x244f, + 0x2450, 0x2451, 0x2452, 0x2453, 0x2454, 0x2455, 0x2456, 0x2457, + 0x2458, 0x2459, 0x245a, 0x245b, 0x245c, 0x245d, 0x245e, 0x245f, + 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, + 0x0039, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, + 0x0031, 0x0031, 0x0031, 0x0032, 0x0028, 0x0028, 0x0028, 0x0028, + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, +}; + +static const unsigned short gNormalizeTable2480[] = { + /* U+2480 */ + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, + 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, + 0x0039, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, + 0x0031, 0x0031, 0x0031, 0x0032, 0x0028, 0x0028, 0x0028, 0x0028, + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0061, 0x0062, + 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, +}; + +static const unsigned short gNormalizeTable24c0[] = { + /* U+24c0 */ + 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, 0x0070, 0x0071, 0x0072, + 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, + 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, + 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, 0x0070, + 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, + 0x0079, 0x007a, 0x0030, 0x24eb, 0x24ec, 0x24ed, 0x24ee, 0x24ef, + 0x24f0, 0x24f1, 0x24f2, 0x24f3, 0x24f4, 0x24f5, 0x24f6, 0x24f7, + 0x24f8, 0x24f9, 0x24fa, 0x24fb, 0x24fc, 0x24fd, 0x24fe, 0x24ff, +}; + +static const unsigned short gNormalizeTable2a00[] = { + /* U+2a00 */ + 0x2a00, 0x2a01, 0x2a02, 0x2a03, 0x2a04, 0x2a05, 0x2a06, 0x2a07, + 0x2a08, 0x2a09, 0x2a0a, 0x2a0b, 0x222b, 0x2a0d, 0x2a0e, 0x2a0f, + 0x2a10, 0x2a11, 0x2a12, 0x2a13, 0x2a14, 0x2a15, 0x2a16, 0x2a17, + 0x2a18, 0x2a19, 0x2a1a, 0x2a1b, 0x2a1c, 0x2a1d, 0x2a1e, 0x2a1f, + 0x2a20, 0x2a21, 0x2a22, 0x2a23, 0x2a24, 0x2a25, 0x2a26, 0x2a27, + 0x2a28, 0x2a29, 0x2a2a, 0x2a2b, 0x2a2c, 0x2a2d, 0x2a2e, 0x2a2f, + 0x2a30, 0x2a31, 0x2a32, 0x2a33, 0x2a34, 0x2a35, 0x2a36, 0x2a37, + 0x2a38, 0x2a39, 0x2a3a, 0x2a3b, 0x2a3c, 0x2a3d, 0x2a3e, 0x2a3f, +}; + +static const unsigned short gNormalizeTable2a40[] = { + /* U+2a40 */ + 0x2a40, 0x2a41, 0x2a42, 0x2a43, 0x2a44, 0x2a45, 0x2a46, 0x2a47, + 0x2a48, 0x2a49, 0x2a4a, 0x2a4b, 0x2a4c, 0x2a4d, 0x2a4e, 0x2a4f, + 0x2a50, 0x2a51, 0x2a52, 0x2a53, 0x2a54, 0x2a55, 0x2a56, 0x2a57, + 0x2a58, 0x2a59, 0x2a5a, 0x2a5b, 0x2a5c, 0x2a5d, 0x2a5e, 0x2a5f, + 0x2a60, 0x2a61, 0x2a62, 0x2a63, 0x2a64, 0x2a65, 0x2a66, 0x2a67, + 0x2a68, 0x2a69, 0x2a6a, 0x2a6b, 0x2a6c, 0x2a6d, 0x2a6e, 0x2a6f, + 0x2a70, 0x2a71, 0x2a72, 0x2a73, 0x003a, 0x003d, 0x003d, 0x2a77, + 0x2a78, 0x2a79, 0x2a7a, 0x2a7b, 0x2a7c, 0x2a7d, 0x2a7e, 0x2a7f, +}; + +static const unsigned short gNormalizeTable2ac0[] = { + /* U+2ac0 */ + 0x2ac0, 0x2ac1, 0x2ac2, 0x2ac3, 0x2ac4, 0x2ac5, 0x2ac6, 0x2ac7, + 0x2ac8, 0x2ac9, 0x2aca, 0x2acb, 0x2acc, 0x2acd, 0x2ace, 0x2acf, + 0x2ad0, 0x2ad1, 0x2ad2, 0x2ad3, 0x2ad4, 0x2ad5, 0x2ad6, 0x2ad7, + 0x2ad8, 0x2ad9, 0x2ada, 0x2adb, 0x2add, 0x2add, 0x2ade, 0x2adf, + 0x2ae0, 0x2ae1, 0x2ae2, 0x2ae3, 0x2ae4, 0x2ae5, 0x2ae6, 0x2ae7, + 0x2ae8, 0x2ae9, 0x2aea, 0x2aeb, 0x2aec, 0x2aed, 0x2aee, 0x2aef, + 0x2af0, 0x2af1, 0x2af2, 0x2af3, 0x2af4, 0x2af5, 0x2af6, 0x2af7, + 0x2af8, 0x2af9, 0x2afa, 0x2afb, 0x2afc, 0x2afd, 0x2afe, 0x2aff, +}; + +static const unsigned short gNormalizeTable2c00[] = { + /* U+2c00 */ + 0x2c30, 0x2c31, 0x2c32, 0x2c33, 0x2c34, 0x2c35, 0x2c36, 0x2c37, + 0x2c38, 0x2c39, 0x2c3a, 0x2c3b, 0x2c3c, 0x2c3d, 0x2c3e, 0x2c3f, + 0x2c40, 0x2c41, 0x2c42, 0x2c43, 0x2c44, 0x2c45, 0x2c46, 0x2c47, + 0x2c48, 0x2c49, 0x2c4a, 0x2c4b, 0x2c4c, 0x2c4d, 0x2c4e, 0x2c4f, + 0x2c50, 0x2c51, 0x2c52, 0x2c53, 0x2c54, 0x2c55, 0x2c56, 0x2c57, + 0x2c58, 0x2c59, 0x2c5a, 0x2c5b, 0x2c5c, 0x2c5d, 0x2c5e, 0x2c2f, + 0x2c30, 0x2c31, 0x2c32, 0x2c33, 0x2c34, 0x2c35, 0x2c36, 0x2c37, + 0x2c38, 0x2c39, 0x2c3a, 0x2c3b, 0x2c3c, 0x2c3d, 0x2c3e, 0x2c3f, +}; + +static const unsigned short gNormalizeTable2c40[] = { + /* U+2c40 */ + 0x2c40, 0x2c41, 0x2c42, 0x2c43, 0x2c44, 0x2c45, 0x2c46, 0x2c47, + 0x2c48, 0x2c49, 0x2c4a, 0x2c4b, 0x2c4c, 0x2c4d, 0x2c4e, 0x2c4f, + 0x2c50, 0x2c51, 0x2c52, 0x2c53, 0x2c54, 0x2c55, 0x2c56, 0x2c57, + 0x2c58, 0x2c59, 0x2c5a, 0x2c5b, 0x2c5c, 0x2c5d, 0x2c5e, 0x2c5f, + 0x2c61, 0x2c61, 0x026b, 0x1d7d, 0x027d, 0x2c65, 0x2c66, 0x2c68, + 0x2c68, 0x2c6a, 0x2c6a, 0x2c6c, 0x2c6c, 0x0251, 0x0271, 0x0250, + 0x0252, 0x2c71, 0x2c73, 0x2c73, 0x2c74, 0x2c76, 0x2c76, 0x2c77, + 0x2c78, 0x2c79, 0x2c7a, 0x2c7b, 0x006a, 0x0076, 0x023f, 0x0240, +}; + +static const unsigned short gNormalizeTable2c80[] = { + /* U+2c80 */ + 0x2c81, 0x2c81, 0x2c83, 0x2c83, 0x2c85, 0x2c85, 0x2c87, 0x2c87, + 0x2c89, 0x2c89, 0x2c8b, 0x2c8b, 0x2c8d, 0x2c8d, 0x2c8f, 0x2c8f, + 0x2c91, 0x2c91, 0x2c93, 0x2c93, 0x2c95, 0x2c95, 0x2c97, 0x2c97, + 0x2c99, 0x2c99, 0x2c9b, 0x2c9b, 0x2c9d, 0x2c9d, 0x2c9f, 0x2c9f, + 0x2ca1, 0x2ca1, 0x2ca3, 0x2ca3, 0x2ca5, 0x2ca5, 0x2ca7, 0x2ca7, + 0x2ca9, 0x2ca9, 0x2cab, 0x2cab, 0x2cad, 0x2cad, 0x2caf, 0x2caf, + 0x2cb1, 0x2cb1, 0x2cb3, 0x2cb3, 0x2cb5, 0x2cb5, 0x2cb7, 0x2cb7, + 0x2cb9, 0x2cb9, 0x2cbb, 0x2cbb, 0x2cbd, 0x2cbd, 0x2cbf, 0x2cbf, +}; + +static const unsigned short gNormalizeTable2cc0[] = { + /* U+2cc0 */ + 0x2cc1, 0x2cc1, 0x2cc3, 0x2cc3, 0x2cc5, 0x2cc5, 0x2cc7, 0x2cc7, + 0x2cc9, 0x2cc9, 0x2ccb, 0x2ccb, 0x2ccd, 0x2ccd, 0x2ccf, 0x2ccf, + 0x2cd1, 0x2cd1, 0x2cd3, 0x2cd3, 0x2cd5, 0x2cd5, 0x2cd7, 0x2cd7, + 0x2cd9, 0x2cd9, 0x2cdb, 0x2cdb, 0x2cdd, 0x2cdd, 0x2cdf, 0x2cdf, + 0x2ce1, 0x2ce1, 0x2ce3, 0x2ce3, 0x2ce4, 0x2ce5, 0x2ce6, 0x2ce7, + 0x2ce8, 0x2ce9, 0x2cea, 0x2cec, 0x2cec, 0x2cee, 0x2cee, 0x2cef, + 0x2cf0, 0x2cf1, 0x2cf2, 0x2cf3, 0x2cf4, 0x2cf5, 0x2cf6, 0x2cf7, + 0x2cf8, 0x2cf9, 0x2cfa, 0x2cfb, 0x2cfc, 0x2cfd, 0x2cfe, 0x2cff, +}; + +static const unsigned short gNormalizeTable2d40[] = { + /* U+2d40 */ + 0x2d40, 0x2d41, 0x2d42, 0x2d43, 0x2d44, 0x2d45, 0x2d46, 0x2d47, + 0x2d48, 0x2d49, 0x2d4a, 0x2d4b, 0x2d4c, 0x2d4d, 0x2d4e, 0x2d4f, + 0x2d50, 0x2d51, 0x2d52, 0x2d53, 0x2d54, 0x2d55, 0x2d56, 0x2d57, + 0x2d58, 0x2d59, 0x2d5a, 0x2d5b, 0x2d5c, 0x2d5d, 0x2d5e, 0x2d5f, + 0x2d60, 0x2d61, 0x2d62, 0x2d63, 0x2d64, 0x2d65, 0x2d66, 0x2d67, + 0x2d68, 0x2d69, 0x2d6a, 0x2d6b, 0x2d6c, 0x2d6d, 0x2d6e, 0x2d61, + 0x2d70, 0x2d71, 0x2d72, 0x2d73, 0x2d74, 0x2d75, 0x2d76, 0x2d77, + 0x2d78, 0x2d79, 0x2d7a, 0x2d7b, 0x2d7c, 0x2d7d, 0x2d7e, 0x2d7f, +}; + +static const unsigned short gNormalizeTable2e80[] = { + /* U+2e80 */ + 0x2e80, 0x2e81, 0x2e82, 0x2e83, 0x2e84, 0x2e85, 0x2e86, 0x2e87, + 0x2e88, 0x2e89, 0x2e8a, 0x2e8b, 0x2e8c, 0x2e8d, 0x2e8e, 0x2e8f, + 0x2e90, 0x2e91, 0x2e92, 0x2e93, 0x2e94, 0x2e95, 0x2e96, 0x2e97, + 0x2e98, 0x2e99, 0x2e9a, 0x2e9b, 0x2e9c, 0x2e9d, 0x2e9e, 0x6bcd, + 0x2ea0, 0x2ea1, 0x2ea2, 0x2ea3, 0x2ea4, 0x2ea5, 0x2ea6, 0x2ea7, + 0x2ea8, 0x2ea9, 0x2eaa, 0x2eab, 0x2eac, 0x2ead, 0x2eae, 0x2eaf, + 0x2eb0, 0x2eb1, 0x2eb2, 0x2eb3, 0x2eb4, 0x2eb5, 0x2eb6, 0x2eb7, + 0x2eb8, 0x2eb9, 0x2eba, 0x2ebb, 0x2ebc, 0x2ebd, 0x2ebe, 0x2ebf, +}; + +static const unsigned short gNormalizeTable2ec0[] = { + /* U+2ec0 */ + 0x2ec0, 0x2ec1, 0x2ec2, 0x2ec3, 0x2ec4, 0x2ec5, 0x2ec6, 0x2ec7, + 0x2ec8, 0x2ec9, 0x2eca, 0x2ecb, 0x2ecc, 0x2ecd, 0x2ece, 0x2ecf, + 0x2ed0, 0x2ed1, 0x2ed2, 0x2ed3, 0x2ed4, 0x2ed5, 0x2ed6, 0x2ed7, + 0x2ed8, 0x2ed9, 0x2eda, 0x2edb, 0x2edc, 0x2edd, 0x2ede, 0x2edf, + 0x2ee0, 0x2ee1, 0x2ee2, 0x2ee3, 0x2ee4, 0x2ee5, 0x2ee6, 0x2ee7, + 0x2ee8, 0x2ee9, 0x2eea, 0x2eeb, 0x2eec, 0x2eed, 0x2eee, 0x2eef, + 0x2ef0, 0x2ef1, 0x2ef2, 0x9f9f, 0x2ef4, 0x2ef5, 0x2ef6, 0x2ef7, + 0x2ef8, 0x2ef9, 0x2efa, 0x2efb, 0x2efc, 0x2efd, 0x2efe, 0x2eff, +}; + +static const unsigned short gNormalizeTable2f00[] = { + /* U+2f00 */ + 0x4e00, 0x4e28, 0x4e36, 0x4e3f, 0x4e59, 0x4e85, 0x4e8c, 0x4ea0, + 0x4eba, 0x513f, 0x5165, 0x516b, 0x5182, 0x5196, 0x51ab, 0x51e0, + 0x51f5, 0x5200, 0x529b, 0x52f9, 0x5315, 0x531a, 0x5338, 0x5341, + 0x535c, 0x5369, 0x5382, 0x53b6, 0x53c8, 0x53e3, 0x56d7, 0x571f, + 0x58eb, 0x5902, 0x590a, 0x5915, 0x5927, 0x5973, 0x5b50, 0x5b80, + 0x5bf8, 0x5c0f, 0x5c22, 0x5c38, 0x5c6e, 0x5c71, 0x5ddb, 0x5de5, + 0x5df1, 0x5dfe, 0x5e72, 0x5e7a, 0x5e7f, 0x5ef4, 0x5efe, 0x5f0b, + 0x5f13, 0x5f50, 0x5f61, 0x5f73, 0x5fc3, 0x6208, 0x6236, 0x624b, +}; + +static const unsigned short gNormalizeTable2f40[] = { + /* U+2f40 */ + 0x652f, 0x6534, 0x6587, 0x6597, 0x65a4, 0x65b9, 0x65e0, 0x65e5, + 0x66f0, 0x6708, 0x6728, 0x6b20, 0x6b62, 0x6b79, 0x6bb3, 0x6bcb, + 0x6bd4, 0x6bdb, 0x6c0f, 0x6c14, 0x6c34, 0x706b, 0x722a, 0x7236, + 0x723b, 0x723f, 0x7247, 0x7259, 0x725b, 0x72ac, 0x7384, 0x7389, + 0x74dc, 0x74e6, 0x7518, 0x751f, 0x7528, 0x7530, 0x758b, 0x7592, + 0x7676, 0x767d, 0x76ae, 0x76bf, 0x76ee, 0x77db, 0x77e2, 0x77f3, + 0x793a, 0x79b8, 0x79be, 0x7a74, 0x7acb, 0x7af9, 0x7c73, 0x7cf8, + 0x7f36, 0x7f51, 0x7f8a, 0x7fbd, 0x8001, 0x800c, 0x8012, 0x8033, +}; + +static const unsigned short gNormalizeTable2f80[] = { + /* U+2f80 */ + 0x807f, 0x8089, 0x81e3, 0x81ea, 0x81f3, 0x81fc, 0x820c, 0x821b, + 0x821f, 0x826e, 0x8272, 0x8278, 0x864d, 0x866b, 0x8840, 0x884c, + 0x8863, 0x897e, 0x898b, 0x89d2, 0x8a00, 0x8c37, 0x8c46, 0x8c55, + 0x8c78, 0x8c9d, 0x8d64, 0x8d70, 0x8db3, 0x8eab, 0x8eca, 0x8f9b, + 0x8fb0, 0x8fb5, 0x9091, 0x9149, 0x91c6, 0x91cc, 0x91d1, 0x9577, + 0x9580, 0x961c, 0x96b6, 0x96b9, 0x96e8, 0x9751, 0x975e, 0x9762, + 0x9769, 0x97cb, 0x97ed, 0x97f3, 0x9801, 0x98a8, 0x98db, 0x98df, + 0x9996, 0x9999, 0x99ac, 0x9aa8, 0x9ad8, 0x9adf, 0x9b25, 0x9b2f, +}; + +static const unsigned short gNormalizeTable2fc0[] = { + /* U+2fc0 */ + 0x9b32, 0x9b3c, 0x9b5a, 0x9ce5, 0x9e75, 0x9e7f, 0x9ea5, 0x9ebb, + 0x9ec3, 0x9ecd, 0x9ed1, 0x9ef9, 0x9efd, 0x9f0e, 0x9f13, 0x9f20, + 0x9f3b, 0x9f4a, 0x9f52, 0x9f8d, 0x9f9c, 0x9fa0, 0x2fd6, 0x2fd7, + 0x2fd8, 0x2fd9, 0x2fda, 0x2fdb, 0x2fdc, 0x2fdd, 0x2fde, 0x2fdf, + 0x2fe0, 0x2fe1, 0x2fe2, 0x2fe3, 0x2fe4, 0x2fe5, 0x2fe6, 0x2fe7, + 0x2fe8, 0x2fe9, 0x2fea, 0x2feb, 0x2fec, 0x2fed, 0x2fee, 0x2fef, + 0x2ff0, 0x2ff1, 0x2ff2, 0x2ff3, 0x2ff4, 0x2ff5, 0x2ff6, 0x2ff7, + 0x2ff8, 0x2ff9, 0x2ffa, 0x2ffb, 0x2ffc, 0x2ffd, 0x2ffe, 0x2fff, +}; + +static const unsigned short gNormalizeTable3000[] = { + /* U+3000 */ + 0x0020, 0x3001, 0x3002, 0x3003, 0x3004, 0x3005, 0x3006, 0x3007, + 0x3008, 0x3009, 0x300a, 0x300b, 0x300c, 0x300d, 0x300e, 0x300f, + 0x3010, 0x3011, 0x3012, 0x3013, 0x3014, 0x3015, 0x3016, 0x3017, + 0x3018, 0x3019, 0x301a, 0x301b, 0x301c, 0x301d, 0x301e, 0x301f, + 0x3020, 0x3021, 0x3022, 0x3023, 0x3024, 0x3025, 0x3026, 0x3027, + 0x3028, 0x3029, 0x302a, 0x302b, 0x302c, 0x302d, 0x302e, 0x302f, + 0x3030, 0x3031, 0x3032, 0x3033, 0x3034, 0x3035, 0x3012, 0x3037, + 0x5341, 0x5344, 0x5345, 0x303b, 0x303c, 0x303d, 0x303e, 0x303f, +}; + +static const unsigned short gNormalizeTable3040[] = { + /* U+3040 */ + 0x3040, 0x3041, 0x3042, 0x3043, 0x3044, 0x3045, 0x3046, 0x3047, + 0x3048, 0x3049, 0x304a, 0x304b, 0x304b, 0x304d, 0x304d, 0x304f, + 0x304f, 0x3051, 0x3051, 0x3053, 0x3053, 0x3055, 0x3055, 0x3057, + 0x3057, 0x3059, 0x3059, 0x305b, 0x305b, 0x305d, 0x305d, 0x305f, + 0x305f, 0x3061, 0x3061, 0x3063, 0x3064, 0x3064, 0x3066, 0x3066, + 0x3068, 0x3068, 0x306a, 0x306b, 0x306c, 0x306d, 0x306e, 0x306f, + 0x306f, 0x306f, 0x3072, 0x3072, 0x3072, 0x3075, 0x3075, 0x3075, + 0x3078, 0x3078, 0x3078, 0x307b, 0x307b, 0x307b, 0x307e, 0x307f, +}; + +static const unsigned short gNormalizeTable3080[] = { + /* U+3080 */ + 0x3080, 0x3081, 0x3082, 0x3083, 0x3084, 0x3085, 0x3086, 0x3087, + 0x3088, 0x3089, 0x308a, 0x308b, 0x308c, 0x308d, 0x308e, 0x308f, + 0x3090, 0x3091, 0x3092, 0x3093, 0x3046, 0x3095, 0x3096, 0x3097, + 0x3098, 0x3099, 0x309a, 0x0020, 0x0020, 0x309d, 0x309d, 0x3088, + 0x30a0, 0x30a1, 0x30a2, 0x30a3, 0x30a4, 0x30a5, 0x30a6, 0x30a7, + 0x30a8, 0x30a9, 0x30aa, 0x30ab, 0x30ab, 0x30ad, 0x30ad, 0x30af, + 0x30af, 0x30b1, 0x30b1, 0x30b3, 0x30b3, 0x30b5, 0x30b5, 0x30b7, + 0x30b7, 0x30b9, 0x30b9, 0x30bb, 0x30bb, 0x30bd, 0x30bd, 0x30bf, +}; + +static const unsigned short gNormalizeTable30c0[] = { + /* U+30c0 */ + 0x30bf, 0x30c1, 0x30c1, 0x30c3, 0x30c4, 0x30c4, 0x30c6, 0x30c6, + 0x30c8, 0x30c8, 0x30ca, 0x30cb, 0x30cc, 0x30cd, 0x30ce, 0x30cf, + 0x30cf, 0x30cf, 0x30d2, 0x30d2, 0x30d2, 0x30d5, 0x30d5, 0x30d5, + 0x30d8, 0x30d8, 0x30d8, 0x30db, 0x30db, 0x30db, 0x30de, 0x30df, + 0x30e0, 0x30e1, 0x30e2, 0x30e3, 0x30e4, 0x30e5, 0x30e6, 0x30e7, + 0x30e8, 0x30e9, 0x30ea, 0x30eb, 0x30ec, 0x30ed, 0x30ee, 0x30ef, + 0x30f0, 0x30f1, 0x30f2, 0x30f3, 0x30a6, 0x30f5, 0x30f6, 0x30ef, + 0x30f0, 0x30f1, 0x30f2, 0x30fb, 0x30fc, 0x30fd, 0x30fd, 0x30b3, +}; + +static const unsigned short gNormalizeTable3100[] = { + /* U+3100 */ + 0x3100, 0x3101, 0x3102, 0x3103, 0x3104, 0x3105, 0x3106, 0x3107, + 0x3108, 0x3109, 0x310a, 0x310b, 0x310c, 0x310d, 0x310e, 0x310f, + 0x3110, 0x3111, 0x3112, 0x3113, 0x3114, 0x3115, 0x3116, 0x3117, + 0x3118, 0x3119, 0x311a, 0x311b, 0x311c, 0x311d, 0x311e, 0x311f, + 0x3120, 0x3121, 0x3122, 0x3123, 0x3124, 0x3125, 0x3126, 0x3127, + 0x3128, 0x3129, 0x312a, 0x312b, 0x312c, 0x312d, 0x312e, 0x312f, + 0x3130, 0x1100, 0x1101, 0x11aa, 0x1102, 0x11ac, 0x11ad, 0x1103, + 0x1104, 0x1105, 0x11b0, 0x11b1, 0x11b2, 0x11b3, 0x11b4, 0x11b5, +}; + +static const unsigned short gNormalizeTable3140[] = { + /* U+3140 */ + 0x111a, 0x1106, 0x1107, 0x1108, 0x1121, 0x1109, 0x110a, 0x110b, + 0x110c, 0x110d, 0x110e, 0x110f, 0x1110, 0x1111, 0x1112, 0x1161, + 0x1162, 0x1163, 0x1164, 0x1165, 0x1166, 0x1167, 0x1168, 0x1169, + 0x116a, 0x116b, 0x116c, 0x116d, 0x116e, 0x116f, 0x1170, 0x1171, + 0x1172, 0x1173, 0x1174, 0x1175, 0x0020, 0x1114, 0x1115, 0x11c7, + 0x11c8, 0x11cc, 0x11ce, 0x11d3, 0x11d7, 0x11d9, 0x111c, 0x11dd, + 0x11df, 0x111d, 0x111e, 0x1120, 0x1122, 0x1123, 0x1127, 0x1129, + 0x112b, 0x112c, 0x112d, 0x112e, 0x112f, 0x1132, 0x1136, 0x1140, +}; + +static const unsigned short gNormalizeTable3180[] = { + /* U+3180 */ + 0x1147, 0x114c, 0x11f1, 0x11f2, 0x1157, 0x1158, 0x1159, 0x1184, + 0x1185, 0x1188, 0x1191, 0x1192, 0x1194, 0x119e, 0x11a1, 0x318f, + 0x3190, 0x3191, 0x4e00, 0x4e8c, 0x4e09, 0x56db, 0x4e0a, 0x4e2d, + 0x4e0b, 0x7532, 0x4e59, 0x4e19, 0x4e01, 0x5929, 0x5730, 0x4eba, + 0x31a0, 0x31a1, 0x31a2, 0x31a3, 0x31a4, 0x31a5, 0x31a6, 0x31a7, + 0x31a8, 0x31a9, 0x31aa, 0x31ab, 0x31ac, 0x31ad, 0x31ae, 0x31af, + 0x31b0, 0x31b1, 0x31b2, 0x31b3, 0x31b4, 0x31b5, 0x31b6, 0x31b7, + 0x31b8, 0x31b9, 0x31ba, 0x31bb, 0x31bc, 0x31bd, 0x31be, 0x31bf, +}; + +static const unsigned short gNormalizeTable3200[] = { + /* U+3200 */ + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x321f, + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, +}; + +static const unsigned short gNormalizeTable3240[] = { + /* U+3240 */ + 0x0028, 0x0028, 0x0028, 0x0028, 0x554f, 0x5e7c, 0x6587, 0x7b8f, + 0x3248, 0x3249, 0x324a, 0x324b, 0x324c, 0x324d, 0x324e, 0x324f, + 0x0070, 0x0032, 0x0032, 0x0032, 0x0032, 0x0032, 0x0032, 0x0032, + 0x0032, 0x0032, 0x0033, 0x0033, 0x0033, 0x0033, 0x0033, 0x0033, + 0x1100, 0x1102, 0x1103, 0x1105, 0x1106, 0x1107, 0x1109, 0x110b, + 0x110c, 0x110e, 0x110f, 0x1110, 0x1111, 0x1112, 0x1100, 0x1102, + 0x1103, 0x1105, 0x1106, 0x1107, 0x1109, 0x110b, 0x110c, 0x110e, + 0x110f, 0x1110, 0x1111, 0x1112, 0x110e, 0x110c, 0x110b, 0x327f, +}; + +static const unsigned short gNormalizeTable3280[] = { + /* U+3280 */ + 0x4e00, 0x4e8c, 0x4e09, 0x56db, 0x4e94, 0x516d, 0x4e03, 0x516b, + 0x4e5d, 0x5341, 0x6708, 0x706b, 0x6c34, 0x6728, 0x91d1, 0x571f, + 0x65e5, 0x682a, 0x6709, 0x793e, 0x540d, 0x7279, 0x8ca1, 0x795d, + 0x52b4, 0x79d8, 0x7537, 0x5973, 0x9069, 0x512a, 0x5370, 0x6ce8, + 0x9805, 0x4f11, 0x5199, 0x6b63, 0x4e0a, 0x4e2d, 0x4e0b, 0x5de6, + 0x53f3, 0x533b, 0x5b97, 0x5b66, 0x76e3, 0x4f01, 0x8cc7, 0x5354, + 0x591c, 0x0033, 0x0033, 0x0033, 0x0033, 0x0034, 0x0034, 0x0034, + 0x0034, 0x0034, 0x0034, 0x0034, 0x0034, 0x0034, 0x0034, 0x0035, +}; + +static const unsigned short gNormalizeTable32c0[] = { + /* U+32c0 */ + 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, + 0x0039, 0x0031, 0x0031, 0x0031, 0x0068, 0x0065, 0x0065, 0x006c, + 0x30a2, 0x30a4, 0x30a6, 0x30a8, 0x30aa, 0x30ab, 0x30ad, 0x30af, + 0x30b1, 0x30b3, 0x30b5, 0x30b7, 0x30b9, 0x30bb, 0x30bd, 0x30bf, + 0x30c1, 0x30c4, 0x30c6, 0x30c8, 0x30ca, 0x30cb, 0x30cc, 0x30cd, + 0x30ce, 0x30cf, 0x30d2, 0x30d5, 0x30d8, 0x30db, 0x30de, 0x30df, + 0x30e0, 0x30e1, 0x30e2, 0x30e4, 0x30e6, 0x30e8, 0x30e9, 0x30ea, + 0x30eb, 0x30ec, 0x30ed, 0x30ef, 0x30f0, 0x30f1, 0x30f2, 0x32ff, +}; + +static const unsigned short gNormalizeTable3300[] = { + /* U+3300 */ + 0x30a2, 0x30a2, 0x30a2, 0x30a2, 0x30a4, 0x30a4, 0x30a6, 0x30a8, + 0x30a8, 0x30aa, 0x30aa, 0x30ab, 0x30ab, 0x30ab, 0x30ab, 0x30ab, + 0x30ad, 0x30ad, 0x30ad, 0x30ad, 0x30ad, 0x30ad, 0x30ad, 0x30ad, + 0x30af, 0x30af, 0x30af, 0x30af, 0x30b1, 0x30b3, 0x30b3, 0x30b5, + 0x30b5, 0x30b7, 0x30bb, 0x30bb, 0x30bf, 0x30c6, 0x30c8, 0x30c8, + 0x30ca, 0x30ce, 0x30cf, 0x30cf, 0x30cf, 0x30cf, 0x30d2, 0x30d2, + 0x30d2, 0x30d2, 0x30d5, 0x30d5, 0x30d5, 0x30d5, 0x30d8, 0x30d8, + 0x30d8, 0x30d8, 0x30d8, 0x30d8, 0x30d8, 0x30db, 0x30db, 0x30db, +}; + +static const unsigned short gNormalizeTable3340[] = { + /* U+3340 */ + 0x30db, 0x30db, 0x30db, 0x30de, 0x30de, 0x30de, 0x30de, 0x30de, + 0x30df, 0x30df, 0x30df, 0x30e1, 0x30e1, 0x30e1, 0x30e4, 0x30e4, + 0x30e6, 0x30ea, 0x30ea, 0x30eb, 0x30eb, 0x30ec, 0x30ec, 0x30ef, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, + 0x0031, 0x0031, 0x0031, 0x0031, 0x0032, 0x0032, 0x0032, 0x0032, + 0x0032, 0x0068, 0x0064, 0x0061, 0x0062, 0x006f, 0x0070, 0x0064, + 0x0064, 0x0064, 0x0069, 0x5e73, 0x662d, 0x5927, 0x660e, 0x682a, +}; + +static const unsigned short gNormalizeTable3380[] = { + /* U+3380 */ + 0x0070, 0x006e, 0x03bc, 0x006d, 0x006b, 0x006b, 0x006d, 0x0067, + 0x0063, 0x006b, 0x0070, 0x006e, 0x03bc, 0x03bc, 0x006d, 0x006b, + 0x0068, 0x006b, 0x006d, 0x0067, 0x0074, 0x03bc, 0x006d, 0x0064, + 0x006b, 0x0066, 0x006e, 0x03bc, 0x006d, 0x0063, 0x006b, 0x006d, + 0x0063, 0x006d, 0x006b, 0x006d, 0x0063, 0x006d, 0x006b, 0x006d, + 0x006d, 0x0070, 0x006b, 0x006d, 0x0067, 0x0072, 0x0072, 0x0072, + 0x0070, 0x006e, 0x03bc, 0x006d, 0x0070, 0x006e, 0x03bc, 0x006d, + 0x006b, 0x006d, 0x0070, 0x006e, 0x03bc, 0x006d, 0x006b, 0x006d, +}; + +static const unsigned short gNormalizeTable33c0[] = { + /* U+33c0 */ + 0x006b, 0x006d, 0x0061, 0x0062, 0x0063, 0x0063, 0x0063, 0x0063, + 0x0064, 0x0067, 0x0068, 0x0068, 0x0069, 0x006b, 0x006b, 0x006b, + 0x006c, 0x006c, 0x006c, 0x006c, 0x006d, 0x006d, 0x006d, 0x0070, + 0x0070, 0x0070, 0x0070, 0x0073, 0x0073, 0x0077, 0x0076, 0x0061, + 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, + 0x0039, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, + 0x0031, 0x0031, 0x0031, 0x0032, 0x0032, 0x0032, 0x0032, 0x0032, + 0x0032, 0x0032, 0x0032, 0x0032, 0x0032, 0x0033, 0x0033, 0x0067, +}; + +static const unsigned short gNormalizeTablea640[] = { + /* U+a640 */ + 0xa641, 0xa641, 0xa643, 0xa643, 0xa645, 0xa645, 0xa647, 0xa647, + 0xa649, 0xa649, 0xa64b, 0xa64b, 0xa64d, 0xa64d, 0xa64f, 0xa64f, + 0xa651, 0xa651, 0xa653, 0xa653, 0xa655, 0xa655, 0xa657, 0xa657, + 0xa659, 0xa659, 0xa65b, 0xa65b, 0xa65d, 0xa65d, 0xa65f, 0xa65f, + 0xa660, 0xa661, 0xa663, 0xa663, 0xa665, 0xa665, 0xa667, 0xa667, + 0xa669, 0xa669, 0xa66b, 0xa66b, 0xa66d, 0xa66d, 0xa66e, 0xa66f, + 0xa670, 0xa671, 0xa672, 0xa673, 0xa674, 0xa675, 0xa676, 0xa677, + 0xa678, 0xa679, 0xa67a, 0xa67b, 0xa67c, 0xa67d, 0xa67e, 0xa67f, +}; + +static const unsigned short gNormalizeTablea680[] = { + /* U+a680 */ + 0xa681, 0xa681, 0xa683, 0xa683, 0xa685, 0xa685, 0xa687, 0xa687, + 0xa689, 0xa689, 0xa68b, 0xa68b, 0xa68d, 0xa68d, 0xa68f, 0xa68f, + 0xa691, 0xa691, 0xa693, 0xa693, 0xa695, 0xa695, 0xa697, 0xa697, + 0xa698, 0xa699, 0xa69a, 0xa69b, 0xa69c, 0xa69d, 0xa69e, 0xa69f, + 0xa6a0, 0xa6a1, 0xa6a2, 0xa6a3, 0xa6a4, 0xa6a5, 0xa6a6, 0xa6a7, + 0xa6a8, 0xa6a9, 0xa6aa, 0xa6ab, 0xa6ac, 0xa6ad, 0xa6ae, 0xa6af, + 0xa6b0, 0xa6b1, 0xa6b2, 0xa6b3, 0xa6b4, 0xa6b5, 0xa6b6, 0xa6b7, + 0xa6b8, 0xa6b9, 0xa6ba, 0xa6bb, 0xa6bc, 0xa6bd, 0xa6be, 0xa6bf, +}; + +static const unsigned short gNormalizeTablea700[] = { + /* U+a700 */ + 0xa700, 0xa701, 0xa702, 0xa703, 0xa704, 0xa705, 0xa706, 0xa707, + 0xa708, 0xa709, 0xa70a, 0xa70b, 0xa70c, 0xa70d, 0xa70e, 0xa70f, + 0xa710, 0xa711, 0xa712, 0xa713, 0xa714, 0xa715, 0xa716, 0xa717, + 0xa718, 0xa719, 0xa71a, 0xa71b, 0xa71c, 0xa71d, 0xa71e, 0xa71f, + 0xa720, 0xa721, 0xa723, 0xa723, 0xa725, 0xa725, 0xa727, 0xa727, + 0xa729, 0xa729, 0xa72b, 0xa72b, 0xa72d, 0xa72d, 0xa72f, 0xa72f, + 0xa730, 0xa731, 0xa733, 0xa733, 0xa735, 0xa735, 0xa737, 0xa737, + 0xa739, 0xa739, 0xa73b, 0xa73b, 0xa73d, 0xa73d, 0xa73f, 0xa73f, +}; + +static const unsigned short gNormalizeTablea740[] = { + /* U+a740 */ + 0xa741, 0xa741, 0xa743, 0xa743, 0xa745, 0xa745, 0xa747, 0xa747, + 0xa749, 0xa749, 0xa74b, 0xa74b, 0xa74d, 0xa74d, 0xa74f, 0xa74f, + 0xa751, 0xa751, 0xa753, 0xa753, 0xa755, 0xa755, 0xa757, 0xa757, + 0xa759, 0xa759, 0xa75b, 0xa75b, 0xa75d, 0xa75d, 0xa75f, 0xa75f, + 0xa761, 0xa761, 0xa763, 0xa763, 0xa765, 0xa765, 0xa767, 0xa767, + 0xa769, 0xa769, 0xa76b, 0xa76b, 0xa76d, 0xa76d, 0xa76f, 0xa76f, + 0xa76f, 0xa771, 0xa772, 0xa773, 0xa774, 0xa775, 0xa776, 0xa777, + 0xa778, 0xa77a, 0xa77a, 0xa77c, 0xa77c, 0x1d79, 0xa77f, 0xa77f, +}; + +static const unsigned short gNormalizeTablea780[] = { + /* U+a780 */ + 0xa781, 0xa781, 0xa783, 0xa783, 0xa785, 0xa785, 0xa787, 0xa787, + 0xa788, 0xa789, 0xa78a, 0xa78c, 0xa78c, 0xa78d, 0xa78e, 0xa78f, + 0xa790, 0xa791, 0xa792, 0xa793, 0xa794, 0xa795, 0xa796, 0xa797, + 0xa798, 0xa799, 0xa79a, 0xa79b, 0xa79c, 0xa79d, 0xa79e, 0xa79f, + 0xa7a0, 0xa7a1, 0xa7a2, 0xa7a3, 0xa7a4, 0xa7a5, 0xa7a6, 0xa7a7, + 0xa7a8, 0xa7a9, 0xa7aa, 0xa7ab, 0xa7ac, 0xa7ad, 0xa7ae, 0xa7af, + 0xa7b0, 0xa7b1, 0xa7b2, 0xa7b3, 0xa7b4, 0xa7b5, 0xa7b6, 0xa7b7, + 0xa7b8, 0xa7b9, 0xa7ba, 0xa7bb, 0xa7bc, 0xa7bd, 0xa7be, 0xa7bf, +}; + +static const unsigned short gNormalizeTablef900[] = { + /* U+f900 */ + 0x8c48, 0x66f4, 0x8eca, 0x8cc8, 0x6ed1, 0x4e32, 0x53e5, 0x9f9c, + 0x9f9c, 0x5951, 0x91d1, 0x5587, 0x5948, 0x61f6, 0x7669, 0x7f85, + 0x863f, 0x87ba, 0x88f8, 0x908f, 0x6a02, 0x6d1b, 0x70d9, 0x73de, + 0x843d, 0x916a, 0x99f1, 0x4e82, 0x5375, 0x6b04, 0x721b, 0x862d, + 0x9e1e, 0x5d50, 0x6feb, 0x85cd, 0x8964, 0x62c9, 0x81d8, 0x881f, + 0x5eca, 0x6717, 0x6d6a, 0x72fc, 0x90ce, 0x4f86, 0x51b7, 0x52de, + 0x64c4, 0x6ad3, 0x7210, 0x76e7, 0x8001, 0x8606, 0x865c, 0x8def, + 0x9732, 0x9b6f, 0x9dfa, 0x788c, 0x797f, 0x7da0, 0x83c9, 0x9304, +}; + +static const unsigned short gNormalizeTablef940[] = { + /* U+f940 */ + 0x9e7f, 0x8ad6, 0x58df, 0x5f04, 0x7c60, 0x807e, 0x7262, 0x78ca, + 0x8cc2, 0x96f7, 0x58d8, 0x5c62, 0x6a13, 0x6dda, 0x6f0f, 0x7d2f, + 0x7e37, 0x964b, 0x52d2, 0x808b, 0x51dc, 0x51cc, 0x7a1c, 0x7dbe, + 0x83f1, 0x9675, 0x8b80, 0x62cf, 0x6a02, 0x8afe, 0x4e39, 0x5be7, + 0x6012, 0x7387, 0x7570, 0x5317, 0x78fb, 0x4fbf, 0x5fa9, 0x4e0d, + 0x6ccc, 0x6578, 0x7d22, 0x53c3, 0x585e, 0x7701, 0x8449, 0x8aaa, + 0x6bba, 0x8fb0, 0x6c88, 0x62fe, 0x82e5, 0x63a0, 0x7565, 0x4eae, + 0x5169, 0x51c9, 0x6881, 0x7ce7, 0x826f, 0x8ad2, 0x91cf, 0x52f5, +}; + +static const unsigned short gNormalizeTablef980[] = { + /* U+f980 */ + 0x5442, 0x5973, 0x5eec, 0x65c5, 0x6ffe, 0x792a, 0x95ad, 0x9a6a, + 0x9e97, 0x9ece, 0x529b, 0x66c6, 0x6b77, 0x8f62, 0x5e74, 0x6190, + 0x6200, 0x649a, 0x6f23, 0x7149, 0x7489, 0x79ca, 0x7df4, 0x806f, + 0x8f26, 0x84ee, 0x9023, 0x934a, 0x5217, 0x52a3, 0x54bd, 0x70c8, + 0x88c2, 0x8aaa, 0x5ec9, 0x5ff5, 0x637b, 0x6bae, 0x7c3e, 0x7375, + 0x4ee4, 0x56f9, 0x5be7, 0x5dba, 0x601c, 0x73b2, 0x7469, 0x7f9a, + 0x8046, 0x9234, 0x96f6, 0x9748, 0x9818, 0x4f8b, 0x79ae, 0x91b4, + 0x96b8, 0x60e1, 0x4e86, 0x50da, 0x5bee, 0x5c3f, 0x6599, 0x6a02, +}; + +static const unsigned short gNormalizeTablef9c0[] = { + /* U+f9c0 */ + 0x71ce, 0x7642, 0x84fc, 0x907c, 0x9f8d, 0x6688, 0x962e, 0x5289, + 0x677b, 0x67f3, 0x6d41, 0x6e9c, 0x7409, 0x7559, 0x786b, 0x7d10, + 0x985e, 0x516d, 0x622e, 0x9678, 0x502b, 0x5d19, 0x6dea, 0x8f2a, + 0x5f8b, 0x6144, 0x6817, 0x7387, 0x9686, 0x5229, 0x540f, 0x5c65, + 0x6613, 0x674e, 0x68a8, 0x6ce5, 0x7406, 0x75e2, 0x7f79, 0x88cf, + 0x88e1, 0x91cc, 0x96e2, 0x533f, 0x6eba, 0x541d, 0x71d0, 0x7498, + 0x85fa, 0x96a3, 0x9c57, 0x9e9f, 0x6797, 0x6dcb, 0x81e8, 0x7acb, + 0x7b20, 0x7c92, 0x72c0, 0x7099, 0x8b58, 0x4ec0, 0x8336, 0x523a, +}; + +static const unsigned short gNormalizeTablefa00[] = { + /* U+fa00 */ + 0x5207, 0x5ea6, 0x62d3, 0x7cd6, 0x5b85, 0x6d1e, 0x66b4, 0x8f3b, + 0x884c, 0x964d, 0x898b, 0x5ed3, 0x5140, 0x55c0, 0xfa0e, 0xfa0f, + 0x585a, 0xfa11, 0x6674, 0xfa13, 0xfa14, 0x51de, 0x732a, 0x76ca, + 0x793c, 0x795e, 0x7965, 0x798f, 0x9756, 0x7cbe, 0x7fbd, 0xfa1f, + 0x8612, 0xfa21, 0x8af8, 0xfa23, 0xfa24, 0x9038, 0x90fd, 0xfa27, + 0xfa28, 0xfa29, 0x98ef, 0x98fc, 0x9928, 0x9db4, 0xfa2e, 0xfa2f, + 0x4fae, 0x50e7, 0x514d, 0x52c9, 0x52e4, 0x5351, 0x559d, 0x5606, + 0x5668, 0x5840, 0x58a8, 0x5c64, 0x5c6e, 0x6094, 0x6168, 0x618e, +}; + +static const unsigned short gNormalizeTablefa40[] = { + /* U+fa40 */ + 0x61f2, 0x654f, 0x65e2, 0x6691, 0x6885, 0x6d77, 0x6e1a, 0x6f22, + 0x716e, 0x722b, 0x7422, 0x7891, 0x793e, 0x7949, 0x7948, 0x7950, + 0x7956, 0x795d, 0x798d, 0x798e, 0x7a40, 0x7a81, 0x7bc0, 0x7df4, + 0x7e09, 0x7e41, 0x7f72, 0x8005, 0x81ed, 0x8279, 0x8279, 0x8457, + 0x8910, 0x8996, 0x8b01, 0x8b39, 0x8cd3, 0x8d08, 0x8fb6, 0x9038, + 0x96e3, 0x97ff, 0x983b, 0x6075, 0xfa6c, 0x8218, 0xfa6e, 0xfa6f, + 0x4e26, 0x51b5, 0x5168, 0x4f80, 0x5145, 0x5180, 0x52c7, 0x52fa, + 0x559d, 0x5555, 0x5599, 0x55e2, 0x585a, 0x58b3, 0x5944, 0x5954, +}; + +static const unsigned short gNormalizeTablefa80[] = { + /* U+fa80 */ + 0x5a62, 0x5b28, 0x5ed2, 0x5ed9, 0x5f69, 0x5fad, 0x60d8, 0x614e, + 0x6108, 0x618e, 0x6160, 0x61f2, 0x6234, 0x63c4, 0x641c, 0x6452, + 0x6556, 0x6674, 0x6717, 0x671b, 0x6756, 0x6b79, 0x6bba, 0x6d41, + 0x6edb, 0x6ecb, 0x6f22, 0x701e, 0x716e, 0x77a7, 0x7235, 0x72af, + 0x732a, 0x7471, 0x7506, 0x753b, 0x761d, 0x761f, 0x76ca, 0x76db, + 0x76f4, 0x774a, 0x7740, 0x78cc, 0x7ab1, 0x7bc0, 0x7c7b, 0x7d5b, + 0x7df4, 0x7f3e, 0x8005, 0x8352, 0x83ef, 0x8779, 0x8941, 0x8986, + 0x8996, 0x8abf, 0x8af8, 0x8acb, 0x8b01, 0x8afe, 0x8aed, 0x8b39, +}; + +static const unsigned short gNormalizeTablefac0[] = { + /* U+fac0 */ + 0x8b8a, 0x8d08, 0x8f38, 0x9072, 0x9199, 0x9276, 0x967c, 0x96e3, + 0x9756, 0x97db, 0x97ff, 0x980b, 0x983b, 0x9b12, 0x9f9c, 0xfacf, + 0xfad0, 0xfad1, 0x3b9d, 0x4018, 0x4039, 0xfad5, 0xfad6, 0xfad7, + 0x9f43, 0x9f8e, 0xfada, 0xfadb, 0xfadc, 0xfadd, 0xfade, 0xfadf, + 0xfae0, 0xfae1, 0xfae2, 0xfae3, 0xfae4, 0xfae5, 0xfae6, 0xfae7, + 0xfae8, 0xfae9, 0xfaea, 0xfaeb, 0xfaec, 0xfaed, 0xfaee, 0xfaef, + 0xfaf0, 0xfaf1, 0xfaf2, 0xfaf3, 0xfaf4, 0xfaf5, 0xfaf6, 0xfaf7, + 0xfaf8, 0xfaf9, 0xfafa, 0xfafb, 0xfafc, 0xfafd, 0xfafe, 0xfaff, +}; + +static const unsigned short gNormalizeTablefb00[] = { + /* U+fb00 */ + 0x0066, 0x0066, 0x0066, 0x0066, 0x0066, 0x0073, 0x0073, 0xfb07, + 0xfb08, 0xfb09, 0xfb0a, 0xfb0b, 0xfb0c, 0xfb0d, 0xfb0e, 0xfb0f, + 0xfb10, 0xfb11, 0xfb12, 0x0574, 0x0574, 0x0574, 0x057e, 0x0574, + 0xfb18, 0xfb19, 0xfb1a, 0xfb1b, 0xfb1c, 0x05d9, 0xfb1e, 0x05f2, + 0x05e2, 0x05d0, 0x05d3, 0x05d4, 0x05db, 0x05dc, 0x05dd, 0x05e8, + 0x05ea, 0x002b, 0x05e9, 0x05e9, 0x05e9, 0x05e9, 0x05d0, 0x05d0, + 0x05d0, 0x05d1, 0x05d2, 0x05d3, 0x05d4, 0x05d5, 0x05d6, 0xfb37, + 0x05d8, 0x05d9, 0x05da, 0x05db, 0x05dc, 0xfb3d, 0x05de, 0xfb3f, +}; + +static const unsigned short gNormalizeTablefb40[] = { + /* U+fb40 */ + 0x05e0, 0x05e1, 0xfb42, 0x05e3, 0x05e4, 0xfb45, 0x05e6, 0x05e7, + 0x05e8, 0x05e9, 0x05ea, 0x05d5, 0x05d1, 0x05db, 0x05e4, 0x05d0, + 0x0671, 0x0671, 0x067b, 0x067b, 0x067b, 0x067b, 0x067e, 0x067e, + 0x067e, 0x067e, 0x0680, 0x0680, 0x0680, 0x0680, 0x067a, 0x067a, + 0x067a, 0x067a, 0x067f, 0x067f, 0x067f, 0x067f, 0x0679, 0x0679, + 0x0679, 0x0679, 0x06a4, 0x06a4, 0x06a4, 0x06a4, 0x06a6, 0x06a6, + 0x06a6, 0x06a6, 0x0684, 0x0684, 0x0684, 0x0684, 0x0683, 0x0683, + 0x0683, 0x0683, 0x0686, 0x0686, 0x0686, 0x0686, 0x0687, 0x0687, +}; + +static const unsigned short gNormalizeTablefb80[] = { + /* U+fb80 */ + 0x0687, 0x0687, 0x068d, 0x068d, 0x068c, 0x068c, 0x068e, 0x068e, + 0x0688, 0x0688, 0x0698, 0x0698, 0x0691, 0x0691, 0x06a9, 0x06a9, + 0x06a9, 0x06a9, 0x06af, 0x06af, 0x06af, 0x06af, 0x06b3, 0x06b3, + 0x06b3, 0x06b3, 0x06b1, 0x06b1, 0x06b1, 0x06b1, 0x06ba, 0x06ba, + 0x06bb, 0x06bb, 0x06bb, 0x06bb, 0x06d5, 0x06d5, 0x06c1, 0x06c1, + 0x06c1, 0x06c1, 0x06be, 0x06be, 0x06be, 0x06be, 0x06d2, 0x06d2, + 0x06d2, 0x06d2, 0xfbb2, 0xfbb3, 0xfbb4, 0xfbb5, 0xfbb6, 0xfbb7, + 0xfbb8, 0xfbb9, 0xfbba, 0xfbbb, 0xfbbc, 0xfbbd, 0xfbbe, 0xfbbf, +}; + +static const unsigned short gNormalizeTablefbc0[] = { + /* U+fbc0 */ + 0xfbc0, 0xfbc1, 0xfbc2, 0xfbc3, 0xfbc4, 0xfbc5, 0xfbc6, 0xfbc7, + 0xfbc8, 0xfbc9, 0xfbca, 0xfbcb, 0xfbcc, 0xfbcd, 0xfbce, 0xfbcf, + 0xfbd0, 0xfbd1, 0xfbd2, 0x06ad, 0x06ad, 0x06ad, 0x06ad, 0x06c7, + 0x06c7, 0x06c6, 0x06c6, 0x06c8, 0x06c8, 0x06c7, 0x06cb, 0x06cb, + 0x06c5, 0x06c5, 0x06c9, 0x06c9, 0x06d0, 0x06d0, 0x06d0, 0x06d0, + 0x0649, 0x0649, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, + 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, + 0x064a, 0x064a, 0x064a, 0x064a, 0x06cc, 0x06cc, 0x06cc, 0x06cc, +}; + +static const unsigned short gNormalizeTablefc00[] = { + /* U+fc00 */ + 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, 0x0628, 0x0628, 0x0628, + 0x0628, 0x0628, 0x0628, 0x062a, 0x062a, 0x062a, 0x062a, 0x062a, + 0x062a, 0x062b, 0x062b, 0x062b, 0x062b, 0x062c, 0x062c, 0x062d, + 0x062d, 0x062e, 0x062e, 0x062e, 0x0633, 0x0633, 0x0633, 0x0633, + 0x0635, 0x0635, 0x0636, 0x0636, 0x0636, 0x0636, 0x0637, 0x0637, + 0x0638, 0x0639, 0x0639, 0x063a, 0x063a, 0x0641, 0x0641, 0x0641, + 0x0641, 0x0641, 0x0641, 0x0642, 0x0642, 0x0642, 0x0642, 0x0643, + 0x0643, 0x0643, 0x0643, 0x0643, 0x0643, 0x0643, 0x0643, 0x0644, +}; + +static const unsigned short gNormalizeTablefc40[] = { + /* U+fc40 */ + 0x0644, 0x0644, 0x0644, 0x0644, 0x0644, 0x0645, 0x0645, 0x0645, + 0x0645, 0x0645, 0x0645, 0x0646, 0x0646, 0x0646, 0x0646, 0x0646, + 0x0646, 0x0647, 0x0647, 0x0647, 0x0647, 0x064a, 0x064a, 0x064a, + 0x064a, 0x064a, 0x064a, 0x0630, 0x0631, 0x0649, 0x0020, 0x0020, + 0x0020, 0x0020, 0x0020, 0x0020, 0x064a, 0x064a, 0x064a, 0x064a, + 0x064a, 0x064a, 0x0628, 0x0628, 0x0628, 0x0628, 0x0628, 0x0628, + 0x062a, 0x062a, 0x062a, 0x062a, 0x062a, 0x062a, 0x062b, 0x062b, + 0x062b, 0x062b, 0x062b, 0x062b, 0x0641, 0x0641, 0x0642, 0x0642, +}; + +static const unsigned short gNormalizeTablefc80[] = { + /* U+fc80 */ + 0x0643, 0x0643, 0x0643, 0x0643, 0x0643, 0x0644, 0x0644, 0x0644, + 0x0645, 0x0645, 0x0646, 0x0646, 0x0646, 0x0646, 0x0646, 0x0646, + 0x0649, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, + 0x064a, 0x064a, 0x064a, 0x064a, 0x0628, 0x0628, 0x0628, 0x0628, + 0x0628, 0x062a, 0x062a, 0x062a, 0x062a, 0x062a, 0x062b, 0x062c, + 0x062c, 0x062d, 0x062d, 0x062e, 0x062e, 0x0633, 0x0633, 0x0633, + 0x0633, 0x0635, 0x0635, 0x0635, 0x0636, 0x0636, 0x0636, 0x0636, + 0x0637, 0x0638, 0x0639, 0x0639, 0x063a, 0x063a, 0x0641, 0x0641, +}; + +static const unsigned short gNormalizeTablefcc0[] = { + /* U+fcc0 */ + 0x0641, 0x0641, 0x0642, 0x0642, 0x0643, 0x0643, 0x0643, 0x0643, + 0x0643, 0x0644, 0x0644, 0x0644, 0x0644, 0x0644, 0x0645, 0x0645, + 0x0645, 0x0645, 0x0646, 0x0646, 0x0646, 0x0646, 0x0646, 0x0647, + 0x0647, 0x0647, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, + 0x064a, 0x0628, 0x0628, 0x062a, 0x062a, 0x062b, 0x062b, 0x0633, + 0x0633, 0x0634, 0x0634, 0x0643, 0x0643, 0x0644, 0x0646, 0x0646, + 0x064a, 0x064a, 0x0640, 0x0640, 0x0640, 0x0637, 0x0637, 0x0639, + 0x0639, 0x063a, 0x063a, 0x0633, 0x0633, 0x0634, 0x0634, 0x062d, +}; + +static const unsigned short gNormalizeTablefd00[] = { + /* U+fd00 */ + 0x062d, 0x062c, 0x062c, 0x062e, 0x062e, 0x0635, 0x0635, 0x0636, + 0x0636, 0x0634, 0x0634, 0x0634, 0x0634, 0x0634, 0x0633, 0x0635, + 0x0636, 0x0637, 0x0637, 0x0639, 0x0639, 0x063a, 0x063a, 0x0633, + 0x0633, 0x0634, 0x0634, 0x062d, 0x062d, 0x062c, 0x062c, 0x062e, + 0x062e, 0x0635, 0x0635, 0x0636, 0x0636, 0x0634, 0x0634, 0x0634, + 0x0634, 0x0634, 0x0633, 0x0635, 0x0636, 0x0634, 0x0634, 0x0634, + 0x0634, 0x0633, 0x0634, 0x0637, 0x0633, 0x0633, 0x0633, 0x0634, + 0x0634, 0x0634, 0x0637, 0x0638, 0x0627, 0x0627, 0xfd3e, 0xfd3f, +}; + +static const unsigned short gNormalizeTablefd40[] = { + /* U+fd40 */ + 0xfd40, 0xfd41, 0xfd42, 0xfd43, 0xfd44, 0xfd45, 0xfd46, 0xfd47, + 0xfd48, 0xfd49, 0xfd4a, 0xfd4b, 0xfd4c, 0xfd4d, 0xfd4e, 0xfd4f, + 0x062a, 0x062a, 0x062a, 0x062a, 0x062a, 0x062a, 0x062a, 0x062a, + 0x062c, 0x062c, 0x062d, 0x062d, 0x0633, 0x0633, 0x0633, 0x0633, + 0x0633, 0x0633, 0x0633, 0x0633, 0x0635, 0x0635, 0x0635, 0x0634, + 0x0634, 0x0634, 0x0634, 0x0634, 0x0634, 0x0634, 0x0636, 0x0636, + 0x0636, 0x0637, 0x0637, 0x0637, 0x0637, 0x0639, 0x0639, 0x0639, + 0x0639, 0x063a, 0x063a, 0x063a, 0x0641, 0x0641, 0x0642, 0x0642, +}; + +static const unsigned short gNormalizeTablefd80[] = { + /* U+fd80 */ + 0x0644, 0x0644, 0x0644, 0x0644, 0x0644, 0x0644, 0x0644, 0x0644, + 0x0644, 0x0645, 0x0645, 0x0645, 0x0645, 0x0645, 0x0645, 0x0645, + 0xfd90, 0xfd91, 0x0645, 0x0647, 0x0647, 0x0646, 0x0646, 0x0646, + 0x0646, 0x0646, 0x0646, 0x0646, 0x064a, 0x064a, 0x0628, 0x062a, + 0x062a, 0x062a, 0x062a, 0x062a, 0x062a, 0x062c, 0x062c, 0x062c, + 0x0633, 0x0635, 0x0634, 0x0636, 0x0644, 0x0644, 0x064a, 0x064a, + 0x064a, 0x0645, 0x0642, 0x0646, 0x0642, 0x0644, 0x0639, 0x0643, + 0x0646, 0x0645, 0x0644, 0x0643, 0x0644, 0x0646, 0x062c, 0x062d, +}; + +static const unsigned short gNormalizeTablefdc0[] = { + /* U+fdc0 */ + 0x0645, 0x0641, 0x0628, 0x0643, 0x0639, 0x0635, 0x0633, 0x0646, + 0xfdc8, 0xfdc9, 0xfdca, 0xfdcb, 0xfdcc, 0xfdcd, 0xfdce, 0xfdcf, + 0xfdd0, 0xfdd1, 0xfdd2, 0xfdd3, 0xfdd4, 0xfdd5, 0xfdd6, 0xfdd7, + 0xfdd8, 0xfdd9, 0xfdda, 0xfddb, 0xfddc, 0xfddd, 0xfdde, 0xfddf, + 0xfde0, 0xfde1, 0xfde2, 0xfde3, 0xfde4, 0xfde5, 0xfde6, 0xfde7, + 0xfde8, 0xfde9, 0xfdea, 0xfdeb, 0xfdec, 0xfded, 0xfdee, 0xfdef, + 0x0635, 0x0642, 0x0627, 0x0627, 0x0645, 0x0635, 0x0631, 0x0639, + 0x0648, 0x0635, 0x0635, 0x062c, 0x0631, 0xfdfd, 0xfdfe, 0xfdff, +}; + +static const unsigned short gNormalizeTablefe00[] = { + /* U+fe00 */ + 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, + 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, + 0x002c, 0x3001, 0x3002, 0x003a, 0x003b, 0x0021, 0x003f, 0x3016, + 0x3017, 0x002e, 0xfe1a, 0xfe1b, 0xfe1c, 0xfe1d, 0xfe1e, 0xfe1f, + 0xfe20, 0xfe21, 0xfe22, 0xfe23, 0xfe24, 0xfe25, 0xfe26, 0xfe27, + 0xfe28, 0xfe29, 0xfe2a, 0xfe2b, 0xfe2c, 0xfe2d, 0xfe2e, 0xfe2f, + 0x002e, 0x2014, 0x2013, 0x005f, 0x005f, 0x0028, 0x0029, 0x007b, + 0x007d, 0x3014, 0x3015, 0x3010, 0x3011, 0x300a, 0x300b, 0x3008, +}; + +static const unsigned short gNormalizeTablefe40[] = { + /* U+fe40 */ + 0x3009, 0x300c, 0x300d, 0x300e, 0x300f, 0xfe45, 0xfe46, 0x005b, + 0x005d, 0x0020, 0x0020, 0x0020, 0x0020, 0x005f, 0x005f, 0x005f, + 0x002c, 0x3001, 0x002e, 0xfe53, 0x003b, 0x003a, 0x003f, 0x0021, + 0x2014, 0x0028, 0x0029, 0x007b, 0x007d, 0x3014, 0x3015, 0x0023, + 0x0026, 0x002a, 0x002b, 0x002d, 0x003c, 0x003e, 0x003d, 0xfe67, + 0x005c, 0x0024, 0x0025, 0x0040, 0xfe6c, 0xfe6d, 0xfe6e, 0xfe6f, + 0x0020, 0x0640, 0x0020, 0xfe73, 0x0020, 0xfe75, 0x0020, 0x0640, + 0x0020, 0x0640, 0x0020, 0x0640, 0x0020, 0x0640, 0x0020, 0x0640, +}; + +static const unsigned short gNormalizeTablefe80[] = { + /* U+fe80 */ + 0x0621, 0x0627, 0x0627, 0x0627, 0x0627, 0x0648, 0x0648, 0x0627, + 0x0627, 0x064a, 0x064a, 0x064a, 0x064a, 0x0627, 0x0627, 0x0628, + 0x0628, 0x0628, 0x0628, 0x0629, 0x0629, 0x062a, 0x062a, 0x062a, + 0x062a, 0x062b, 0x062b, 0x062b, 0x062b, 0x062c, 0x062c, 0x062c, + 0x062c, 0x062d, 0x062d, 0x062d, 0x062d, 0x062e, 0x062e, 0x062e, + 0x062e, 0x062f, 0x062f, 0x0630, 0x0630, 0x0631, 0x0631, 0x0632, + 0x0632, 0x0633, 0x0633, 0x0633, 0x0633, 0x0634, 0x0634, 0x0634, + 0x0634, 0x0635, 0x0635, 0x0635, 0x0635, 0x0636, 0x0636, 0x0636, +}; + +static const unsigned short gNormalizeTablefec0[] = { + /* U+fec0 */ + 0x0636, 0x0637, 0x0637, 0x0637, 0x0637, 0x0638, 0x0638, 0x0638, + 0x0638, 0x0639, 0x0639, 0x0639, 0x0639, 0x063a, 0x063a, 0x063a, + 0x063a, 0x0641, 0x0641, 0x0641, 0x0641, 0x0642, 0x0642, 0x0642, + 0x0642, 0x0643, 0x0643, 0x0643, 0x0643, 0x0644, 0x0644, 0x0644, + 0x0644, 0x0645, 0x0645, 0x0645, 0x0645, 0x0646, 0x0646, 0x0646, + 0x0646, 0x0647, 0x0647, 0x0647, 0x0647, 0x0648, 0x0648, 0x0649, + 0x0649, 0x064a, 0x064a, 0x064a, 0x064a, 0x0644, 0x0644, 0x0644, + 0x0644, 0x0644, 0x0644, 0x0644, 0x0644, 0xfefd, 0xfefe, 0x0020, +}; + +static const unsigned short gNormalizeTableff00[] = { + /* U+ff00 */ + 0xff00, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, +}; + +static const unsigned short gNormalizeTableff40[] = { + /* U+ff40 */ + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x2985, + 0x2986, 0x3002, 0x300c, 0x300d, 0x3001, 0x30fb, 0x30f2, 0x30a1, + 0x30a3, 0x30a5, 0x30a7, 0x30a9, 0x30e3, 0x30e5, 0x30e7, 0x30c3, + 0x30fc, 0x30a2, 0x30a4, 0x30a6, 0x30a8, 0x30aa, 0x30ab, 0x30ad, + 0x30af, 0x30b1, 0x30b3, 0x30b5, 0x30b7, 0x30b9, 0x30bb, 0x30bd, +}; + +static const unsigned short gNormalizeTableff80[] = { + /* U+ff80 */ + 0x30bf, 0x30c1, 0x30c4, 0x30c6, 0x30c8, 0x30ca, 0x30cb, 0x30cc, + 0x30cd, 0x30ce, 0x30cf, 0x30d2, 0x30d5, 0x30d8, 0x30db, 0x30de, + 0x30df, 0x30e0, 0x30e1, 0x30e2, 0x30e4, 0x30e6, 0x30e8, 0x30e9, + 0x30ea, 0x30eb, 0x30ec, 0x30ed, 0x30ef, 0x30f3, 0x3099, 0x309a, + 0x0020, 0x1100, 0x1101, 0x11aa, 0x1102, 0x11ac, 0x11ad, 0x1103, + 0x1104, 0x1105, 0x11b0, 0x11b1, 0x11b2, 0x11b3, 0x11b4, 0x11b5, + 0x111a, 0x1106, 0x1107, 0x1108, 0x1121, 0x1109, 0x110a, 0x110b, + 0x110c, 0x110d, 0x110e, 0x110f, 0x1110, 0x1111, 0x1112, 0xffbf, +}; + +static const unsigned short gNormalizeTableffc0[] = { + /* U+ffc0 */ + 0xffc0, 0xffc1, 0x1161, 0x1162, 0x1163, 0x1164, 0x1165, 0x1166, + 0xffc8, 0xffc9, 0x1167, 0x1168, 0x1169, 0x116a, 0x116b, 0x116c, + 0xffd0, 0xffd1, 0x116d, 0x116e, 0x116f, 0x1170, 0x1171, 0x1172, + 0xffd8, 0xffd9, 0x1173, 0x1174, 0x1175, 0xffdd, 0xffde, 0xffdf, + 0x00a2, 0x00a3, 0x00ac, 0x0020, 0x00a6, 0x00a5, 0x20a9, 0xffe7, + 0x2502, 0x2190, 0x2191, 0x2192, 0x2193, 0x25a0, 0x25cb, 0xffef, + 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, + 0x0020, 0xfff9, 0xfffa, 0xfffb, 0xfffc, 0xfffd, 0xfffe, 0xffff, +}; + +static const unsigned short* gNormalizeTable[] = { + 0, + gNormalizeTable0040, + gNormalizeTable0080, + gNormalizeTable00c0, + gNormalizeTable0100, + gNormalizeTable0140, + gNormalizeTable0180, + gNormalizeTable01c0, + gNormalizeTable0200, + gNormalizeTable0240, + gNormalizeTable0280, + gNormalizeTable02c0, + 0, + gNormalizeTable0340, + gNormalizeTable0380, + gNormalizeTable03c0, + gNormalizeTable0400, + gNormalizeTable0440, + gNormalizeTable0480, + gNormalizeTable04c0, + gNormalizeTable0500, + gNormalizeTable0540, + gNormalizeTable0580, + 0, + gNormalizeTable0600, + gNormalizeTable0640, + 0, + gNormalizeTable06c0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + gNormalizeTable0900, + gNormalizeTable0940, + 0, + gNormalizeTable09c0, + gNormalizeTable0a00, + gNormalizeTable0a40, + 0, + 0, + 0, + gNormalizeTable0b40, + gNormalizeTable0b80, + gNormalizeTable0bc0, + 0, + gNormalizeTable0c40, + 0, + gNormalizeTable0cc0, + 0, + gNormalizeTable0d40, + 0, + gNormalizeTable0dc0, + gNormalizeTable0e00, + 0, + gNormalizeTable0e80, + gNormalizeTable0ec0, + gNormalizeTable0f00, + gNormalizeTable0f40, + gNormalizeTable0f80, + 0, + gNormalizeTable1000, + 0, + gNormalizeTable1080, + gNormalizeTable10c0, + 0, + gNormalizeTable1140, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + gNormalizeTable1780, + 0, + gNormalizeTable1800, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + gNormalizeTable1b00, + gNormalizeTable1b40, + 0, + 0, + 0, + 0, + 0, + 0, + gNormalizeTable1d00, + gNormalizeTable1d40, + gNormalizeTable1d80, + 0, + gNormalizeTable1e00, + gNormalizeTable1e40, + gNormalizeTable1e80, + gNormalizeTable1ec0, + gNormalizeTable1f00, + gNormalizeTable1f40, + gNormalizeTable1f80, + gNormalizeTable1fc0, + gNormalizeTable2000, + gNormalizeTable2040, + gNormalizeTable2080, + 0, + gNormalizeTable2100, + gNormalizeTable2140, + gNormalizeTable2180, + gNormalizeTable21c0, + gNormalizeTable2200, + gNormalizeTable2240, + gNormalizeTable2280, + gNormalizeTable22c0, + gNormalizeTable2300, + 0, + 0, + 0, + 0, + gNormalizeTable2440, + gNormalizeTable2480, + gNormalizeTable24c0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + gNormalizeTable2a00, + gNormalizeTable2a40, + 0, + gNormalizeTable2ac0, + 0, + 0, + 0, + 0, + gNormalizeTable2c00, + gNormalizeTable2c40, + gNormalizeTable2c80, + gNormalizeTable2cc0, + 0, + gNormalizeTable2d40, + 0, + 0, + 0, + 0, + gNormalizeTable2e80, + gNormalizeTable2ec0, + gNormalizeTable2f00, + gNormalizeTable2f40, + gNormalizeTable2f80, + gNormalizeTable2fc0, + gNormalizeTable3000, + gNormalizeTable3040, + gNormalizeTable3080, + gNormalizeTable30c0, + gNormalizeTable3100, + gNormalizeTable3140, + gNormalizeTable3180, + 0, + gNormalizeTable3200, + gNormalizeTable3240, + gNormalizeTable3280, + gNormalizeTable32c0, + gNormalizeTable3300, + gNormalizeTable3340, + gNormalizeTable3380, + gNormalizeTable33c0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + gNormalizeTablea640, + gNormalizeTablea680, + 0, + gNormalizeTablea700, + gNormalizeTablea740, + gNormalizeTablea780, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + gNormalizeTablef900, + gNormalizeTablef940, + gNormalizeTablef980, + gNormalizeTablef9c0, + gNormalizeTablefa00, + gNormalizeTablefa40, + gNormalizeTablefa80, + gNormalizeTablefac0, + gNormalizeTablefb00, + gNormalizeTablefb40, + gNormalizeTablefb80, + gNormalizeTablefbc0, + gNormalizeTablefc00, + gNormalizeTablefc40, + gNormalizeTablefc80, + gNormalizeTablefcc0, + gNormalizeTablefd00, + gNormalizeTablefd40, + gNormalizeTablefd80, + gNormalizeTablefdc0, + gNormalizeTablefe00, + gNormalizeTablefe40, + gNormalizeTablefe80, + gNormalizeTablefec0, + gNormalizeTableff00, + gNormalizeTableff40, + gNormalizeTableff80, + gNormalizeTableffc0, +}; + +unsigned int normalize_character(const unsigned int c) { + if (c >= 0x10000 || !gNormalizeTable[c >> 6]) return c; + return gNormalizeTable[c >> 6][c & 0x3f]; +} diff --git a/comm/mailnews/extensions/fts3/README.mozilla b/comm/mailnews/extensions/fts3/README.mozilla new file mode 100644 index 0000000000..0bfe7deb35 --- /dev/null +++ b/comm/mailnews/extensions/fts3/README.mozilla @@ -0,0 +1,3 @@ +fts3_porter.c code is from SQLite3. + +This customized tokenizer "mozporter" by Mozilla supports CJK indexing using bi-gram. So you have to use bi-gram search string if you wanto to search CJK character. diff --git a/comm/mailnews/extensions/fts3/components.conf b/comm/mailnews/extensions/fts3/components.conf new file mode 100644 index 0000000000..9a6fc2fb3c --- /dev/null +++ b/comm/mailnews/extensions/fts3/components.conf @@ -0,0 +1,12 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, you can obtain one at http://mozilla.org/MPL/2.0/. + +Classes = [ + { + "cid": "{a67d724d-0015-4e2e-8cad-b84775330924}", + "contract_ids": ["@mozilla.org/messenger/fts3tokenizer;1"], + "type": "nsFts3Tokenizer", + "headers": ["/comm/mailnews/extensions/fts3/nsFts3Tokenizer.h"], + }, +] diff --git a/comm/mailnews/extensions/fts3/data/README b/comm/mailnews/extensions/fts3/data/README new file mode 100644 index 0000000000..a6617918b2 --- /dev/null +++ b/comm/mailnews/extensions/fts3/data/README @@ -0,0 +1,5 @@ +The data files in this directory come from the ICU project: +http://bugs.icu-project.org/trac/browser/icu/trunk/source/data/unidata/norm2 + +They are intended to be consumed by the ICU project's gennorm2 script. We have +our own script that processes them. diff --git a/comm/mailnews/extensions/fts3/data/generate_table.py b/comm/mailnews/extensions/fts3/data/generate_table.py new file mode 100644 index 0000000000..80e0db996c --- /dev/null +++ b/comm/mailnews/extensions/fts3/data/generate_table.py @@ -0,0 +1,269 @@ +#!/usr/bin/python +# ***** BEGIN LICENSE BLOCK ***** +# Version: MPL 1.1/GPL 2.0/LGPL 2.1 +# +# The contents of this file are subject to the Mozilla Public License Version +# 1.1 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +# for the specific language governing rights and limitations under the +# License. +# +# The Original Code is Mozilla Thunderbird. +# +# The Initial Developer of the Original Code is Mozilla Japan. +# Portions created by the Initial Developer are Copyright (C) 2010 +# the Initial Developer. All Rights Reserved. +# +# Contributor(s): +# Makoto Kato <m_kato@ga2.so-net.ne.jp> +# Andrew Sutherland <asutherland@asutherland.org> +# +# Alternatively, the contents of this file may be used under the terms of +# either the GNU General Public License Version 2 or later (the "GPL"), or +# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), +# in which case the provisions of the GPL or the LGPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of either the GPL or the LGPL, and not to allow others to +# use your version of this file under the terms of the MPL, indicate your +# decision by deleting the provisions above and replace them with the notice +# and other provisions required by the GPL or the LGPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the MPL, the GPL or the LGPL. +# +# ***** END LICENSE BLOCK ***** + +import re + + +def print_table(f, t): + i = f + while i <= t: + c = array[i] + print("0x%04x," % c, end=" ") + i = i + 1 + if not i % 8: + print("\n\t", end=" ") + + +print( + """/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* ***** BEGIN LICENSE BLOCK ***** + * Version: MPL 1.1/GPL 2.0/LGPL 2.1 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is mozilla.org code. + * + * The Initial Developer of the Original Code is Mozilla Japan. + * Portions created by the Initial Developer are Copyright (C) 2010 + * the Initial Developer. All Rights Reserved. + * + * Contributor(s): + * Makoto Kato <m_kato@ga2.so-net.ne.jp> + * Andrew Sutherland <asutherland@asutherland.org> + * + * Alternatively, the contents of this file may be used under the terms of + * either of the GNU General Public License Version 2 or later (the "GPL"), + * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), + * in which case the provisions of the GPL or the LGPL are applicable instead + * of those above. If you wish to allow use of your version of this file only + * under the terms of either the GPL or the LGPL, and not to allow others to + * use your version of this file under the terms of the MPL, indicate your + * decision by deleting the provisions above and replace them with the notice + * and other provisions required by the GPL or the LGPL. If you do not delete + * the provisions above, a recipient may use your version of this file under + * the terms of any one of the MPL, the GPL or the LGPL. + * + * ***** END LICENSE BLOCK ***** */ + +/* THIS FILE IS GENERATED BY generate_table.py. DON'T EDIT THIS */ +""" +) + +p = re.compile("([0-9A-F]{4,5})(?:\.\.([0-9A-F]{4,5}))?[=\>]([0-9A-F]{4,5})?") +G_FROM = 1 +G_TO = 2 +G_FIRSTVAL = 3 + +# Array whose value at index i is the unicode value unicode character i should +# map to. +array = [] +# Contents of gNormalizeTable. We insert zero entries for sub-pages where we +# have no mappings. We insert references to the tables where we do have +# such tables. +globalTable = ["0"] +# The (exclusive) upper bound of the conversion table, unicode character-wise. +# This is 0x10000 because our generated table is only 16-bit. This also limits +# the values we can map to; we perform an identity mapping for target values +# that >= maxmapping. +maxmapping = 0x10000 +sizePerTable = 64 + +# Map characters that the mapping tells us to obliterate to the NUKE_CHAR +# (such lines look like "FFF0..FFF8>") +# We do this because if we didn't do this, we would emit these characters as +# part of a token, which we definitely don't want. +NUKE_CHAR = 0x20 + +# --- load case folding table +# entries in the file look like: +# 0041>0061 +# 02D8>0020 0306 +# 2000..200A>0020 +# +# The 0041 (uppercase A) tells us it lowercases to 0061 (lowercase a). +# The 02D8 is a "spacing clone[s] of diacritic" breve which gets decomposed into +# a space character and a breve. This entry/type of entry also shows up in +# 'nfkc.txt'. +# The 2000..200A covers a range of space characters and maps them down to the +# 'normal' space character. + +file = open("nfkc_cf.txt") + +m = None +line = "\n" +i = 0x0 +low = high = val = 0 +while i < maxmapping and line: + if not m: + line = file.readline() + m = p.match(line) + if not m: + continue + low = int(m.group(G_FROM), 16) + # if G_TO is present, use it, otherwise fallback to low + high = m.group(G_TO) and int(m.group(G_TO), 16) or low + # if G_FIRSTVAL is present use it, otherwise use NUKE_CHAR + val = m.group(G_FIRSTVAL) and int(m.group(G_FIRSTVAL), 16) or NUKE_CHAR + continue + + if low <= i <= high: + if val >= maxmapping: + array.append(i) + else: + array.append(val) + if i == high: + m = None + else: + array.append(i) + i = i + 1 +file.close() + +# --- load normalization / decomposition table +# It is important that this file gets processed second because the other table +# will tell us about mappings from uppercase U with diaeresis to lowercase u +# with diaeresis. We obviously don't want that clobbering our value. (Although +# this would work out if we propagated backwards rather than forwards...) +# +# - entries in this file that we care about look like: +# 00A0>0020 +# 0100=0041 0304 +# +# They are found in the "Canonical and compatibility decomposition mappings" +# section. +# +# The 00A0 is mapping NBSP to the normal space character. +# The 0100 (a capital A with a bar over top of) is equivalent to 0041 (capital +# A) plus a 0304 (combining overline). We do not care about the combining +# marks which is why our regular expression does not capture it. +# +# +# - entries that we do not care about look like: +# 0300..0314:230 +# +# These map marks to their canonical combining class which appears to be a way +# of specifying the precedence / order in which marks should be combined. The +# key thing is we don't care about them. +file = open("nfkc.txt") +line = file.readline() +m = p.match(line) +while line: + if not m: + line = file.readline() + m = p.match(line) + continue + + low = int(m.group(G_FROM), 16) + # if G_TO is present, use it, otherwise fallback to low + high = m.group(G_TO) and int(m.group(G_TO), 16) or low + # if G_FIRSTVAL is present use it, otherwise fall back to NUKE_CHAR + val = m.group(G_FIRSTVAL) and int(m.group(G_FIRSTVAL), 16) or NUKE_CHAR + for i in range(low, high + 1): + if i < maxmapping and val < maxmapping: + array[i] = val + m = None +file.close() + +# --- generate a normalized table to support case and accent folding + +i = 0 +needTerm = False +while i < maxmapping: + if not i % sizePerTable: + # table is empty? + j = i + while j < i + sizePerTable: + if array[j] != j: + break + j += 1 + + if j == i + sizePerTable: + if i: + globalTable.append("0") + i += sizePerTable + continue + + if needTerm: + print("};\n") + globalTable.append("gNormalizeTable%04x" % i) + print("static const unsigned short gNormalizeTable%04x[] = {\n\t" % i, end=" ") + print("/* U+%04x */\n\t" % i, end=" ") + needTerm = True + # Decomposition does not case-fold, so we want to compensate by + # performing a lookup here. Because decomposition chains can be + # example: 01d5, a capital U with a diaeresis and a bar. yes, really. + # 01d5 -> 00dc -> 0055 (U) -> 0075 (u) + c = array[i] + while c != array[c]: + c = array[c] + if 0x41 <= c <= 0x5A: + raise Exception("got an uppercase character somehow: %x => %x" % (i, c)) + print("0x%04x," % c, end=" ") + i = i + 1 + if not i % 8: + print("\n\t", end=" ") + +print("};\n\nstatic const unsigned short* gNormalizeTable[] = {", end=" ") +i = 0 +while i < (maxmapping / sizePerTable): + if not i % 4: + print("\n\t", end=" ") + print(globalTable[i] + ",", end=" ") + i += 1 + +print( + """ +}; + +unsigned int normalize_character(const unsigned int c) +{ + if (c >= """ + + ("0x%x" % (maxmapping,)) + + """ || !gNormalizeTable[c >> 6]) + return c; + return gNormalizeTable[c >> 6][c & 0x3f]; +} +""" +) diff --git a/comm/mailnews/extensions/fts3/data/nfkc.txt b/comm/mailnews/extensions/fts3/data/nfkc.txt new file mode 100644 index 0000000000..08aaf353f9 --- /dev/null +++ b/comm/mailnews/extensions/fts3/data/nfkc.txt @@ -0,0 +1,5786 @@ +# Copyright (C) 1999-2010, International Business Machines +# Corporation and others. All Rights Reserved. +# +# file name: nfkc.txt +# +# machine-generated on: 2009-11-30 +# + +# Canonical_Combining_Class (ccc) values +0300..0314:230 +0315:232 +0316..0319:220 +031A:232 +031B:216 +031C..0320:220 +0321..0322:202 +0323..0326:220 +0327..0328:202 +0329..0333:220 +0334..0338:1 +0339..033C:220 +033D..0344:230 +0345:240 +0346:230 +0347..0349:220 +034A..034C:230 +034D..034E:220 +0350..0352:230 +0353..0356:220 +0357:230 +0358:232 +0359..035A:220 +035B:230 +035C:233 +035D..035E:234 +035F:233 +0360..0361:234 +0362:233 +0363..036F:230 +0483..0487:230 +0591:220 +0592..0595:230 +0596:220 +0597..0599:230 +059A:222 +059B:220 +059C..05A1:230 +05A2..05A7:220 +05A8..05A9:230 +05AA:220 +05AB..05AC:230 +05AD:222 +05AE:228 +05AF:230 +05B0:10 +05B1:11 +05B2:12 +05B3:13 +05B4:14 +05B5:15 +05B6:16 +05B7:17 +05B8:18 +05B9..05BA:19 +05BB:20 +05BC:21 +05BD:22 +05BF:23 +05C1:24 +05C2:25 +05C4:230 +05C5:220 +05C7:18 +0610..0617:230 +0618:30 +0619:31 +061A:32 +064B:27 +064C:28 +064D:29 +064E:30 +064F:31 +0650:32 +0651:33 +0652:34 +0653..0654:230 +0655..0656:220 +0657..065B:230 +065C:220 +065D..065E:230 +0670:35 +06D6..06DC:230 +06DF..06E2:230 +06E3:220 +06E4:230 +06E7..06E8:230 +06EA:220 +06EB..06EC:230 +06ED:220 +0711:36 +0730:230 +0731:220 +0732..0733:230 +0734:220 +0735..0736:230 +0737..0739:220 +073A:230 +073B..073C:220 +073D:230 +073E:220 +073F..0741:230 +0742:220 +0743:230 +0744:220 +0745:230 +0746:220 +0747:230 +0748:220 +0749..074A:230 +07EB..07F1:230 +07F2:220 +07F3:230 +0816..0819:230 +081B..0823:230 +0825..0827:230 +0829..082D:230 +093C:7 +094D:9 +0951:230 +0952:220 +0953..0954:230 +09BC:7 +09CD:9 +0A3C:7 +0A4D:9 +0ABC:7 +0ACD:9 +0B3C:7 +0B4D:9 +0BCD:9 +0C4D:9 +0C55:84 +0C56:91 +0CBC:7 +0CCD:9 +0D4D:9 +0DCA:9 +0E38..0E39:103 +0E3A:9 +0E48..0E4B:107 +0EB8..0EB9:118 +0EC8..0ECB:122 +0F18..0F19:220 +0F35:220 +0F37:220 +0F39:216 +0F71:129 +0F72:130 +0F74:132 +0F7A..0F7D:130 +0F80:130 +0F82..0F83:230 +0F84:9 +0F86..0F87:230 +0FC6:220 +1037:7 +1039..103A:9 +108D:220 +135F:230 +1714:9 +1734:9 +17D2:9 +17DD:230 +18A9:228 +1939:222 +193A:230 +193B:220 +1A17:230 +1A18:220 +1A60:9 +1A75..1A7C:230 +1A7F:220 +1B34:7 +1B44:9 +1B6B:230 +1B6C:220 +1B6D..1B73:230 +1BAA:9 +1C37:7 +1CD0..1CD2:230 +1CD4:1 +1CD5..1CD9:220 +1CDA..1CDB:230 +1CDC..1CDF:220 +1CE0:230 +1CE2..1CE8:1 +1CED:220 +1DC0..1DC1:230 +1DC2:220 +1DC3..1DC9:230 +1DCA:220 +1DCB..1DCC:230 +1DCD:234 +1DCE:214 +1DCF:220 +1DD0:202 +1DD1..1DE6:230 +1DFD:220 +1DFE:230 +1DFF:220 +20D0..20D1:230 +20D2..20D3:1 +20D4..20D7:230 +20D8..20DA:1 +20DB..20DC:230 +20E1:230 +20E5..20E6:1 +20E7:230 +20E8:220 +20E9:230 +20EA..20EB:1 +20EC..20EF:220 +20F0:230 +2CEF..2CF1:230 +2DE0..2DFF:230 +302A:218 +302B:228 +302C:232 +302D:222 +302E..302F:224 +3099..309A:8 +A66F:230 +A67C..A67D:230 +A6F0..A6F1:230 +A806:9 +A8C4:9 +A8E0..A8F1:230 +A92B..A92D:220 +A953:9 +A9B3:7 +A9C0:9 +AAB0:230 +AAB2..AAB3:230 +AAB4:220 +AAB7..AAB8:230 +AABE..AABF:230 +AAC1:230 +ABED:9 +FB1E:26 +FE20..FE26:230 +101FD:220 +10A0D:220 +10A0F:230 +10A38:230 +10A39:1 +10A3A:220 +10A3F:9 +110B9:9 +110BA:7 +1D165..1D166:216 +1D167..1D169:1 +1D16D:226 +1D16E..1D172:216 +1D17B..1D182:220 +1D185..1D189:230 +1D18A..1D18B:220 +1D1AA..1D1AD:230 +1D242..1D244:230 + +# Canonical and compatibility decomposition mappings +00A0>0020 +00A8>0020 0308 +00AA>0061 +00AF>0020 0304 +00B2>0032 +00B3>0033 +00B4>0020 0301 +00B5>03BC +00B8>0020 0327 +00B9>0031 +00BA>006F +00BC>0031 2044 0034 +00BD>0031 2044 0032 +00BE>0033 2044 0034 +00C0=0041 0300 +00C1=0041 0301 +00C2=0041 0302 +00C3=0041 0303 +00C4=0041 0308 +00C5=0041 030A +00C7=0043 0327 +00C8=0045 0300 +00C9=0045 0301 +00CA=0045 0302 +00CB=0045 0308 +00CC=0049 0300 +00CD=0049 0301 +00CE=0049 0302 +00CF=0049 0308 +00D1=004E 0303 +00D2=004F 0300 +00D3=004F 0301 +00D4=004F 0302 +00D5=004F 0303 +00D6=004F 0308 +00D9=0055 0300 +00DA=0055 0301 +00DB=0055 0302 +00DC=0055 0308 +00DD=0059 0301 +00E0=0061 0300 +00E1=0061 0301 +00E2=0061 0302 +00E3=0061 0303 +00E4=0061 0308 +00E5=0061 030A +00E7=0063 0327 +00E8=0065 0300 +00E9=0065 0301 +00EA=0065 0302 +00EB=0065 0308 +00EC=0069 0300 +00ED=0069 0301 +00EE=0069 0302 +00EF=0069 0308 +00F1=006E 0303 +00F2=006F 0300 +00F3=006F 0301 +00F4=006F 0302 +00F5=006F 0303 +00F6=006F 0308 +00F9=0075 0300 +00FA=0075 0301 +00FB=0075 0302 +00FC=0075 0308 +00FD=0079 0301 +00FF=0079 0308 +0100=0041 0304 +0101=0061 0304 +0102=0041 0306 +0103=0061 0306 +0104=0041 0328 +0105=0061 0328 +0106=0043 0301 +0107=0063 0301 +0108=0043 0302 +0109=0063 0302 +010A=0043 0307 +010B=0063 0307 +010C=0043 030C +010D=0063 030C +010E=0044 030C +010F=0064 030C +0112=0045 0304 +0113=0065 0304 +0114=0045 0306 +0115=0065 0306 +0116=0045 0307 +0117=0065 0307 +0118=0045 0328 +0119=0065 0328 +011A=0045 030C +011B=0065 030C +011C=0047 0302 +011D=0067 0302 +011E=0047 0306 +011F=0067 0306 +0120=0047 0307 +0121=0067 0307 +0122=0047 0327 +0123=0067 0327 +0124=0048 0302 +0125=0068 0302 +0128=0049 0303 +0129=0069 0303 +012A=0049 0304 +012B=0069 0304 +012C=0049 0306 +012D=0069 0306 +012E=0049 0328 +012F=0069 0328 +0130=0049 0307 +0132>0049 004A +0133>0069 006A +0134=004A 0302 +0135=006A 0302 +0136=004B 0327 +0137=006B 0327 +0139=004C 0301 +013A=006C 0301 +013B=004C 0327 +013C=006C 0327 +013D=004C 030C +013E=006C 030C +013F>004C 00B7 +0140>006C 00B7 +0143=004E 0301 +0144=006E 0301 +0145=004E 0327 +0146=006E 0327 +0147=004E 030C +0148=006E 030C +0149>02BC 006E +014C=004F 0304 +014D=006F 0304 +014E=004F 0306 +014F=006F 0306 +0150=004F 030B +0151=006F 030B +0154=0052 0301 +0155=0072 0301 +0156=0052 0327 +0157=0072 0327 +0158=0052 030C +0159=0072 030C +015A=0053 0301 +015B=0073 0301 +015C=0053 0302 +015D=0073 0302 +015E=0053 0327 +015F=0073 0327 +0160=0053 030C +0161=0073 030C +0162=0054 0327 +0163=0074 0327 +0164=0054 030C +0165=0074 030C +0168=0055 0303 +0169=0075 0303 +016A=0055 0304 +016B=0075 0304 +016C=0055 0306 +016D=0075 0306 +016E=0055 030A +016F=0075 030A +0170=0055 030B +0171=0075 030B +0172=0055 0328 +0173=0075 0328 +0174=0057 0302 +0175=0077 0302 +0176=0059 0302 +0177=0079 0302 +0178=0059 0308 +0179=005A 0301 +017A=007A 0301 +017B=005A 0307 +017C=007A 0307 +017D=005A 030C +017E=007A 030C +017F>0073 +01A0=004F 031B +01A1=006F 031B +01AF=0055 031B +01B0=0075 031B +01C4>0044 017D +01C5>0044 017E +01C6>0064 017E +01C7>004C 004A +01C8>004C 006A +01C9>006C 006A +01CA>004E 004A +01CB>004E 006A +01CC>006E 006A +01CD=0041 030C +01CE=0061 030C +01CF=0049 030C +01D0=0069 030C +01D1=004F 030C +01D2=006F 030C +01D3=0055 030C +01D4=0075 030C +01D5=00DC 0304 +01D6=00FC 0304 +01D7=00DC 0301 +01D8=00FC 0301 +01D9=00DC 030C +01DA=00FC 030C +01DB=00DC 0300 +01DC=00FC 0300 +01DE=00C4 0304 +01DF=00E4 0304 +01E0=0226 0304 +01E1=0227 0304 +01E2=00C6 0304 +01E3=00E6 0304 +01E6=0047 030C +01E7=0067 030C +01E8=004B 030C +01E9=006B 030C +01EA=004F 0328 +01EB=006F 0328 +01EC=01EA 0304 +01ED=01EB 0304 +01EE=01B7 030C +01EF=0292 030C +01F0=006A 030C +01F1>0044 005A +01F2>0044 007A +01F3>0064 007A +01F4=0047 0301 +01F5=0067 0301 +01F8=004E 0300 +01F9=006E 0300 +01FA=00C5 0301 +01FB=00E5 0301 +01FC=00C6 0301 +01FD=00E6 0301 +01FE=00D8 0301 +01FF=00F8 0301 +0200=0041 030F +0201=0061 030F +0202=0041 0311 +0203=0061 0311 +0204=0045 030F +0205=0065 030F +0206=0045 0311 +0207=0065 0311 +0208=0049 030F +0209=0069 030F +020A=0049 0311 +020B=0069 0311 +020C=004F 030F +020D=006F 030F +020E=004F 0311 +020F=006F 0311 +0210=0052 030F +0211=0072 030F +0212=0052 0311 +0213=0072 0311 +0214=0055 030F +0215=0075 030F +0216=0055 0311 +0217=0075 0311 +0218=0053 0326 +0219=0073 0326 +021A=0054 0326 +021B=0074 0326 +021E=0048 030C +021F=0068 030C +0226=0041 0307 +0227=0061 0307 +0228=0045 0327 +0229=0065 0327 +022A=00D6 0304 +022B=00F6 0304 +022C=00D5 0304 +022D=00F5 0304 +022E=004F 0307 +022F=006F 0307 +0230=022E 0304 +0231=022F 0304 +0232=0059 0304 +0233=0079 0304 +02B0>0068 +02B1>0266 +02B2>006A +02B3>0072 +02B4>0279 +02B5>027B +02B6>0281 +02B7>0077 +02B8>0079 +02D8>0020 0306 +02D9>0020 0307 +02DA>0020 030A +02DB>0020 0328 +02DC>0020 0303 +02DD>0020 030B +02E0>0263 +02E1>006C +02E2>0073 +02E3>0078 +02E4>0295 +0340>0300 +0341>0301 +0343>0313 +0344>0308 0301 +0374>02B9 +037A>0020 0345 +037E>003B +0384>0020 0301 +0385>00A8 0301 +0386=0391 0301 +0387>00B7 +0388=0395 0301 +0389=0397 0301 +038A=0399 0301 +038C=039F 0301 +038E=03A5 0301 +038F=03A9 0301 +0390=03CA 0301 +03AA=0399 0308 +03AB=03A5 0308 +03AC=03B1 0301 +03AD=03B5 0301 +03AE=03B7 0301 +03AF=03B9 0301 +03B0=03CB 0301 +03CA=03B9 0308 +03CB=03C5 0308 +03CC=03BF 0301 +03CD=03C5 0301 +03CE=03C9 0301 +03D0>03B2 +03D1>03B8 +03D2>03A5 +03D3>03D2 0301 +03D4>03D2 0308 +03D5>03C6 +03D6>03C0 +03F0>03BA +03F1>03C1 +03F2>03C2 +03F4>0398 +03F5>03B5 +03F9>03A3 +0400=0415 0300 +0401=0415 0308 +0403=0413 0301 +0407=0406 0308 +040C=041A 0301 +040D=0418 0300 +040E=0423 0306 +0419=0418 0306 +0439=0438 0306 +0450=0435 0300 +0451=0435 0308 +0453=0433 0301 +0457=0456 0308 +045C=043A 0301 +045D=0438 0300 +045E=0443 0306 +0476=0474 030F +0477=0475 030F +04C1=0416 0306 +04C2=0436 0306 +04D0=0410 0306 +04D1=0430 0306 +04D2=0410 0308 +04D3=0430 0308 +04D6=0415 0306 +04D7=0435 0306 +04DA=04D8 0308 +04DB=04D9 0308 +04DC=0416 0308 +04DD=0436 0308 +04DE=0417 0308 +04DF=0437 0308 +04E2=0418 0304 +04E3=0438 0304 +04E4=0418 0308 +04E5=0438 0308 +04E6=041E 0308 +04E7=043E 0308 +04EA=04E8 0308 +04EB=04E9 0308 +04EC=042D 0308 +04ED=044D 0308 +04EE=0423 0304 +04EF=0443 0304 +04F0=0423 0308 +04F1=0443 0308 +04F2=0423 030B +04F3=0443 030B +04F4=0427 0308 +04F5=0447 0308 +04F8=042B 0308 +04F9=044B 0308 +0587>0565 0582 +0622=0627 0653 +0623=0627 0654 +0624=0648 0654 +0625=0627 0655 +0626=064A 0654 +0675>0627 0674 +0676>0648 0674 +0677>06C7 0674 +0678>064A 0674 +06C0=06D5 0654 +06C2=06C1 0654 +06D3=06D2 0654 +0929=0928 093C +0931=0930 093C +0934=0933 093C +0958>0915 093C +0959>0916 093C +095A>0917 093C +095B>091C 093C +095C>0921 093C +095D>0922 093C +095E>092B 093C +095F>092F 093C +09CB=09C7 09BE +09CC=09C7 09D7 +09DC>09A1 09BC +09DD>09A2 09BC +09DF>09AF 09BC +0A33>0A32 0A3C +0A36>0A38 0A3C +0A59>0A16 0A3C +0A5A>0A17 0A3C +0A5B>0A1C 0A3C +0A5E>0A2B 0A3C +0B48=0B47 0B56 +0B4B=0B47 0B3E +0B4C=0B47 0B57 +0B5C>0B21 0B3C +0B5D>0B22 0B3C +0B94=0B92 0BD7 +0BCA=0BC6 0BBE +0BCB=0BC7 0BBE +0BCC=0BC6 0BD7 +0C48=0C46 0C56 +0CC0=0CBF 0CD5 +0CC7=0CC6 0CD5 +0CC8=0CC6 0CD6 +0CCA=0CC6 0CC2 +0CCB=0CCA 0CD5 +0D4A=0D46 0D3E +0D4B=0D47 0D3E +0D4C=0D46 0D57 +0DDA=0DD9 0DCA +0DDC=0DD9 0DCF +0DDD=0DDC 0DCA +0DDE=0DD9 0DDF +0E33>0E4D 0E32 +0EB3>0ECD 0EB2 +0EDC>0EAB 0E99 +0EDD>0EAB 0EA1 +0F0C>0F0B +0F43>0F42 0FB7 +0F4D>0F4C 0FB7 +0F52>0F51 0FB7 +0F57>0F56 0FB7 +0F5C>0F5B 0FB7 +0F69>0F40 0FB5 +0F73>0F71 0F72 +0F75>0F71 0F74 +0F76>0FB2 0F80 +0F77>0FB2 0F81 +0F78>0FB3 0F80 +0F79>0FB3 0F81 +0F81>0F71 0F80 +0F93>0F92 0FB7 +0F9D>0F9C 0FB7 +0FA2>0FA1 0FB7 +0FA7>0FA6 0FB7 +0FAC>0FAB 0FB7 +0FB9>0F90 0FB5 +1026=1025 102E +10FC>10DC +1B06=1B05 1B35 +1B08=1B07 1B35 +1B0A=1B09 1B35 +1B0C=1B0B 1B35 +1B0E=1B0D 1B35 +1B12=1B11 1B35 +1B3B=1B3A 1B35 +1B3D=1B3C 1B35 +1B40=1B3E 1B35 +1B41=1B3F 1B35 +1B43=1B42 1B35 +1D2C>0041 +1D2D>00C6 +1D2E>0042 +1D30>0044 +1D31>0045 +1D32>018E +1D33>0047 +1D34>0048 +1D35>0049 +1D36>004A +1D37>004B +1D38>004C +1D39>004D +1D3A>004E +1D3C>004F +1D3D>0222 +1D3E>0050 +1D3F>0052 +1D40>0054 +1D41>0055 +1D42>0057 +1D43>0061 +1D44>0250 +1D45>0251 +1D46>1D02 +1D47>0062 +1D48>0064 +1D49>0065 +1D4A>0259 +1D4B>025B +1D4C>025C +1D4D>0067 +1D4F>006B +1D50>006D +1D51>014B +1D52>006F +1D53>0254 +1D54>1D16 +1D55>1D17 +1D56>0070 +1D57>0074 +1D58>0075 +1D59>1D1D +1D5A>026F +1D5B>0076 +1D5C>1D25 +1D5D>03B2 +1D5E>03B3 +1D5F>03B4 +1D60>03C6 +1D61>03C7 +1D62>0069 +1D63>0072 +1D64>0075 +1D65>0076 +1D66>03B2 +1D67>03B3 +1D68>03C1 +1D69>03C6 +1D6A>03C7 +1D78>043D +1D9B>0252 +1D9C>0063 +1D9D>0255 +1D9E>00F0 +1D9F>025C +1DA0>0066 +1DA1>025F +1DA2>0261 +1DA3>0265 +1DA4>0268 +1DA5>0269 +1DA6>026A +1DA7>1D7B +1DA8>029D +1DA9>026D +1DAA>1D85 +1DAB>029F +1DAC>0271 +1DAD>0270 +1DAE>0272 +1DAF>0273 +1DB0>0274 +1DB1>0275 +1DB2>0278 +1DB3>0282 +1DB4>0283 +1DB5>01AB +1DB6>0289 +1DB7>028A +1DB8>1D1C +1DB9>028B +1DBA>028C +1DBB>007A +1DBC>0290 +1DBD>0291 +1DBE>0292 +1DBF>03B8 +1E00=0041 0325 +1E01=0061 0325 +1E02=0042 0307 +1E03=0062 0307 +1E04=0042 0323 +1E05=0062 0323 +1E06=0042 0331 +1E07=0062 0331 +1E08=00C7 0301 +1E09=00E7 0301 +1E0A=0044 0307 +1E0B=0064 0307 +1E0C=0044 0323 +1E0D=0064 0323 +1E0E=0044 0331 +1E0F=0064 0331 +1E10=0044 0327 +1E11=0064 0327 +1E12=0044 032D +1E13=0064 032D +1E14=0112 0300 +1E15=0113 0300 +1E16=0112 0301 +1E17=0113 0301 +1E18=0045 032D +1E19=0065 032D +1E1A=0045 0330 +1E1B=0065 0330 +1E1C=0228 0306 +1E1D=0229 0306 +1E1E=0046 0307 +1E1F=0066 0307 +1E20=0047 0304 +1E21=0067 0304 +1E22=0048 0307 +1E23=0068 0307 +1E24=0048 0323 +1E25=0068 0323 +1E26=0048 0308 +1E27=0068 0308 +1E28=0048 0327 +1E29=0068 0327 +1E2A=0048 032E +1E2B=0068 032E +1E2C=0049 0330 +1E2D=0069 0330 +1E2E=00CF 0301 +1E2F=00EF 0301 +1E30=004B 0301 +1E31=006B 0301 +1E32=004B 0323 +1E33=006B 0323 +1E34=004B 0331 +1E35=006B 0331 +1E36=004C 0323 +1E37=006C 0323 +1E38=1E36 0304 +1E39=1E37 0304 +1E3A=004C 0331 +1E3B=006C 0331 +1E3C=004C 032D +1E3D=006C 032D +1E3E=004D 0301 +1E3F=006D 0301 +1E40=004D 0307 +1E41=006D 0307 +1E42=004D 0323 +1E43=006D 0323 +1E44=004E 0307 +1E45=006E 0307 +1E46=004E 0323 +1E47=006E 0323 +1E48=004E 0331 +1E49=006E 0331 +1E4A=004E 032D +1E4B=006E 032D +1E4C=00D5 0301 +1E4D=00F5 0301 +1E4E=00D5 0308 +1E4F=00F5 0308 +1E50=014C 0300 +1E51=014D 0300 +1E52=014C 0301 +1E53=014D 0301 +1E54=0050 0301 +1E55=0070 0301 +1E56=0050 0307 +1E57=0070 0307 +1E58=0052 0307 +1E59=0072 0307 +1E5A=0052 0323 +1E5B=0072 0323 +1E5C=1E5A 0304 +1E5D=1E5B 0304 +1E5E=0052 0331 +1E5F=0072 0331 +1E60=0053 0307 +1E61=0073 0307 +1E62=0053 0323 +1E63=0073 0323 +1E64=015A 0307 +1E65=015B 0307 +1E66=0160 0307 +1E67=0161 0307 +1E68=1E62 0307 +1E69=1E63 0307 +1E6A=0054 0307 +1E6B=0074 0307 +1E6C=0054 0323 +1E6D=0074 0323 +1E6E=0054 0331 +1E6F=0074 0331 +1E70=0054 032D +1E71=0074 032D +1E72=0055 0324 +1E73=0075 0324 +1E74=0055 0330 +1E75=0075 0330 +1E76=0055 032D +1E77=0075 032D +1E78=0168 0301 +1E79=0169 0301 +1E7A=016A 0308 +1E7B=016B 0308 +1E7C=0056 0303 +1E7D=0076 0303 +1E7E=0056 0323 +1E7F=0076 0323 +1E80=0057 0300 +1E81=0077 0300 +1E82=0057 0301 +1E83=0077 0301 +1E84=0057 0308 +1E85=0077 0308 +1E86=0057 0307 +1E87=0077 0307 +1E88=0057 0323 +1E89=0077 0323 +1E8A=0058 0307 +1E8B=0078 0307 +1E8C=0058 0308 +1E8D=0078 0308 +1E8E=0059 0307 +1E8F=0079 0307 +1E90=005A 0302 +1E91=007A 0302 +1E92=005A 0323 +1E93=007A 0323 +1E94=005A 0331 +1E95=007A 0331 +1E96=0068 0331 +1E97=0074 0308 +1E98=0077 030A +1E99=0079 030A +1E9A>0061 02BE +1E9B>017F 0307 +1EA0=0041 0323 +1EA1=0061 0323 +1EA2=0041 0309 +1EA3=0061 0309 +1EA4=00C2 0301 +1EA5=00E2 0301 +1EA6=00C2 0300 +1EA7=00E2 0300 +1EA8=00C2 0309 +1EA9=00E2 0309 +1EAA=00C2 0303 +1EAB=00E2 0303 +1EAC=1EA0 0302 +1EAD=1EA1 0302 +1EAE=0102 0301 +1EAF=0103 0301 +1EB0=0102 0300 +1EB1=0103 0300 +1EB2=0102 0309 +1EB3=0103 0309 +1EB4=0102 0303 +1EB5=0103 0303 +1EB6=1EA0 0306 +1EB7=1EA1 0306 +1EB8=0045 0323 +1EB9=0065 0323 +1EBA=0045 0309 +1EBB=0065 0309 +1EBC=0045 0303 +1EBD=0065 0303 +1EBE=00CA 0301 +1EBF=00EA 0301 +1EC0=00CA 0300 +1EC1=00EA 0300 +1EC2=00CA 0309 +1EC3=00EA 0309 +1EC4=00CA 0303 +1EC5=00EA 0303 +1EC6=1EB8 0302 +1EC7=1EB9 0302 +1EC8=0049 0309 +1EC9=0069 0309 +1ECA=0049 0323 +1ECB=0069 0323 +1ECC=004F 0323 +1ECD=006F 0323 +1ECE=004F 0309 +1ECF=006F 0309 +1ED0=00D4 0301 +1ED1=00F4 0301 +1ED2=00D4 0300 +1ED3=00F4 0300 +1ED4=00D4 0309 +1ED5=00F4 0309 +1ED6=00D4 0303 +1ED7=00F4 0303 +1ED8=1ECC 0302 +1ED9=1ECD 0302 +1EDA=01A0 0301 +1EDB=01A1 0301 +1EDC=01A0 0300 +1EDD=01A1 0300 +1EDE=01A0 0309 +1EDF=01A1 0309 +1EE0=01A0 0303 +1EE1=01A1 0303 +1EE2=01A0 0323 +1EE3=01A1 0323 +1EE4=0055 0323 +1EE5=0075 0323 +1EE6=0055 0309 +1EE7=0075 0309 +1EE8=01AF 0301 +1EE9=01B0 0301 +1EEA=01AF 0300 +1EEB=01B0 0300 +1EEC=01AF 0309 +1EED=01B0 0309 +1EEE=01AF 0303 +1EEF=01B0 0303 +1EF0=01AF 0323 +1EF1=01B0 0323 +1EF2=0059 0300 +1EF3=0079 0300 +1EF4=0059 0323 +1EF5=0079 0323 +1EF6=0059 0309 +1EF7=0079 0309 +1EF8=0059 0303 +1EF9=0079 0303 +1F00=03B1 0313 +1F01=03B1 0314 +1F02=1F00 0300 +1F03=1F01 0300 +1F04=1F00 0301 +1F05=1F01 0301 +1F06=1F00 0342 +1F07=1F01 0342 +1F08=0391 0313 +1F09=0391 0314 +1F0A=1F08 0300 +1F0B=1F09 0300 +1F0C=1F08 0301 +1F0D=1F09 0301 +1F0E=1F08 0342 +1F0F=1F09 0342 +1F10=03B5 0313 +1F11=03B5 0314 +1F12=1F10 0300 +1F13=1F11 0300 +1F14=1F10 0301 +1F15=1F11 0301 +1F18=0395 0313 +1F19=0395 0314 +1F1A=1F18 0300 +1F1B=1F19 0300 +1F1C=1F18 0301 +1F1D=1F19 0301 +1F20=03B7 0313 +1F21=03B7 0314 +1F22=1F20 0300 +1F23=1F21 0300 +1F24=1F20 0301 +1F25=1F21 0301 +1F26=1F20 0342 +1F27=1F21 0342 +1F28=0397 0313 +1F29=0397 0314 +1F2A=1F28 0300 +1F2B=1F29 0300 +1F2C=1F28 0301 +1F2D=1F29 0301 +1F2E=1F28 0342 +1F2F=1F29 0342 +1F30=03B9 0313 +1F31=03B9 0314 +1F32=1F30 0300 +1F33=1F31 0300 +1F34=1F30 0301 +1F35=1F31 0301 +1F36=1F30 0342 +1F37=1F31 0342 +1F38=0399 0313 +1F39=0399 0314 +1F3A=1F38 0300 +1F3B=1F39 0300 +1F3C=1F38 0301 +1F3D=1F39 0301 +1F3E=1F38 0342 +1F3F=1F39 0342 +1F40=03BF 0313 +1F41=03BF 0314 +1F42=1F40 0300 +1F43=1F41 0300 +1F44=1F40 0301 +1F45=1F41 0301 +1F48=039F 0313 +1F49=039F 0314 +1F4A=1F48 0300 +1F4B=1F49 0300 +1F4C=1F48 0301 +1F4D=1F49 0301 +1F50=03C5 0313 +1F51=03C5 0314 +1F52=1F50 0300 +1F53=1F51 0300 +1F54=1F50 0301 +1F55=1F51 0301 +1F56=1F50 0342 +1F57=1F51 0342 +1F59=03A5 0314 +1F5B=1F59 0300 +1F5D=1F59 0301 +1F5F=1F59 0342 +1F60=03C9 0313 +1F61=03C9 0314 +1F62=1F60 0300 +1F63=1F61 0300 +1F64=1F60 0301 +1F65=1F61 0301 +1F66=1F60 0342 +1F67=1F61 0342 +1F68=03A9 0313 +1F69=03A9 0314 +1F6A=1F68 0300 +1F6B=1F69 0300 +1F6C=1F68 0301 +1F6D=1F69 0301 +1F6E=1F68 0342 +1F6F=1F69 0342 +1F70=03B1 0300 +1F71>03AC +1F72=03B5 0300 +1F73>03AD +1F74=03B7 0300 +1F75>03AE +1F76=03B9 0300 +1F77>03AF +1F78=03BF 0300 +1F79>03CC +1F7A=03C5 0300 +1F7B>03CD +1F7C=03C9 0300 +1F7D>03CE +1F80=1F00 0345 +1F81=1F01 0345 +1F82=1F02 0345 +1F83=1F03 0345 +1F84=1F04 0345 +1F85=1F05 0345 +1F86=1F06 0345 +1F87=1F07 0345 +1F88=1F08 0345 +1F89=1F09 0345 +1F8A=1F0A 0345 +1F8B=1F0B 0345 +1F8C=1F0C 0345 +1F8D=1F0D 0345 +1F8E=1F0E 0345 +1F8F=1F0F 0345 +1F90=1F20 0345 +1F91=1F21 0345 +1F92=1F22 0345 +1F93=1F23 0345 +1F94=1F24 0345 +1F95=1F25 0345 +1F96=1F26 0345 +1F97=1F27 0345 +1F98=1F28 0345 +1F99=1F29 0345 +1F9A=1F2A 0345 +1F9B=1F2B 0345 +1F9C=1F2C 0345 +1F9D=1F2D 0345 +1F9E=1F2E 0345 +1F9F=1F2F 0345 +1FA0=1F60 0345 +1FA1=1F61 0345 +1FA2=1F62 0345 +1FA3=1F63 0345 +1FA4=1F64 0345 +1FA5=1F65 0345 +1FA6=1F66 0345 +1FA7=1F67 0345 +1FA8=1F68 0345 +1FA9=1F69 0345 +1FAA=1F6A 0345 +1FAB=1F6B 0345 +1FAC=1F6C 0345 +1FAD=1F6D 0345 +1FAE=1F6E 0345 +1FAF=1F6F 0345 +1FB0=03B1 0306 +1FB1=03B1 0304 +1FB2=1F70 0345 +1FB3=03B1 0345 +1FB4=03AC 0345 +1FB6=03B1 0342 +1FB7=1FB6 0345 +1FB8=0391 0306 +1FB9=0391 0304 +1FBA=0391 0300 +1FBB>0386 +1FBC=0391 0345 +1FBD>0020 0313 +1FBE>03B9 +1FBF>0020 0313 +1FC0>0020 0342 +1FC1>00A8 0342 +1FC2=1F74 0345 +1FC3=03B7 0345 +1FC4=03AE 0345 +1FC6=03B7 0342 +1FC7=1FC6 0345 +1FC8=0395 0300 +1FC9>0388 +1FCA=0397 0300 +1FCB>0389 +1FCC=0397 0345 +1FCD>1FBF 0300 +1FCE>1FBF 0301 +1FCF>1FBF 0342 +1FD0=03B9 0306 +1FD1=03B9 0304 +1FD2=03CA 0300 +1FD3>0390 +1FD6=03B9 0342 +1FD7=03CA 0342 +1FD8=0399 0306 +1FD9=0399 0304 +1FDA=0399 0300 +1FDB>038A +1FDD>1FFE 0300 +1FDE>1FFE 0301 +1FDF>1FFE 0342 +1FE0=03C5 0306 +1FE1=03C5 0304 +1FE2=03CB 0300 +1FE3>03B0 +1FE4=03C1 0313 +1FE5=03C1 0314 +1FE6=03C5 0342 +1FE7=03CB 0342 +1FE8=03A5 0306 +1FE9=03A5 0304 +1FEA=03A5 0300 +1FEB>038E +1FEC=03A1 0314 +1FED>00A8 0300 +1FEE>0385 +1FEF>0060 +1FF2=1F7C 0345 +1FF3=03C9 0345 +1FF4=03CE 0345 +1FF6=03C9 0342 +1FF7=1FF6 0345 +1FF8=039F 0300 +1FF9>038C +1FFA=03A9 0300 +1FFB>038F +1FFC=03A9 0345 +1FFD>00B4 +1FFE>0020 0314 +2000>2002 +2001>2003 +2002>0020 +2003>0020 +2004>0020 +2005>0020 +2006>0020 +2007>0020 +2008>0020 +2009>0020 +200A>0020 +2011>2010 +2017>0020 0333 +2024>002E +2025>002E 002E +2026>002E 002E 002E +202F>0020 +2033>2032 2032 +2034>2032 2032 2032 +2036>2035 2035 +2037>2035 2035 2035 +203C>0021 0021 +203E>0020 0305 +2047>003F 003F +2048>003F 0021 +2049>0021 003F +2057>2032 2032 2032 2032 +205F>0020 +2070>0030 +2071>0069 +2074>0034 +2075>0035 +2076>0036 +2077>0037 +2078>0038 +2079>0039 +207A>002B +207B>2212 +207C>003D +207D>0028 +207E>0029 +207F>006E +2080>0030 +2081>0031 +2082>0032 +2083>0033 +2084>0034 +2085>0035 +2086>0036 +2087>0037 +2088>0038 +2089>0039 +208A>002B +208B>2212 +208C>003D +208D>0028 +208E>0029 +2090>0061 +2091>0065 +2092>006F +2093>0078 +2094>0259 +20A8>0052 0073 +2100>0061 002F 0063 +2101>0061 002F 0073 +2102>0043 +2103>00B0 0043 +2105>0063 002F 006F +2106>0063 002F 0075 +2107>0190 +2109>00B0 0046 +210A>0067 +210B>0048 +210C>0048 +210D>0048 +210E>0068 +210F>0127 +2110>0049 +2111>0049 +2112>004C +2113>006C +2115>004E +2116>004E 006F +2119>0050 +211A>0051 +211B>0052 +211C>0052 +211D>0052 +2120>0053 004D +2121>0054 0045 004C +2122>0054 004D +2124>005A +2126>03A9 +2128>005A +212A>004B +212B>00C5 +212C>0042 +212D>0043 +212F>0065 +2130>0045 +2131>0046 +2133>004D +2134>006F +2135>05D0 +2136>05D1 +2137>05D2 +2138>05D3 +2139>0069 +213B>0046 0041 0058 +213C>03C0 +213D>03B3 +213E>0393 +213F>03A0 +2140>2211 +2145>0044 +2146>0064 +2147>0065 +2148>0069 +2149>006A +2150>0031 2044 0037 +2151>0031 2044 0039 +2152>0031 2044 0031 0030 +2153>0031 2044 0033 +2154>0032 2044 0033 +2155>0031 2044 0035 +2156>0032 2044 0035 +2157>0033 2044 0035 +2158>0034 2044 0035 +2159>0031 2044 0036 +215A>0035 2044 0036 +215B>0031 2044 0038 +215C>0033 2044 0038 +215D>0035 2044 0038 +215E>0037 2044 0038 +215F>0031 2044 +2160>0049 +2161>0049 0049 +2162>0049 0049 0049 +2163>0049 0056 +2164>0056 +2165>0056 0049 +2166>0056 0049 0049 +2167>0056 0049 0049 0049 +2168>0049 0058 +2169>0058 +216A>0058 0049 +216B>0058 0049 0049 +216C>004C +216D>0043 +216E>0044 +216F>004D +2170>0069 +2171>0069 0069 +2172>0069 0069 0069 +2173>0069 0076 +2174>0076 +2175>0076 0069 +2176>0076 0069 0069 +2177>0076 0069 0069 0069 +2178>0069 0078 +2179>0078 +217A>0078 0069 +217B>0078 0069 0069 +217C>006C +217D>0063 +217E>0064 +217F>006D +2189>0030 2044 0033 +219A=2190 0338 +219B=2192 0338 +21AE=2194 0338 +21CD=21D0 0338 +21CE=21D4 0338 +21CF=21D2 0338 +2204=2203 0338 +2209=2208 0338 +220C=220B 0338 +2224=2223 0338 +2226=2225 0338 +222C>222B 222B +222D>222B 222B 222B +222F>222E 222E +2230>222E 222E 222E +2241=223C 0338 +2244=2243 0338 +2247=2245 0338 +2249=2248 0338 +2260=003D 0338 +2262=2261 0338 +226D=224D 0338 +226E=003C 0338 +226F=003E 0338 +2270=2264 0338 +2271=2265 0338 +2274=2272 0338 +2275=2273 0338 +2278=2276 0338 +2279=2277 0338 +2280=227A 0338 +2281=227B 0338 +2284=2282 0338 +2285=2283 0338 +2288=2286 0338 +2289=2287 0338 +22AC=22A2 0338 +22AD=22A8 0338 +22AE=22A9 0338 +22AF=22AB 0338 +22E0=227C 0338 +22E1=227D 0338 +22E2=2291 0338 +22E3=2292 0338 +22EA=22B2 0338 +22EB=22B3 0338 +22EC=22B4 0338 +22ED=22B5 0338 +2329>3008 +232A>3009 +2460>0031 +2461>0032 +2462>0033 +2463>0034 +2464>0035 +2465>0036 +2466>0037 +2467>0038 +2468>0039 +2469>0031 0030 +246A>0031 0031 +246B>0031 0032 +246C>0031 0033 +246D>0031 0034 +246E>0031 0035 +246F>0031 0036 +2470>0031 0037 +2471>0031 0038 +2472>0031 0039 +2473>0032 0030 +2474>0028 0031 0029 +2475>0028 0032 0029 +2476>0028 0033 0029 +2477>0028 0034 0029 +2478>0028 0035 0029 +2479>0028 0036 0029 +247A>0028 0037 0029 +247B>0028 0038 0029 +247C>0028 0039 0029 +247D>0028 0031 0030 0029 +247E>0028 0031 0031 0029 +247F>0028 0031 0032 0029 +2480>0028 0031 0033 0029 +2481>0028 0031 0034 0029 +2482>0028 0031 0035 0029 +2483>0028 0031 0036 0029 +2484>0028 0031 0037 0029 +2485>0028 0031 0038 0029 +2486>0028 0031 0039 0029 +2487>0028 0032 0030 0029 +2488>0031 002E +2489>0032 002E +248A>0033 002E +248B>0034 002E +248C>0035 002E +248D>0036 002E +248E>0037 002E +248F>0038 002E +2490>0039 002E +2491>0031 0030 002E +2492>0031 0031 002E +2493>0031 0032 002E +2494>0031 0033 002E +2495>0031 0034 002E +2496>0031 0035 002E +2497>0031 0036 002E +2498>0031 0037 002E +2499>0031 0038 002E +249A>0031 0039 002E +249B>0032 0030 002E +249C>0028 0061 0029 +249D>0028 0062 0029 +249E>0028 0063 0029 +249F>0028 0064 0029 +24A0>0028 0065 0029 +24A1>0028 0066 0029 +24A2>0028 0067 0029 +24A3>0028 0068 0029 +24A4>0028 0069 0029 +24A5>0028 006A 0029 +24A6>0028 006B 0029 +24A7>0028 006C 0029 +24A8>0028 006D 0029 +24A9>0028 006E 0029 +24AA>0028 006F 0029 +24AB>0028 0070 0029 +24AC>0028 0071 0029 +24AD>0028 0072 0029 +24AE>0028 0073 0029 +24AF>0028 0074 0029 +24B0>0028 0075 0029 +24B1>0028 0076 0029 +24B2>0028 0077 0029 +24B3>0028 0078 0029 +24B4>0028 0079 0029 +24B5>0028 007A 0029 +24B6>0041 +24B7>0042 +24B8>0043 +24B9>0044 +24BA>0045 +24BB>0046 +24BC>0047 +24BD>0048 +24BE>0049 +24BF>004A +24C0>004B +24C1>004C +24C2>004D +24C3>004E +24C4>004F +24C5>0050 +24C6>0051 +24C7>0052 +24C8>0053 +24C9>0054 +24CA>0055 +24CB>0056 +24CC>0057 +24CD>0058 +24CE>0059 +24CF>005A +24D0>0061 +24D1>0062 +24D2>0063 +24D3>0064 +24D4>0065 +24D5>0066 +24D6>0067 +24D7>0068 +24D8>0069 +24D9>006A +24DA>006B +24DB>006C +24DC>006D +24DD>006E +24DE>006F +24DF>0070 +24E0>0071 +24E1>0072 +24E2>0073 +24E3>0074 +24E4>0075 +24E5>0076 +24E6>0077 +24E7>0078 +24E8>0079 +24E9>007A +24EA>0030 +2A0C>222B 222B 222B 222B +2A74>003A 003A 003D +2A75>003D 003D +2A76>003D 003D 003D +2ADC>2ADD 0338 +2C7C>006A +2C7D>0056 +2D6F>2D61 +2E9F>6BCD +2EF3>9F9F +2F00>4E00 +2F01>4E28 +2F02>4E36 +2F03>4E3F +2F04>4E59 +2F05>4E85 +2F06>4E8C +2F07>4EA0 +2F08>4EBA +2F09>513F +2F0A>5165 +2F0B>516B +2F0C>5182 +2F0D>5196 +2F0E>51AB +2F0F>51E0 +2F10>51F5 +2F11>5200 +2F12>529B +2F13>52F9 +2F14>5315 +2F15>531A +2F16>5338 +2F17>5341 +2F18>535C +2F19>5369 +2F1A>5382 +2F1B>53B6 +2F1C>53C8 +2F1D>53E3 +2F1E>56D7 +2F1F>571F +2F20>58EB +2F21>5902 +2F22>590A +2F23>5915 +2F24>5927 +2F25>5973 +2F26>5B50 +2F27>5B80 +2F28>5BF8 +2F29>5C0F +2F2A>5C22 +2F2B>5C38 +2F2C>5C6E +2F2D>5C71 +2F2E>5DDB +2F2F>5DE5 +2F30>5DF1 +2F31>5DFE +2F32>5E72 +2F33>5E7A +2F34>5E7F +2F35>5EF4 +2F36>5EFE +2F37>5F0B +2F38>5F13 +2F39>5F50 +2F3A>5F61 +2F3B>5F73 +2F3C>5FC3 +2F3D>6208 +2F3E>6236 +2F3F>624B +2F40>652F +2F41>6534 +2F42>6587 +2F43>6597 +2F44>65A4 +2F45>65B9 +2F46>65E0 +2F47>65E5 +2F48>66F0 +2F49>6708 +2F4A>6728 +2F4B>6B20 +2F4C>6B62 +2F4D>6B79 +2F4E>6BB3 +2F4F>6BCB +2F50>6BD4 +2F51>6BDB +2F52>6C0F +2F53>6C14 +2F54>6C34 +2F55>706B +2F56>722A +2F57>7236 +2F58>723B +2F59>723F +2F5A>7247 +2F5B>7259 +2F5C>725B +2F5D>72AC +2F5E>7384 +2F5F>7389 +2F60>74DC +2F61>74E6 +2F62>7518 +2F63>751F +2F64>7528 +2F65>7530 +2F66>758B +2F67>7592 +2F68>7676 +2F69>767D +2F6A>76AE +2F6B>76BF +2F6C>76EE +2F6D>77DB +2F6E>77E2 +2F6F>77F3 +2F70>793A +2F71>79B8 +2F72>79BE +2F73>7A74 +2F74>7ACB +2F75>7AF9 +2F76>7C73 +2F77>7CF8 +2F78>7F36 +2F79>7F51 +2F7A>7F8A +2F7B>7FBD +2F7C>8001 +2F7D>800C +2F7E>8012 +2F7F>8033 +2F80>807F +2F81>8089 +2F82>81E3 +2F83>81EA +2F84>81F3 +2F85>81FC +2F86>820C +2F87>821B +2F88>821F +2F89>826E +2F8A>8272 +2F8B>8278 +2F8C>864D +2F8D>866B +2F8E>8840 +2F8F>884C +2F90>8863 +2F91>897E +2F92>898B +2F93>89D2 +2F94>8A00 +2F95>8C37 +2F96>8C46 +2F97>8C55 +2F98>8C78 +2F99>8C9D +2F9A>8D64 +2F9B>8D70 +2F9C>8DB3 +2F9D>8EAB +2F9E>8ECA +2F9F>8F9B +2FA0>8FB0 +2FA1>8FB5 +2FA2>9091 +2FA3>9149 +2FA4>91C6 +2FA5>91CC +2FA6>91D1 +2FA7>9577 +2FA8>9580 +2FA9>961C +2FAA>96B6 +2FAB>96B9 +2FAC>96E8 +2FAD>9751 +2FAE>975E +2FAF>9762 +2FB0>9769 +2FB1>97CB +2FB2>97ED +2FB3>97F3 +2FB4>9801 +2FB5>98A8 +2FB6>98DB +2FB7>98DF +2FB8>9996 +2FB9>9999 +2FBA>99AC +2FBB>9AA8 +2FBC>9AD8 +2FBD>9ADF +2FBE>9B25 +2FBF>9B2F +2FC0>9B32 +2FC1>9B3C +2FC2>9B5A +2FC3>9CE5 +2FC4>9E75 +2FC5>9E7F +2FC6>9EA5 +2FC7>9EBB +2FC8>9EC3 +2FC9>9ECD +2FCA>9ED1 +2FCB>9EF9 +2FCC>9EFD +2FCD>9F0E +2FCE>9F13 +2FCF>9F20 +2FD0>9F3B +2FD1>9F4A +2FD2>9F52 +2FD3>9F8D +2FD4>9F9C +2FD5>9FA0 +3000>0020 +3036>3012 +3038>5341 +3039>5344 +303A>5345 +304C=304B 3099 +304E=304D 3099 +3050=304F 3099 +3052=3051 3099 +3054=3053 3099 +3056=3055 3099 +3058=3057 3099 +305A=3059 3099 +305C=305B 3099 +305E=305D 3099 +3060=305F 3099 +3062=3061 3099 +3065=3064 3099 +3067=3066 3099 +3069=3068 3099 +3070=306F 3099 +3071=306F 309A +3073=3072 3099 +3074=3072 309A +3076=3075 3099 +3077=3075 309A +3079=3078 3099 +307A=3078 309A +307C=307B 3099 +307D=307B 309A +3094=3046 3099 +309B>0020 3099 +309C>0020 309A +309E=309D 3099 +309F>3088 308A +30AC=30AB 3099 +30AE=30AD 3099 +30B0=30AF 3099 +30B2=30B1 3099 +30B4=30B3 3099 +30B6=30B5 3099 +30B8=30B7 3099 +30BA=30B9 3099 +30BC=30BB 3099 +30BE=30BD 3099 +30C0=30BF 3099 +30C2=30C1 3099 +30C5=30C4 3099 +30C7=30C6 3099 +30C9=30C8 3099 +30D0=30CF 3099 +30D1=30CF 309A +30D3=30D2 3099 +30D4=30D2 309A +30D6=30D5 3099 +30D7=30D5 309A +30D9=30D8 3099 +30DA=30D8 309A +30DC=30DB 3099 +30DD=30DB 309A +30F4=30A6 3099 +30F7=30EF 3099 +30F8=30F0 3099 +30F9=30F1 3099 +30FA=30F2 3099 +30FE=30FD 3099 +30FF>30B3 30C8 +3131>1100 +3132>1101 +3133>11AA +3134>1102 +3135>11AC +3136>11AD +3137>1103 +3138>1104 +3139>1105 +313A>11B0 +313B>11B1 +313C>11B2 +313D>11B3 +313E>11B4 +313F>11B5 +3140>111A +3141>1106 +3142>1107 +3143>1108 +3144>1121 +3145>1109 +3146>110A +3147>110B +3148>110C +3149>110D +314A>110E +314B>110F +314C>1110 +314D>1111 +314E>1112 +314F>1161 +3150>1162 +3151>1163 +3152>1164 +3153>1165 +3154>1166 +3155>1167 +3156>1168 +3157>1169 +3158>116A +3159>116B +315A>116C +315B>116D +315C>116E +315D>116F +315E>1170 +315F>1171 +3160>1172 +3161>1173 +3162>1174 +3163>1175 +3164>1160 +3165>1114 +3166>1115 +3167>11C7 +3168>11C8 +3169>11CC +316A>11CE +316B>11D3 +316C>11D7 +316D>11D9 +316E>111C +316F>11DD +3170>11DF +3171>111D +3172>111E +3173>1120 +3174>1122 +3175>1123 +3176>1127 +3177>1129 +3178>112B +3179>112C +317A>112D +317B>112E +317C>112F +317D>1132 +317E>1136 +317F>1140 +3180>1147 +3181>114C +3182>11F1 +3183>11F2 +3184>1157 +3185>1158 +3186>1159 +3187>1184 +3188>1185 +3189>1188 +318A>1191 +318B>1192 +318C>1194 +318D>119E +318E>11A1 +3192>4E00 +3193>4E8C +3194>4E09 +3195>56DB +3196>4E0A +3197>4E2D +3198>4E0B +3199>7532 +319A>4E59 +319B>4E19 +319C>4E01 +319D>5929 +319E>5730 +319F>4EBA +3200>0028 1100 0029 +3201>0028 1102 0029 +3202>0028 1103 0029 +3203>0028 1105 0029 +3204>0028 1106 0029 +3205>0028 1107 0029 +3206>0028 1109 0029 +3207>0028 110B 0029 +3208>0028 110C 0029 +3209>0028 110E 0029 +320A>0028 110F 0029 +320B>0028 1110 0029 +320C>0028 1111 0029 +320D>0028 1112 0029 +320E>0028 1100 1161 0029 +320F>0028 1102 1161 0029 +3210>0028 1103 1161 0029 +3211>0028 1105 1161 0029 +3212>0028 1106 1161 0029 +3213>0028 1107 1161 0029 +3214>0028 1109 1161 0029 +3215>0028 110B 1161 0029 +3216>0028 110C 1161 0029 +3217>0028 110E 1161 0029 +3218>0028 110F 1161 0029 +3219>0028 1110 1161 0029 +321A>0028 1111 1161 0029 +321B>0028 1112 1161 0029 +321C>0028 110C 116E 0029 +321D>0028 110B 1169 110C 1165 11AB 0029 +321E>0028 110B 1169 1112 116E 0029 +3220>0028 4E00 0029 +3221>0028 4E8C 0029 +3222>0028 4E09 0029 +3223>0028 56DB 0029 +3224>0028 4E94 0029 +3225>0028 516D 0029 +3226>0028 4E03 0029 +3227>0028 516B 0029 +3228>0028 4E5D 0029 +3229>0028 5341 0029 +322A>0028 6708 0029 +322B>0028 706B 0029 +322C>0028 6C34 0029 +322D>0028 6728 0029 +322E>0028 91D1 0029 +322F>0028 571F 0029 +3230>0028 65E5 0029 +3231>0028 682A 0029 +3232>0028 6709 0029 +3233>0028 793E 0029 +3234>0028 540D 0029 +3235>0028 7279 0029 +3236>0028 8CA1 0029 +3237>0028 795D 0029 +3238>0028 52B4 0029 +3239>0028 4EE3 0029 +323A>0028 547C 0029 +323B>0028 5B66 0029 +323C>0028 76E3 0029 +323D>0028 4F01 0029 +323E>0028 8CC7 0029 +323F>0028 5354 0029 +3240>0028 796D 0029 +3241>0028 4F11 0029 +3242>0028 81EA 0029 +3243>0028 81F3 0029 +3244>554F +3245>5E7C +3246>6587 +3247>7B8F +3250>0050 0054 0045 +3251>0032 0031 +3252>0032 0032 +3253>0032 0033 +3254>0032 0034 +3255>0032 0035 +3256>0032 0036 +3257>0032 0037 +3258>0032 0038 +3259>0032 0039 +325A>0033 0030 +325B>0033 0031 +325C>0033 0032 +325D>0033 0033 +325E>0033 0034 +325F>0033 0035 +3260>1100 +3261>1102 +3262>1103 +3263>1105 +3264>1106 +3265>1107 +3266>1109 +3267>110B +3268>110C +3269>110E +326A>110F +326B>1110 +326C>1111 +326D>1112 +326E>1100 1161 +326F>1102 1161 +3270>1103 1161 +3271>1105 1161 +3272>1106 1161 +3273>1107 1161 +3274>1109 1161 +3275>110B 1161 +3276>110C 1161 +3277>110E 1161 +3278>110F 1161 +3279>1110 1161 +327A>1111 1161 +327B>1112 1161 +327C>110E 1161 11B7 1100 1169 +327D>110C 116E 110B 1174 +327E>110B 116E +3280>4E00 +3281>4E8C +3282>4E09 +3283>56DB +3284>4E94 +3285>516D +3286>4E03 +3287>516B +3288>4E5D +3289>5341 +328A>6708 +328B>706B +328C>6C34 +328D>6728 +328E>91D1 +328F>571F +3290>65E5 +3291>682A +3292>6709 +3293>793E +3294>540D +3295>7279 +3296>8CA1 +3297>795D +3298>52B4 +3299>79D8 +329A>7537 +329B>5973 +329C>9069 +329D>512A +329E>5370 +329F>6CE8 +32A0>9805 +32A1>4F11 +32A2>5199 +32A3>6B63 +32A4>4E0A +32A5>4E2D +32A6>4E0B +32A7>5DE6 +32A8>53F3 +32A9>533B +32AA>5B97 +32AB>5B66 +32AC>76E3 +32AD>4F01 +32AE>8CC7 +32AF>5354 +32B0>591C +32B1>0033 0036 +32B2>0033 0037 +32B3>0033 0038 +32B4>0033 0039 +32B5>0034 0030 +32B6>0034 0031 +32B7>0034 0032 +32B8>0034 0033 +32B9>0034 0034 +32BA>0034 0035 +32BB>0034 0036 +32BC>0034 0037 +32BD>0034 0038 +32BE>0034 0039 +32BF>0035 0030 +32C0>0031 6708 +32C1>0032 6708 +32C2>0033 6708 +32C3>0034 6708 +32C4>0035 6708 +32C5>0036 6708 +32C6>0037 6708 +32C7>0038 6708 +32C8>0039 6708 +32C9>0031 0030 6708 +32CA>0031 0031 6708 +32CB>0031 0032 6708 +32CC>0048 0067 +32CD>0065 0072 0067 +32CE>0065 0056 +32CF>004C 0054 0044 +32D0>30A2 +32D1>30A4 +32D2>30A6 +32D3>30A8 +32D4>30AA +32D5>30AB +32D6>30AD +32D7>30AF +32D8>30B1 +32D9>30B3 +32DA>30B5 +32DB>30B7 +32DC>30B9 +32DD>30BB +32DE>30BD +32DF>30BF +32E0>30C1 +32E1>30C4 +32E2>30C6 +32E3>30C8 +32E4>30CA +32E5>30CB +32E6>30CC +32E7>30CD +32E8>30CE +32E9>30CF +32EA>30D2 +32EB>30D5 +32EC>30D8 +32ED>30DB +32EE>30DE +32EF>30DF +32F0>30E0 +32F1>30E1 +32F2>30E2 +32F3>30E4 +32F4>30E6 +32F5>30E8 +32F6>30E9 +32F7>30EA +32F8>30EB +32F9>30EC +32FA>30ED +32FB>30EF +32FC>30F0 +32FD>30F1 +32FE>30F2 +3300>30A2 30D1 30FC 30C8 +3301>30A2 30EB 30D5 30A1 +3302>30A2 30F3 30DA 30A2 +3303>30A2 30FC 30EB +3304>30A4 30CB 30F3 30B0 +3305>30A4 30F3 30C1 +3306>30A6 30A9 30F3 +3307>30A8 30B9 30AF 30FC 30C9 +3308>30A8 30FC 30AB 30FC +3309>30AA 30F3 30B9 +330A>30AA 30FC 30E0 +330B>30AB 30A4 30EA +330C>30AB 30E9 30C3 30C8 +330D>30AB 30ED 30EA 30FC +330E>30AC 30ED 30F3 +330F>30AC 30F3 30DE +3310>30AE 30AC +3311>30AE 30CB 30FC +3312>30AD 30E5 30EA 30FC +3313>30AE 30EB 30C0 30FC +3314>30AD 30ED +3315>30AD 30ED 30B0 30E9 30E0 +3316>30AD 30ED 30E1 30FC 30C8 30EB +3317>30AD 30ED 30EF 30C3 30C8 +3318>30B0 30E9 30E0 +3319>30B0 30E9 30E0 30C8 30F3 +331A>30AF 30EB 30BC 30A4 30ED +331B>30AF 30ED 30FC 30CD +331C>30B1 30FC 30B9 +331D>30B3 30EB 30CA +331E>30B3 30FC 30DD +331F>30B5 30A4 30AF 30EB +3320>30B5 30F3 30C1 30FC 30E0 +3321>30B7 30EA 30F3 30B0 +3322>30BB 30F3 30C1 +3323>30BB 30F3 30C8 +3324>30C0 30FC 30B9 +3325>30C7 30B7 +3326>30C9 30EB +3327>30C8 30F3 +3328>30CA 30CE +3329>30CE 30C3 30C8 +332A>30CF 30A4 30C4 +332B>30D1 30FC 30BB 30F3 30C8 +332C>30D1 30FC 30C4 +332D>30D0 30FC 30EC 30EB +332E>30D4 30A2 30B9 30C8 30EB +332F>30D4 30AF 30EB +3330>30D4 30B3 +3331>30D3 30EB +3332>30D5 30A1 30E9 30C3 30C9 +3333>30D5 30A3 30FC 30C8 +3334>30D6 30C3 30B7 30A7 30EB +3335>30D5 30E9 30F3 +3336>30D8 30AF 30BF 30FC 30EB +3337>30DA 30BD +3338>30DA 30CB 30D2 +3339>30D8 30EB 30C4 +333A>30DA 30F3 30B9 +333B>30DA 30FC 30B8 +333C>30D9 30FC 30BF +333D>30DD 30A4 30F3 30C8 +333E>30DC 30EB 30C8 +333F>30DB 30F3 +3340>30DD 30F3 30C9 +3341>30DB 30FC 30EB +3342>30DB 30FC 30F3 +3343>30DE 30A4 30AF 30ED +3344>30DE 30A4 30EB +3345>30DE 30C3 30CF +3346>30DE 30EB 30AF +3347>30DE 30F3 30B7 30E7 30F3 +3348>30DF 30AF 30ED 30F3 +3349>30DF 30EA +334A>30DF 30EA 30D0 30FC 30EB +334B>30E1 30AC +334C>30E1 30AC 30C8 30F3 +334D>30E1 30FC 30C8 30EB +334E>30E4 30FC 30C9 +334F>30E4 30FC 30EB +3350>30E6 30A2 30F3 +3351>30EA 30C3 30C8 30EB +3352>30EA 30E9 +3353>30EB 30D4 30FC +3354>30EB 30FC 30D6 30EB +3355>30EC 30E0 +3356>30EC 30F3 30C8 30B2 30F3 +3357>30EF 30C3 30C8 +3358>0030 70B9 +3359>0031 70B9 +335A>0032 70B9 +335B>0033 70B9 +335C>0034 70B9 +335D>0035 70B9 +335E>0036 70B9 +335F>0037 70B9 +3360>0038 70B9 +3361>0039 70B9 +3362>0031 0030 70B9 +3363>0031 0031 70B9 +3364>0031 0032 70B9 +3365>0031 0033 70B9 +3366>0031 0034 70B9 +3367>0031 0035 70B9 +3368>0031 0036 70B9 +3369>0031 0037 70B9 +336A>0031 0038 70B9 +336B>0031 0039 70B9 +336C>0032 0030 70B9 +336D>0032 0031 70B9 +336E>0032 0032 70B9 +336F>0032 0033 70B9 +3370>0032 0034 70B9 +3371>0068 0050 0061 +3372>0064 0061 +3373>0041 0055 +3374>0062 0061 0072 +3375>006F 0056 +3376>0070 0063 +3377>0064 006D +3378>0064 006D 00B2 +3379>0064 006D 00B3 +337A>0049 0055 +337B>5E73 6210 +337C>662D 548C +337D>5927 6B63 +337E>660E 6CBB +337F>682A 5F0F 4F1A 793E +3380>0070 0041 +3381>006E 0041 +3382>03BC 0041 +3383>006D 0041 +3384>006B 0041 +3385>004B 0042 +3386>004D 0042 +3387>0047 0042 +3388>0063 0061 006C +3389>006B 0063 0061 006C +338A>0070 0046 +338B>006E 0046 +338C>03BC 0046 +338D>03BC 0067 +338E>006D 0067 +338F>006B 0067 +3390>0048 007A +3391>006B 0048 007A +3392>004D 0048 007A +3393>0047 0048 007A +3394>0054 0048 007A +3395>03BC 2113 +3396>006D 2113 +3397>0064 2113 +3398>006B 2113 +3399>0066 006D +339A>006E 006D +339B>03BC 006D +339C>006D 006D +339D>0063 006D +339E>006B 006D +339F>006D 006D 00B2 +33A0>0063 006D 00B2 +33A1>006D 00B2 +33A2>006B 006D 00B2 +33A3>006D 006D 00B3 +33A4>0063 006D 00B3 +33A5>006D 00B3 +33A6>006B 006D 00B3 +33A7>006D 2215 0073 +33A8>006D 2215 0073 00B2 +33A9>0050 0061 +33AA>006B 0050 0061 +33AB>004D 0050 0061 +33AC>0047 0050 0061 +33AD>0072 0061 0064 +33AE>0072 0061 0064 2215 0073 +33AF>0072 0061 0064 2215 0073 00B2 +33B0>0070 0073 +33B1>006E 0073 +33B2>03BC 0073 +33B3>006D 0073 +33B4>0070 0056 +33B5>006E 0056 +33B6>03BC 0056 +33B7>006D 0056 +33B8>006B 0056 +33B9>004D 0056 +33BA>0070 0057 +33BB>006E 0057 +33BC>03BC 0057 +33BD>006D 0057 +33BE>006B 0057 +33BF>004D 0057 +33C0>006B 03A9 +33C1>004D 03A9 +33C2>0061 002E 006D 002E +33C3>0042 0071 +33C4>0063 0063 +33C5>0063 0064 +33C6>0043 2215 006B 0067 +33C7>0043 006F 002E +33C8>0064 0042 +33C9>0047 0079 +33CA>0068 0061 +33CB>0048 0050 +33CC>0069 006E +33CD>004B 004B +33CE>004B 004D +33CF>006B 0074 +33D0>006C 006D +33D1>006C 006E +33D2>006C 006F 0067 +33D3>006C 0078 +33D4>006D 0062 +33D5>006D 0069 006C +33D6>006D 006F 006C +33D7>0050 0048 +33D8>0070 002E 006D 002E +33D9>0050 0050 004D +33DA>0050 0052 +33DB>0073 0072 +33DC>0053 0076 +33DD>0057 0062 +33DE>0056 2215 006D +33DF>0041 2215 006D +33E0>0031 65E5 +33E1>0032 65E5 +33E2>0033 65E5 +33E3>0034 65E5 +33E4>0035 65E5 +33E5>0036 65E5 +33E6>0037 65E5 +33E7>0038 65E5 +33E8>0039 65E5 +33E9>0031 0030 65E5 +33EA>0031 0031 65E5 +33EB>0031 0032 65E5 +33EC>0031 0033 65E5 +33ED>0031 0034 65E5 +33EE>0031 0035 65E5 +33EF>0031 0036 65E5 +33F0>0031 0037 65E5 +33F1>0031 0038 65E5 +33F2>0031 0039 65E5 +33F3>0032 0030 65E5 +33F4>0032 0031 65E5 +33F5>0032 0032 65E5 +33F6>0032 0033 65E5 +33F7>0032 0034 65E5 +33F8>0032 0035 65E5 +33F9>0032 0036 65E5 +33FA>0032 0037 65E5 +33FB>0032 0038 65E5 +33FC>0032 0039 65E5 +33FD>0033 0030 65E5 +33FE>0033 0031 65E5 +33FF>0067 0061 006C +A770>A76F +F900>8C48 +F901>66F4 +F902>8ECA +F903>8CC8 +F904>6ED1 +F905>4E32 +F906>53E5 +F907>9F9C +F908>9F9C +F909>5951 +F90A>91D1 +F90B>5587 +F90C>5948 +F90D>61F6 +F90E>7669 +F90F>7F85 +F910>863F +F911>87BA +F912>88F8 +F913>908F +F914>6A02 +F915>6D1B +F916>70D9 +F917>73DE +F918>843D +F919>916A +F91A>99F1 +F91B>4E82 +F91C>5375 +F91D>6B04 +F91E>721B +F91F>862D +F920>9E1E +F921>5D50 +F922>6FEB +F923>85CD +F924>8964 +F925>62C9 +F926>81D8 +F927>881F +F928>5ECA +F929>6717 +F92A>6D6A +F92B>72FC +F92C>90CE +F92D>4F86 +F92E>51B7 +F92F>52DE +F930>64C4 +F931>6AD3 +F932>7210 +F933>76E7 +F934>8001 +F935>8606 +F936>865C +F937>8DEF +F938>9732 +F939>9B6F +F93A>9DFA +F93B>788C +F93C>797F +F93D>7DA0 +F93E>83C9 +F93F>9304 +F940>9E7F +F941>8AD6 +F942>58DF +F943>5F04 +F944>7C60 +F945>807E +F946>7262 +F947>78CA +F948>8CC2 +F949>96F7 +F94A>58D8 +F94B>5C62 +F94C>6A13 +F94D>6DDA +F94E>6F0F +F94F>7D2F +F950>7E37 +F951>964B +F952>52D2 +F953>808B +F954>51DC +F955>51CC +F956>7A1C +F957>7DBE +F958>83F1 +F959>9675 +F95A>8B80 +F95B>62CF +F95C>6A02 +F95D>8AFE +F95E>4E39 +F95F>5BE7 +F960>6012 +F961>7387 +F962>7570 +F963>5317 +F964>78FB +F965>4FBF +F966>5FA9 +F967>4E0D +F968>6CCC +F969>6578 +F96A>7D22 +F96B>53C3 +F96C>585E +F96D>7701 +F96E>8449 +F96F>8AAA +F970>6BBA +F971>8FB0 +F972>6C88 +F973>62FE +F974>82E5 +F975>63A0 +F976>7565 +F977>4EAE +F978>5169 +F979>51C9 +F97A>6881 +F97B>7CE7 +F97C>826F +F97D>8AD2 +F97E>91CF +F97F>52F5 +F980>5442 +F981>5973 +F982>5EEC +F983>65C5 +F984>6FFE +F985>792A +F986>95AD +F987>9A6A +F988>9E97 +F989>9ECE +F98A>529B +F98B>66C6 +F98C>6B77 +F98D>8F62 +F98E>5E74 +F98F>6190 +F990>6200 +F991>649A +F992>6F23 +F993>7149 +F994>7489 +F995>79CA +F996>7DF4 +F997>806F +F998>8F26 +F999>84EE +F99A>9023 +F99B>934A +F99C>5217 +F99D>52A3 +F99E>54BD +F99F>70C8 +F9A0>88C2 +F9A1>8AAA +F9A2>5EC9 +F9A3>5FF5 +F9A4>637B +F9A5>6BAE +F9A6>7C3E +F9A7>7375 +F9A8>4EE4 +F9A9>56F9 +F9AA>5BE7 +F9AB>5DBA +F9AC>601C +F9AD>73B2 +F9AE>7469 +F9AF>7F9A +F9B0>8046 +F9B1>9234 +F9B2>96F6 +F9B3>9748 +F9B4>9818 +F9B5>4F8B +F9B6>79AE +F9B7>91B4 +F9B8>96B8 +F9B9>60E1 +F9BA>4E86 +F9BB>50DA +F9BC>5BEE +F9BD>5C3F +F9BE>6599 +F9BF>6A02 +F9C0>71CE +F9C1>7642 +F9C2>84FC +F9C3>907C +F9C4>9F8D +F9C5>6688 +F9C6>962E +F9C7>5289 +F9C8>677B +F9C9>67F3 +F9CA>6D41 +F9CB>6E9C +F9CC>7409 +F9CD>7559 +F9CE>786B +F9CF>7D10 +F9D0>985E +F9D1>516D +F9D2>622E +F9D3>9678 +F9D4>502B +F9D5>5D19 +F9D6>6DEA +F9D7>8F2A +F9D8>5F8B +F9D9>6144 +F9DA>6817 +F9DB>7387 +F9DC>9686 +F9DD>5229 +F9DE>540F +F9DF>5C65 +F9E0>6613 +F9E1>674E +F9E2>68A8 +F9E3>6CE5 +F9E4>7406 +F9E5>75E2 +F9E6>7F79 +F9E7>88CF +F9E8>88E1 +F9E9>91CC +F9EA>96E2 +F9EB>533F +F9EC>6EBA +F9ED>541D +F9EE>71D0 +F9EF>7498 +F9F0>85FA +F9F1>96A3 +F9F2>9C57 +F9F3>9E9F +F9F4>6797 +F9F5>6DCB +F9F6>81E8 +F9F7>7ACB +F9F8>7B20 +F9F9>7C92 +F9FA>72C0 +F9FB>7099 +F9FC>8B58 +F9FD>4EC0 +F9FE>8336 +F9FF>523A +FA00>5207 +FA01>5EA6 +FA02>62D3 +FA03>7CD6 +FA04>5B85 +FA05>6D1E +FA06>66B4 +FA07>8F3B +FA08>884C +FA09>964D +FA0A>898B +FA0B>5ED3 +FA0C>5140 +FA0D>55C0 +FA10>585A +FA12>6674 +FA15>51DE +FA16>732A +FA17>76CA +FA18>793C +FA19>795E +FA1A>7965 +FA1B>798F +FA1C>9756 +FA1D>7CBE +FA1E>7FBD +FA20>8612 +FA22>8AF8 +FA25>9038 +FA26>90FD +FA2A>98EF +FA2B>98FC +FA2C>9928 +FA2D>9DB4 +FA30>4FAE +FA31>50E7 +FA32>514D +FA33>52C9 +FA34>52E4 +FA35>5351 +FA36>559D +FA37>5606 +FA38>5668 +FA39>5840 +FA3A>58A8 +FA3B>5C64 +FA3C>5C6E +FA3D>6094 +FA3E>6168 +FA3F>618E +FA40>61F2 +FA41>654F +FA42>65E2 +FA43>6691 +FA44>6885 +FA45>6D77 +FA46>6E1A +FA47>6F22 +FA48>716E +FA49>722B +FA4A>7422 +FA4B>7891 +FA4C>793E +FA4D>7949 +FA4E>7948 +FA4F>7950 +FA50>7956 +FA51>795D +FA52>798D +FA53>798E +FA54>7A40 +FA55>7A81 +FA56>7BC0 +FA57>7DF4 +FA58>7E09 +FA59>7E41 +FA5A>7F72 +FA5B>8005 +FA5C>81ED +FA5D>8279 +FA5E>8279 +FA5F>8457 +FA60>8910 +FA61>8996 +FA62>8B01 +FA63>8B39 +FA64>8CD3 +FA65>8D08 +FA66>8FB6 +FA67>9038 +FA68>96E3 +FA69>97FF +FA6A>983B +FA6B>6075 +FA6C>242EE +FA6D>8218 +FA70>4E26 +FA71>51B5 +FA72>5168 +FA73>4F80 +FA74>5145 +FA75>5180 +FA76>52C7 +FA77>52FA +FA78>559D +FA79>5555 +FA7A>5599 +FA7B>55E2 +FA7C>585A +FA7D>58B3 +FA7E>5944 +FA7F>5954 +FA80>5A62 +FA81>5B28 +FA82>5ED2 +FA83>5ED9 +FA84>5F69 +FA85>5FAD +FA86>60D8 +FA87>614E +FA88>6108 +FA89>618E +FA8A>6160 +FA8B>61F2 +FA8C>6234 +FA8D>63C4 +FA8E>641C +FA8F>6452 +FA90>6556 +FA91>6674 +FA92>6717 +FA93>671B +FA94>6756 +FA95>6B79 +FA96>6BBA +FA97>6D41 +FA98>6EDB +FA99>6ECB +FA9A>6F22 +FA9B>701E +FA9C>716E +FA9D>77A7 +FA9E>7235 +FA9F>72AF +FAA0>732A +FAA1>7471 +FAA2>7506 +FAA3>753B +FAA4>761D +FAA5>761F +FAA6>76CA +FAA7>76DB +FAA8>76F4 +FAA9>774A +FAAA>7740 +FAAB>78CC +FAAC>7AB1 +FAAD>7BC0 +FAAE>7C7B +FAAF>7D5B +FAB0>7DF4 +FAB1>7F3E +FAB2>8005 +FAB3>8352 +FAB4>83EF +FAB5>8779 +FAB6>8941 +FAB7>8986 +FAB8>8996 +FAB9>8ABF +FABA>8AF8 +FABB>8ACB +FABC>8B01 +FABD>8AFE +FABE>8AED +FABF>8B39 +FAC0>8B8A +FAC1>8D08 +FAC2>8F38 +FAC3>9072 +FAC4>9199 +FAC5>9276 +FAC6>967C +FAC7>96E3 +FAC8>9756 +FAC9>97DB +FACA>97FF +FACB>980B +FACC>983B +FACD>9B12 +FACE>9F9C +FACF>2284A +FAD0>22844 +FAD1>233D5 +FAD2>3B9D +FAD3>4018 +FAD4>4039 +FAD5>25249 +FAD6>25CD0 +FAD7>27ED3 +FAD8>9F43 +FAD9>9F8E +FB00>0066 0066 +FB01>0066 0069 +FB02>0066 006C +FB03>0066 0066 0069 +FB04>0066 0066 006C +FB05>017F 0074 +FB06>0073 0074 +FB13>0574 0576 +FB14>0574 0565 +FB15>0574 056B +FB16>057E 0576 +FB17>0574 056D +FB1D>05D9 05B4 +FB1F>05F2 05B7 +FB20>05E2 +FB21>05D0 +FB22>05D3 +FB23>05D4 +FB24>05DB +FB25>05DC +FB26>05DD +FB27>05E8 +FB28>05EA +FB29>002B +FB2A>05E9 05C1 +FB2B>05E9 05C2 +FB2C>FB49 05C1 +FB2D>FB49 05C2 +FB2E>05D0 05B7 +FB2F>05D0 05B8 +FB30>05D0 05BC +FB31>05D1 05BC +FB32>05D2 05BC +FB33>05D3 05BC +FB34>05D4 05BC +FB35>05D5 05BC +FB36>05D6 05BC +FB38>05D8 05BC +FB39>05D9 05BC +FB3A>05DA 05BC +FB3B>05DB 05BC +FB3C>05DC 05BC +FB3E>05DE 05BC +FB40>05E0 05BC +FB41>05E1 05BC +FB43>05E3 05BC +FB44>05E4 05BC +FB46>05E6 05BC +FB47>05E7 05BC +FB48>05E8 05BC +FB49>05E9 05BC +FB4A>05EA 05BC +FB4B>05D5 05B9 +FB4C>05D1 05BF +FB4D>05DB 05BF +FB4E>05E4 05BF +FB4F>05D0 05DC +FB50>0671 +FB51>0671 +FB52>067B +FB53>067B +FB54>067B +FB55>067B +FB56>067E +FB57>067E +FB58>067E +FB59>067E +FB5A>0680 +FB5B>0680 +FB5C>0680 +FB5D>0680 +FB5E>067A +FB5F>067A +FB60>067A +FB61>067A +FB62>067F +FB63>067F +FB64>067F +FB65>067F +FB66>0679 +FB67>0679 +FB68>0679 +FB69>0679 +FB6A>06A4 +FB6B>06A4 +FB6C>06A4 +FB6D>06A4 +FB6E>06A6 +FB6F>06A6 +FB70>06A6 +FB71>06A6 +FB72>0684 +FB73>0684 +FB74>0684 +FB75>0684 +FB76>0683 +FB77>0683 +FB78>0683 +FB79>0683 +FB7A>0686 +FB7B>0686 +FB7C>0686 +FB7D>0686 +FB7E>0687 +FB7F>0687 +FB80>0687 +FB81>0687 +FB82>068D +FB83>068D +FB84>068C +FB85>068C +FB86>068E +FB87>068E +FB88>0688 +FB89>0688 +FB8A>0698 +FB8B>0698 +FB8C>0691 +FB8D>0691 +FB8E>06A9 +FB8F>06A9 +FB90>06A9 +FB91>06A9 +FB92>06AF +FB93>06AF +FB94>06AF +FB95>06AF +FB96>06B3 +FB97>06B3 +FB98>06B3 +FB99>06B3 +FB9A>06B1 +FB9B>06B1 +FB9C>06B1 +FB9D>06B1 +FB9E>06BA +FB9F>06BA +FBA0>06BB +FBA1>06BB +FBA2>06BB +FBA3>06BB +FBA4>06C0 +FBA5>06C0 +FBA6>06C1 +FBA7>06C1 +FBA8>06C1 +FBA9>06C1 +FBAA>06BE +FBAB>06BE +FBAC>06BE +FBAD>06BE +FBAE>06D2 +FBAF>06D2 +FBB0>06D3 +FBB1>06D3 +FBD3>06AD +FBD4>06AD +FBD5>06AD +FBD6>06AD +FBD7>06C7 +FBD8>06C7 +FBD9>06C6 +FBDA>06C6 +FBDB>06C8 +FBDC>06C8 +FBDD>0677 +FBDE>06CB +FBDF>06CB +FBE0>06C5 +FBE1>06C5 +FBE2>06C9 +FBE3>06C9 +FBE4>06D0 +FBE5>06D0 +FBE6>06D0 +FBE7>06D0 +FBE8>0649 +FBE9>0649 +FBEA>0626 0627 +FBEB>0626 0627 +FBEC>0626 06D5 +FBED>0626 06D5 +FBEE>0626 0648 +FBEF>0626 0648 +FBF0>0626 06C7 +FBF1>0626 06C7 +FBF2>0626 06C6 +FBF3>0626 06C6 +FBF4>0626 06C8 +FBF5>0626 06C8 +FBF6>0626 06D0 +FBF7>0626 06D0 +FBF8>0626 06D0 +FBF9>0626 0649 +FBFA>0626 0649 +FBFB>0626 0649 +FBFC>06CC +FBFD>06CC +FBFE>06CC +FBFF>06CC +FC00>0626 062C +FC01>0626 062D +FC02>0626 0645 +FC03>0626 0649 +FC04>0626 064A +FC05>0628 062C +FC06>0628 062D +FC07>0628 062E +FC08>0628 0645 +FC09>0628 0649 +FC0A>0628 064A +FC0B>062A 062C +FC0C>062A 062D +FC0D>062A 062E +FC0E>062A 0645 +FC0F>062A 0649 +FC10>062A 064A +FC11>062B 062C +FC12>062B 0645 +FC13>062B 0649 +FC14>062B 064A +FC15>062C 062D +FC16>062C 0645 +FC17>062D 062C +FC18>062D 0645 +FC19>062E 062C +FC1A>062E 062D +FC1B>062E 0645 +FC1C>0633 062C +FC1D>0633 062D +FC1E>0633 062E +FC1F>0633 0645 +FC20>0635 062D +FC21>0635 0645 +FC22>0636 062C +FC23>0636 062D +FC24>0636 062E +FC25>0636 0645 +FC26>0637 062D +FC27>0637 0645 +FC28>0638 0645 +FC29>0639 062C +FC2A>0639 0645 +FC2B>063A 062C +FC2C>063A 0645 +FC2D>0641 062C +FC2E>0641 062D +FC2F>0641 062E +FC30>0641 0645 +FC31>0641 0649 +FC32>0641 064A +FC33>0642 062D +FC34>0642 0645 +FC35>0642 0649 +FC36>0642 064A +FC37>0643 0627 +FC38>0643 062C +FC39>0643 062D +FC3A>0643 062E +FC3B>0643 0644 +FC3C>0643 0645 +FC3D>0643 0649 +FC3E>0643 064A +FC3F>0644 062C +FC40>0644 062D +FC41>0644 062E +FC42>0644 0645 +FC43>0644 0649 +FC44>0644 064A +FC45>0645 062C +FC46>0645 062D +FC47>0645 062E +FC48>0645 0645 +FC49>0645 0649 +FC4A>0645 064A +FC4B>0646 062C +FC4C>0646 062D +FC4D>0646 062E +FC4E>0646 0645 +FC4F>0646 0649 +FC50>0646 064A +FC51>0647 062C +FC52>0647 0645 +FC53>0647 0649 +FC54>0647 064A +FC55>064A 062C +FC56>064A 062D +FC57>064A 062E +FC58>064A 0645 +FC59>064A 0649 +FC5A>064A 064A +FC5B>0630 0670 +FC5C>0631 0670 +FC5D>0649 0670 +FC5E>0020 064C 0651 +FC5F>0020 064D 0651 +FC60>0020 064E 0651 +FC61>0020 064F 0651 +FC62>0020 0650 0651 +FC63>0020 0651 0670 +FC64>0626 0631 +FC65>0626 0632 +FC66>0626 0645 +FC67>0626 0646 +FC68>0626 0649 +FC69>0626 064A +FC6A>0628 0631 +FC6B>0628 0632 +FC6C>0628 0645 +FC6D>0628 0646 +FC6E>0628 0649 +FC6F>0628 064A +FC70>062A 0631 +FC71>062A 0632 +FC72>062A 0645 +FC73>062A 0646 +FC74>062A 0649 +FC75>062A 064A +FC76>062B 0631 +FC77>062B 0632 +FC78>062B 0645 +FC79>062B 0646 +FC7A>062B 0649 +FC7B>062B 064A +FC7C>0641 0649 +FC7D>0641 064A +FC7E>0642 0649 +FC7F>0642 064A +FC80>0643 0627 +FC81>0643 0644 +FC82>0643 0645 +FC83>0643 0649 +FC84>0643 064A +FC85>0644 0645 +FC86>0644 0649 +FC87>0644 064A +FC88>0645 0627 +FC89>0645 0645 +FC8A>0646 0631 +FC8B>0646 0632 +FC8C>0646 0645 +FC8D>0646 0646 +FC8E>0646 0649 +FC8F>0646 064A +FC90>0649 0670 +FC91>064A 0631 +FC92>064A 0632 +FC93>064A 0645 +FC94>064A 0646 +FC95>064A 0649 +FC96>064A 064A +FC97>0626 062C +FC98>0626 062D +FC99>0626 062E +FC9A>0626 0645 +FC9B>0626 0647 +FC9C>0628 062C +FC9D>0628 062D +FC9E>0628 062E +FC9F>0628 0645 +FCA0>0628 0647 +FCA1>062A 062C +FCA2>062A 062D +FCA3>062A 062E +FCA4>062A 0645 +FCA5>062A 0647 +FCA6>062B 0645 +FCA7>062C 062D +FCA8>062C 0645 +FCA9>062D 062C +FCAA>062D 0645 +FCAB>062E 062C +FCAC>062E 0645 +FCAD>0633 062C +FCAE>0633 062D +FCAF>0633 062E +FCB0>0633 0645 +FCB1>0635 062D +FCB2>0635 062E +FCB3>0635 0645 +FCB4>0636 062C +FCB5>0636 062D +FCB6>0636 062E +FCB7>0636 0645 +FCB8>0637 062D +FCB9>0638 0645 +FCBA>0639 062C +FCBB>0639 0645 +FCBC>063A 062C +FCBD>063A 0645 +FCBE>0641 062C +FCBF>0641 062D +FCC0>0641 062E +FCC1>0641 0645 +FCC2>0642 062D +FCC3>0642 0645 +FCC4>0643 062C +FCC5>0643 062D +FCC6>0643 062E +FCC7>0643 0644 +FCC8>0643 0645 +FCC9>0644 062C +FCCA>0644 062D +FCCB>0644 062E +FCCC>0644 0645 +FCCD>0644 0647 +FCCE>0645 062C +FCCF>0645 062D +FCD0>0645 062E +FCD1>0645 0645 +FCD2>0646 062C +FCD3>0646 062D +FCD4>0646 062E +FCD5>0646 0645 +FCD6>0646 0647 +FCD7>0647 062C +FCD8>0647 0645 +FCD9>0647 0670 +FCDA>064A 062C +FCDB>064A 062D +FCDC>064A 062E +FCDD>064A 0645 +FCDE>064A 0647 +FCDF>0626 0645 +FCE0>0626 0647 +FCE1>0628 0645 +FCE2>0628 0647 +FCE3>062A 0645 +FCE4>062A 0647 +FCE5>062B 0645 +FCE6>062B 0647 +FCE7>0633 0645 +FCE8>0633 0647 +FCE9>0634 0645 +FCEA>0634 0647 +FCEB>0643 0644 +FCEC>0643 0645 +FCED>0644 0645 +FCEE>0646 0645 +FCEF>0646 0647 +FCF0>064A 0645 +FCF1>064A 0647 +FCF2>0640 064E 0651 +FCF3>0640 064F 0651 +FCF4>0640 0650 0651 +FCF5>0637 0649 +FCF6>0637 064A +FCF7>0639 0649 +FCF8>0639 064A +FCF9>063A 0649 +FCFA>063A 064A +FCFB>0633 0649 +FCFC>0633 064A +FCFD>0634 0649 +FCFE>0634 064A +FCFF>062D 0649 +FD00>062D 064A +FD01>062C 0649 +FD02>062C 064A +FD03>062E 0649 +FD04>062E 064A +FD05>0635 0649 +FD06>0635 064A +FD07>0636 0649 +FD08>0636 064A +FD09>0634 062C +FD0A>0634 062D +FD0B>0634 062E +FD0C>0634 0645 +FD0D>0634 0631 +FD0E>0633 0631 +FD0F>0635 0631 +FD10>0636 0631 +FD11>0637 0649 +FD12>0637 064A +FD13>0639 0649 +FD14>0639 064A +FD15>063A 0649 +FD16>063A 064A +FD17>0633 0649 +FD18>0633 064A +FD19>0634 0649 +FD1A>0634 064A +FD1B>062D 0649 +FD1C>062D 064A +FD1D>062C 0649 +FD1E>062C 064A +FD1F>062E 0649 +FD20>062E 064A +FD21>0635 0649 +FD22>0635 064A +FD23>0636 0649 +FD24>0636 064A +FD25>0634 062C +FD26>0634 062D +FD27>0634 062E +FD28>0634 0645 +FD29>0634 0631 +FD2A>0633 0631 +FD2B>0635 0631 +FD2C>0636 0631 +FD2D>0634 062C +FD2E>0634 062D +FD2F>0634 062E +FD30>0634 0645 +FD31>0633 0647 +FD32>0634 0647 +FD33>0637 0645 +FD34>0633 062C +FD35>0633 062D +FD36>0633 062E +FD37>0634 062C +FD38>0634 062D +FD39>0634 062E +FD3A>0637 0645 +FD3B>0638 0645 +FD3C>0627 064B +FD3D>0627 064B +FD50>062A 062C 0645 +FD51>062A 062D 062C +FD52>062A 062D 062C +FD53>062A 062D 0645 +FD54>062A 062E 0645 +FD55>062A 0645 062C +FD56>062A 0645 062D +FD57>062A 0645 062E +FD58>062C 0645 062D +FD59>062C 0645 062D +FD5A>062D 0645 064A +FD5B>062D 0645 0649 +FD5C>0633 062D 062C +FD5D>0633 062C 062D +FD5E>0633 062C 0649 +FD5F>0633 0645 062D +FD60>0633 0645 062D +FD61>0633 0645 062C +FD62>0633 0645 0645 +FD63>0633 0645 0645 +FD64>0635 062D 062D +FD65>0635 062D 062D +FD66>0635 0645 0645 +FD67>0634 062D 0645 +FD68>0634 062D 0645 +FD69>0634 062C 064A +FD6A>0634 0645 062E +FD6B>0634 0645 062E +FD6C>0634 0645 0645 +FD6D>0634 0645 0645 +FD6E>0636 062D 0649 +FD6F>0636 062E 0645 +FD70>0636 062E 0645 +FD71>0637 0645 062D +FD72>0637 0645 062D +FD73>0637 0645 0645 +FD74>0637 0645 064A +FD75>0639 062C 0645 +FD76>0639 0645 0645 +FD77>0639 0645 0645 +FD78>0639 0645 0649 +FD79>063A 0645 0645 +FD7A>063A 0645 064A +FD7B>063A 0645 0649 +FD7C>0641 062E 0645 +FD7D>0641 062E 0645 +FD7E>0642 0645 062D +FD7F>0642 0645 0645 +FD80>0644 062D 0645 +FD81>0644 062D 064A +FD82>0644 062D 0649 +FD83>0644 062C 062C +FD84>0644 062C 062C +FD85>0644 062E 0645 +FD86>0644 062E 0645 +FD87>0644 0645 062D +FD88>0644 0645 062D +FD89>0645 062D 062C +FD8A>0645 062D 0645 +FD8B>0645 062D 064A +FD8C>0645 062C 062D +FD8D>0645 062C 0645 +FD8E>0645 062E 062C +FD8F>0645 062E 0645 +FD92>0645 062C 062E +FD93>0647 0645 062C +FD94>0647 0645 0645 +FD95>0646 062D 0645 +FD96>0646 062D 0649 +FD97>0646 062C 0645 +FD98>0646 062C 0645 +FD99>0646 062C 0649 +FD9A>0646 0645 064A +FD9B>0646 0645 0649 +FD9C>064A 0645 0645 +FD9D>064A 0645 0645 +FD9E>0628 062E 064A +FD9F>062A 062C 064A +FDA0>062A 062C 0649 +FDA1>062A 062E 064A +FDA2>062A 062E 0649 +FDA3>062A 0645 064A +FDA4>062A 0645 0649 +FDA5>062C 0645 064A +FDA6>062C 062D 0649 +FDA7>062C 0645 0649 +FDA8>0633 062E 0649 +FDA9>0635 062D 064A +FDAA>0634 062D 064A +FDAB>0636 062D 064A +FDAC>0644 062C 064A +FDAD>0644 0645 064A +FDAE>064A 062D 064A +FDAF>064A 062C 064A +FDB0>064A 0645 064A +FDB1>0645 0645 064A +FDB2>0642 0645 064A +FDB3>0646 062D 064A +FDB4>0642 0645 062D +FDB5>0644 062D 0645 +FDB6>0639 0645 064A +FDB7>0643 0645 064A +FDB8>0646 062C 062D +FDB9>0645 062E 064A +FDBA>0644 062C 0645 +FDBB>0643 0645 0645 +FDBC>0644 062C 0645 +FDBD>0646 062C 062D +FDBE>062C 062D 064A +FDBF>062D 062C 064A +FDC0>0645 062C 064A +FDC1>0641 0645 064A +FDC2>0628 062D 064A +FDC3>0643 0645 0645 +FDC4>0639 062C 0645 +FDC5>0635 0645 0645 +FDC6>0633 062E 064A +FDC7>0646 062C 064A +FDF0>0635 0644 06D2 +FDF1>0642 0644 06D2 +FDF2>0627 0644 0644 0647 +FDF3>0627 0643 0628 0631 +FDF4>0645 062D 0645 062F +FDF5>0635 0644 0639 0645 +FDF6>0631 0633 0648 0644 +FDF7>0639 0644 064A 0647 +FDF8>0648 0633 0644 0645 +FDF9>0635 0644 0649 +FDFA>0635 0644 0649 0020 0627 0644 0644 0647 0020 0639 0644 064A 0647 0020 0648 0633 0644 0645 +FDFB>062C 0644 0020 062C 0644 0627 0644 0647 +FDFC>0631 06CC 0627 0644 +FE10>002C +FE11>3001 +FE12>3002 +FE13>003A +FE14>003B +FE15>0021 +FE16>003F +FE17>3016 +FE18>3017 +FE19>2026 +FE30>2025 +FE31>2014 +FE32>2013 +FE33>005F +FE34>005F +FE35>0028 +FE36>0029 +FE37>007B +FE38>007D +FE39>3014 +FE3A>3015 +FE3B>3010 +FE3C>3011 +FE3D>300A +FE3E>300B +FE3F>3008 +FE40>3009 +FE41>300C +FE42>300D +FE43>300E +FE44>300F +FE47>005B +FE48>005D +FE49>203E +FE4A>203E +FE4B>203E +FE4C>203E +FE4D>005F +FE4E>005F +FE4F>005F +FE50>002C +FE51>3001 +FE52>002E +FE54>003B +FE55>003A +FE56>003F +FE57>0021 +FE58>2014 +FE59>0028 +FE5A>0029 +FE5B>007B +FE5C>007D +FE5D>3014 +FE5E>3015 +FE5F>0023 +FE60>0026 +FE61>002A +FE62>002B +FE63>002D +FE64>003C +FE65>003E +FE66>003D +FE68>005C +FE69>0024 +FE6A>0025 +FE6B>0040 +FE70>0020 064B +FE71>0640 064B +FE72>0020 064C +FE74>0020 064D +FE76>0020 064E +FE77>0640 064E +FE78>0020 064F +FE79>0640 064F +FE7A>0020 0650 +FE7B>0640 0650 +FE7C>0020 0651 +FE7D>0640 0651 +FE7E>0020 0652 +FE7F>0640 0652 +FE80>0621 +FE81>0622 +FE82>0622 +FE83>0623 +FE84>0623 +FE85>0624 +FE86>0624 +FE87>0625 +FE88>0625 +FE89>0626 +FE8A>0626 +FE8B>0626 +FE8C>0626 +FE8D>0627 +FE8E>0627 +FE8F>0628 +FE90>0628 +FE91>0628 +FE92>0628 +FE93>0629 +FE94>0629 +FE95>062A +FE96>062A +FE97>062A +FE98>062A +FE99>062B +FE9A>062B +FE9B>062B +FE9C>062B +FE9D>062C +FE9E>062C +FE9F>062C +FEA0>062C +FEA1>062D +FEA2>062D +FEA3>062D +FEA4>062D +FEA5>062E +FEA6>062E +FEA7>062E +FEA8>062E +FEA9>062F +FEAA>062F +FEAB>0630 +FEAC>0630 +FEAD>0631 +FEAE>0631 +FEAF>0632 +FEB0>0632 +FEB1>0633 +FEB2>0633 +FEB3>0633 +FEB4>0633 +FEB5>0634 +FEB6>0634 +FEB7>0634 +FEB8>0634 +FEB9>0635 +FEBA>0635 +FEBB>0635 +FEBC>0635 +FEBD>0636 +FEBE>0636 +FEBF>0636 +FEC0>0636 +FEC1>0637 +FEC2>0637 +FEC3>0637 +FEC4>0637 +FEC5>0638 +FEC6>0638 +FEC7>0638 +FEC8>0638 +FEC9>0639 +FECA>0639 +FECB>0639 +FECC>0639 +FECD>063A +FECE>063A +FECF>063A +FED0>063A +FED1>0641 +FED2>0641 +FED3>0641 +FED4>0641 +FED5>0642 +FED6>0642 +FED7>0642 +FED8>0642 +FED9>0643 +FEDA>0643 +FEDB>0643 +FEDC>0643 +FEDD>0644 +FEDE>0644 +FEDF>0644 +FEE0>0644 +FEE1>0645 +FEE2>0645 +FEE3>0645 +FEE4>0645 +FEE5>0646 +FEE6>0646 +FEE7>0646 +FEE8>0646 +FEE9>0647 +FEEA>0647 +FEEB>0647 +FEEC>0647 +FEED>0648 +FEEE>0648 +FEEF>0649 +FEF0>0649 +FEF1>064A +FEF2>064A +FEF3>064A +FEF4>064A +FEF5>0644 0622 +FEF6>0644 0622 +FEF7>0644 0623 +FEF8>0644 0623 +FEF9>0644 0625 +FEFA>0644 0625 +FEFB>0644 0627 +FEFC>0644 0627 +FF01>0021 +FF02>0022 +FF03>0023 +FF04>0024 +FF05>0025 +FF06>0026 +FF07>0027 +FF08>0028 +FF09>0029 +FF0A>002A +FF0B>002B +FF0C>002C +FF0D>002D +FF0E>002E +FF0F>002F +FF10>0030 +FF11>0031 +FF12>0032 +FF13>0033 +FF14>0034 +FF15>0035 +FF16>0036 +FF17>0037 +FF18>0038 +FF19>0039 +FF1A>003A +FF1B>003B +FF1C>003C +FF1D>003D +FF1E>003E +FF1F>003F +FF20>0040 +FF21>0041 +FF22>0042 +FF23>0043 +FF24>0044 +FF25>0045 +FF26>0046 +FF27>0047 +FF28>0048 +FF29>0049 +FF2A>004A +FF2B>004B +FF2C>004C +FF2D>004D +FF2E>004E +FF2F>004F +FF30>0050 +FF31>0051 +FF32>0052 +FF33>0053 +FF34>0054 +FF35>0055 +FF36>0056 +FF37>0057 +FF38>0058 +FF39>0059 +FF3A>005A +FF3B>005B +FF3C>005C +FF3D>005D +FF3E>005E +FF3F>005F +FF40>0060 +FF41>0061 +FF42>0062 +FF43>0063 +FF44>0064 +FF45>0065 +FF46>0066 +FF47>0067 +FF48>0068 +FF49>0069 +FF4A>006A +FF4B>006B +FF4C>006C +FF4D>006D +FF4E>006E +FF4F>006F +FF50>0070 +FF51>0071 +FF52>0072 +FF53>0073 +FF54>0074 +FF55>0075 +FF56>0076 +FF57>0077 +FF58>0078 +FF59>0079 +FF5A>007A +FF5B>007B +FF5C>007C +FF5D>007D +FF5E>007E +FF5F>2985 +FF60>2986 +FF61>3002 +FF62>300C +FF63>300D +FF64>3001 +FF65>30FB +FF66>30F2 +FF67>30A1 +FF68>30A3 +FF69>30A5 +FF6A>30A7 +FF6B>30A9 +FF6C>30E3 +FF6D>30E5 +FF6E>30E7 +FF6F>30C3 +FF70>30FC +FF71>30A2 +FF72>30A4 +FF73>30A6 +FF74>30A8 +FF75>30AA +FF76>30AB +FF77>30AD +FF78>30AF +FF79>30B1 +FF7A>30B3 +FF7B>30B5 +FF7C>30B7 +FF7D>30B9 +FF7E>30BB +FF7F>30BD +FF80>30BF +FF81>30C1 +FF82>30C4 +FF83>30C6 +FF84>30C8 +FF85>30CA +FF86>30CB +FF87>30CC +FF88>30CD +FF89>30CE +FF8A>30CF +FF8B>30D2 +FF8C>30D5 +FF8D>30D8 +FF8E>30DB +FF8F>30DE +FF90>30DF +FF91>30E0 +FF92>30E1 +FF93>30E2 +FF94>30E4 +FF95>30E6 +FF96>30E8 +FF97>30E9 +FF98>30EA +FF99>30EB +FF9A>30EC +FF9B>30ED +FF9C>30EF +FF9D>30F3 +FF9E>3099 +FF9F>309A +FFA0>3164 +FFA1>3131 +FFA2>3132 +FFA3>3133 +FFA4>3134 +FFA5>3135 +FFA6>3136 +FFA7>3137 +FFA8>3138 +FFA9>3139 +FFAA>313A +FFAB>313B +FFAC>313C +FFAD>313D +FFAE>313E +FFAF>313F +FFB0>3140 +FFB1>3141 +FFB2>3142 +FFB3>3143 +FFB4>3144 +FFB5>3145 +FFB6>3146 +FFB7>3147 +FFB8>3148 +FFB9>3149 +FFBA>314A +FFBB>314B +FFBC>314C +FFBD>314D +FFBE>314E +FFC2>314F +FFC3>3150 +FFC4>3151 +FFC5>3152 +FFC6>3153 +FFC7>3154 +FFCA>3155 +FFCB>3156 +FFCC>3157 +FFCD>3158 +FFCE>3159 +FFCF>315A +FFD2>315B +FFD3>315C +FFD4>315D +FFD5>315E +FFD6>315F +FFD7>3160 +FFDA>3161 +FFDB>3162 +FFDC>3163 +FFE0>00A2 +FFE1>00A3 +FFE2>00AC +FFE3>00AF +FFE4>00A6 +FFE5>00A5 +FFE6>20A9 +FFE8>2502 +FFE9>2190 +FFEA>2191 +FFEB>2192 +FFEC>2193 +FFED>25A0 +FFEE>25CB +1109A=11099 110BA +1109C=1109B 110BA +110AB=110A5 110BA +1D15E>1D157 1D165 +1D15F>1D158 1D165 +1D160>1D15F 1D16E +1D161>1D15F 1D16F +1D162>1D15F 1D170 +1D163>1D15F 1D171 +1D164>1D15F 1D172 +1D1BB>1D1B9 1D165 +1D1BC>1D1BA 1D165 +1D1BD>1D1BB 1D16E +1D1BE>1D1BC 1D16E +1D1BF>1D1BB 1D16F +1D1C0>1D1BC 1D16F +1D400>0041 +1D401>0042 +1D402>0043 +1D403>0044 +1D404>0045 +1D405>0046 +1D406>0047 +1D407>0048 +1D408>0049 +1D409>004A +1D40A>004B +1D40B>004C +1D40C>004D +1D40D>004E +1D40E>004F +1D40F>0050 +1D410>0051 +1D411>0052 +1D412>0053 +1D413>0054 +1D414>0055 +1D415>0056 +1D416>0057 +1D417>0058 +1D418>0059 +1D419>005A +1D41A>0061 +1D41B>0062 +1D41C>0063 +1D41D>0064 +1D41E>0065 +1D41F>0066 +1D420>0067 +1D421>0068 +1D422>0069 +1D423>006A +1D424>006B +1D425>006C +1D426>006D +1D427>006E +1D428>006F +1D429>0070 +1D42A>0071 +1D42B>0072 +1D42C>0073 +1D42D>0074 +1D42E>0075 +1D42F>0076 +1D430>0077 +1D431>0078 +1D432>0079 +1D433>007A +1D434>0041 +1D435>0042 +1D436>0043 +1D437>0044 +1D438>0045 +1D439>0046 +1D43A>0047 +1D43B>0048 +1D43C>0049 +1D43D>004A +1D43E>004B +1D43F>004C +1D440>004D +1D441>004E +1D442>004F +1D443>0050 +1D444>0051 +1D445>0052 +1D446>0053 +1D447>0054 +1D448>0055 +1D449>0056 +1D44A>0057 +1D44B>0058 +1D44C>0059 +1D44D>005A +1D44E>0061 +1D44F>0062 +1D450>0063 +1D451>0064 +1D452>0065 +1D453>0066 +1D454>0067 +1D456>0069 +1D457>006A +1D458>006B +1D459>006C +1D45A>006D +1D45B>006E +1D45C>006F +1D45D>0070 +1D45E>0071 +1D45F>0072 +1D460>0073 +1D461>0074 +1D462>0075 +1D463>0076 +1D464>0077 +1D465>0078 +1D466>0079 +1D467>007A +1D468>0041 +1D469>0042 +1D46A>0043 +1D46B>0044 +1D46C>0045 +1D46D>0046 +1D46E>0047 +1D46F>0048 +1D470>0049 +1D471>004A +1D472>004B +1D473>004C +1D474>004D +1D475>004E +1D476>004F +1D477>0050 +1D478>0051 +1D479>0052 +1D47A>0053 +1D47B>0054 +1D47C>0055 +1D47D>0056 +1D47E>0057 +1D47F>0058 +1D480>0059 +1D481>005A +1D482>0061 +1D483>0062 +1D484>0063 +1D485>0064 +1D486>0065 +1D487>0066 +1D488>0067 +1D489>0068 +1D48A>0069 +1D48B>006A +1D48C>006B +1D48D>006C +1D48E>006D +1D48F>006E +1D490>006F +1D491>0070 +1D492>0071 +1D493>0072 +1D494>0073 +1D495>0074 +1D496>0075 +1D497>0076 +1D498>0077 +1D499>0078 +1D49A>0079 +1D49B>007A +1D49C>0041 +1D49E>0043 +1D49F>0044 +1D4A2>0047 +1D4A5>004A +1D4A6>004B +1D4A9>004E +1D4AA>004F +1D4AB>0050 +1D4AC>0051 +1D4AE>0053 +1D4AF>0054 +1D4B0>0055 +1D4B1>0056 +1D4B2>0057 +1D4B3>0058 +1D4B4>0059 +1D4B5>005A +1D4B6>0061 +1D4B7>0062 +1D4B8>0063 +1D4B9>0064 +1D4BB>0066 +1D4BD>0068 +1D4BE>0069 +1D4BF>006A +1D4C0>006B +1D4C1>006C +1D4C2>006D +1D4C3>006E +1D4C5>0070 +1D4C6>0071 +1D4C7>0072 +1D4C8>0073 +1D4C9>0074 +1D4CA>0075 +1D4CB>0076 +1D4CC>0077 +1D4CD>0078 +1D4CE>0079 +1D4CF>007A +1D4D0>0041 +1D4D1>0042 +1D4D2>0043 +1D4D3>0044 +1D4D4>0045 +1D4D5>0046 +1D4D6>0047 +1D4D7>0048 +1D4D8>0049 +1D4D9>004A +1D4DA>004B +1D4DB>004C +1D4DC>004D +1D4DD>004E +1D4DE>004F +1D4DF>0050 +1D4E0>0051 +1D4E1>0052 +1D4E2>0053 +1D4E3>0054 +1D4E4>0055 +1D4E5>0056 +1D4E6>0057 +1D4E7>0058 +1D4E8>0059 +1D4E9>005A +1D4EA>0061 +1D4EB>0062 +1D4EC>0063 +1D4ED>0064 +1D4EE>0065 +1D4EF>0066 +1D4F0>0067 +1D4F1>0068 +1D4F2>0069 +1D4F3>006A +1D4F4>006B +1D4F5>006C +1D4F6>006D +1D4F7>006E +1D4F8>006F +1D4F9>0070 +1D4FA>0071 +1D4FB>0072 +1D4FC>0073 +1D4FD>0074 +1D4FE>0075 +1D4FF>0076 +1D500>0077 +1D501>0078 +1D502>0079 +1D503>007A +1D504>0041 +1D505>0042 +1D507>0044 +1D508>0045 +1D509>0046 +1D50A>0047 +1D50D>004A +1D50E>004B +1D50F>004C +1D510>004D +1D511>004E +1D512>004F +1D513>0050 +1D514>0051 +1D516>0053 +1D517>0054 +1D518>0055 +1D519>0056 +1D51A>0057 +1D51B>0058 +1D51C>0059 +1D51E>0061 +1D51F>0062 +1D520>0063 +1D521>0064 +1D522>0065 +1D523>0066 +1D524>0067 +1D525>0068 +1D526>0069 +1D527>006A +1D528>006B +1D529>006C +1D52A>006D +1D52B>006E +1D52C>006F +1D52D>0070 +1D52E>0071 +1D52F>0072 +1D530>0073 +1D531>0074 +1D532>0075 +1D533>0076 +1D534>0077 +1D535>0078 +1D536>0079 +1D537>007A +1D538>0041 +1D539>0042 +1D53B>0044 +1D53C>0045 +1D53D>0046 +1D53E>0047 +1D540>0049 +1D541>004A +1D542>004B +1D543>004C +1D544>004D +1D546>004F +1D54A>0053 +1D54B>0054 +1D54C>0055 +1D54D>0056 +1D54E>0057 +1D54F>0058 +1D550>0059 +1D552>0061 +1D553>0062 +1D554>0063 +1D555>0064 +1D556>0065 +1D557>0066 +1D558>0067 +1D559>0068 +1D55A>0069 +1D55B>006A +1D55C>006B +1D55D>006C +1D55E>006D +1D55F>006E +1D560>006F +1D561>0070 +1D562>0071 +1D563>0072 +1D564>0073 +1D565>0074 +1D566>0075 +1D567>0076 +1D568>0077 +1D569>0078 +1D56A>0079 +1D56B>007A +1D56C>0041 +1D56D>0042 +1D56E>0043 +1D56F>0044 +1D570>0045 +1D571>0046 +1D572>0047 +1D573>0048 +1D574>0049 +1D575>004A +1D576>004B +1D577>004C +1D578>004D +1D579>004E +1D57A>004F +1D57B>0050 +1D57C>0051 +1D57D>0052 +1D57E>0053 +1D57F>0054 +1D580>0055 +1D581>0056 +1D582>0057 +1D583>0058 +1D584>0059 +1D585>005A +1D586>0061 +1D587>0062 +1D588>0063 +1D589>0064 +1D58A>0065 +1D58B>0066 +1D58C>0067 +1D58D>0068 +1D58E>0069 +1D58F>006A +1D590>006B +1D591>006C +1D592>006D +1D593>006E +1D594>006F +1D595>0070 +1D596>0071 +1D597>0072 +1D598>0073 +1D599>0074 +1D59A>0075 +1D59B>0076 +1D59C>0077 +1D59D>0078 +1D59E>0079 +1D59F>007A +1D5A0>0041 +1D5A1>0042 +1D5A2>0043 +1D5A3>0044 +1D5A4>0045 +1D5A5>0046 +1D5A6>0047 +1D5A7>0048 +1D5A8>0049 +1D5A9>004A +1D5AA>004B +1D5AB>004C +1D5AC>004D +1D5AD>004E +1D5AE>004F +1D5AF>0050 +1D5B0>0051 +1D5B1>0052 +1D5B2>0053 +1D5B3>0054 +1D5B4>0055 +1D5B5>0056 +1D5B6>0057 +1D5B7>0058 +1D5B8>0059 +1D5B9>005A +1D5BA>0061 +1D5BB>0062 +1D5BC>0063 +1D5BD>0064 +1D5BE>0065 +1D5BF>0066 +1D5C0>0067 +1D5C1>0068 +1D5C2>0069 +1D5C3>006A +1D5C4>006B +1D5C5>006C +1D5C6>006D +1D5C7>006E +1D5C8>006F +1D5C9>0070 +1D5CA>0071 +1D5CB>0072 +1D5CC>0073 +1D5CD>0074 +1D5CE>0075 +1D5CF>0076 +1D5D0>0077 +1D5D1>0078 +1D5D2>0079 +1D5D3>007A +1D5D4>0041 +1D5D5>0042 +1D5D6>0043 +1D5D7>0044 +1D5D8>0045 +1D5D9>0046 +1D5DA>0047 +1D5DB>0048 +1D5DC>0049 +1D5DD>004A +1D5DE>004B +1D5DF>004C +1D5E0>004D +1D5E1>004E +1D5E2>004F +1D5E3>0050 +1D5E4>0051 +1D5E5>0052 +1D5E6>0053 +1D5E7>0054 +1D5E8>0055 +1D5E9>0056 +1D5EA>0057 +1D5EB>0058 +1D5EC>0059 +1D5ED>005A +1D5EE>0061 +1D5EF>0062 +1D5F0>0063 +1D5F1>0064 +1D5F2>0065 +1D5F3>0066 +1D5F4>0067 +1D5F5>0068 +1D5F6>0069 +1D5F7>006A +1D5F8>006B +1D5F9>006C +1D5FA>006D +1D5FB>006E +1D5FC>006F +1D5FD>0070 +1D5FE>0071 +1D5FF>0072 +1D600>0073 +1D601>0074 +1D602>0075 +1D603>0076 +1D604>0077 +1D605>0078 +1D606>0079 +1D607>007A +1D608>0041 +1D609>0042 +1D60A>0043 +1D60B>0044 +1D60C>0045 +1D60D>0046 +1D60E>0047 +1D60F>0048 +1D610>0049 +1D611>004A +1D612>004B +1D613>004C +1D614>004D +1D615>004E +1D616>004F +1D617>0050 +1D618>0051 +1D619>0052 +1D61A>0053 +1D61B>0054 +1D61C>0055 +1D61D>0056 +1D61E>0057 +1D61F>0058 +1D620>0059 +1D621>005A +1D622>0061 +1D623>0062 +1D624>0063 +1D625>0064 +1D626>0065 +1D627>0066 +1D628>0067 +1D629>0068 +1D62A>0069 +1D62B>006A +1D62C>006B +1D62D>006C +1D62E>006D +1D62F>006E +1D630>006F +1D631>0070 +1D632>0071 +1D633>0072 +1D634>0073 +1D635>0074 +1D636>0075 +1D637>0076 +1D638>0077 +1D639>0078 +1D63A>0079 +1D63B>007A +1D63C>0041 +1D63D>0042 +1D63E>0043 +1D63F>0044 +1D640>0045 +1D641>0046 +1D642>0047 +1D643>0048 +1D644>0049 +1D645>004A +1D646>004B +1D647>004C +1D648>004D +1D649>004E +1D64A>004F +1D64B>0050 +1D64C>0051 +1D64D>0052 +1D64E>0053 +1D64F>0054 +1D650>0055 +1D651>0056 +1D652>0057 +1D653>0058 +1D654>0059 +1D655>005A +1D656>0061 +1D657>0062 +1D658>0063 +1D659>0064 +1D65A>0065 +1D65B>0066 +1D65C>0067 +1D65D>0068 +1D65E>0069 +1D65F>006A +1D660>006B +1D661>006C +1D662>006D +1D663>006E +1D664>006F +1D665>0070 +1D666>0071 +1D667>0072 +1D668>0073 +1D669>0074 +1D66A>0075 +1D66B>0076 +1D66C>0077 +1D66D>0078 +1D66E>0079 +1D66F>007A +1D670>0041 +1D671>0042 +1D672>0043 +1D673>0044 +1D674>0045 +1D675>0046 +1D676>0047 +1D677>0048 +1D678>0049 +1D679>004A +1D67A>004B +1D67B>004C +1D67C>004D +1D67D>004E +1D67E>004F +1D67F>0050 +1D680>0051 +1D681>0052 +1D682>0053 +1D683>0054 +1D684>0055 +1D685>0056 +1D686>0057 +1D687>0058 +1D688>0059 +1D689>005A +1D68A>0061 +1D68B>0062 +1D68C>0063 +1D68D>0064 +1D68E>0065 +1D68F>0066 +1D690>0067 +1D691>0068 +1D692>0069 +1D693>006A +1D694>006B +1D695>006C +1D696>006D +1D697>006E +1D698>006F +1D699>0070 +1D69A>0071 +1D69B>0072 +1D69C>0073 +1D69D>0074 +1D69E>0075 +1D69F>0076 +1D6A0>0077 +1D6A1>0078 +1D6A2>0079 +1D6A3>007A +1D6A4>0131 +1D6A5>0237 +1D6A8>0391 +1D6A9>0392 +1D6AA>0393 +1D6AB>0394 +1D6AC>0395 +1D6AD>0396 +1D6AE>0397 +1D6AF>0398 +1D6B0>0399 +1D6B1>039A +1D6B2>039B +1D6B3>039C +1D6B4>039D +1D6B5>039E +1D6B6>039F +1D6B7>03A0 +1D6B8>03A1 +1D6B9>03F4 +1D6BA>03A3 +1D6BB>03A4 +1D6BC>03A5 +1D6BD>03A6 +1D6BE>03A7 +1D6BF>03A8 +1D6C0>03A9 +1D6C1>2207 +1D6C2>03B1 +1D6C3>03B2 +1D6C4>03B3 +1D6C5>03B4 +1D6C6>03B5 +1D6C7>03B6 +1D6C8>03B7 +1D6C9>03B8 +1D6CA>03B9 +1D6CB>03BA +1D6CC>03BB +1D6CD>03BC +1D6CE>03BD +1D6CF>03BE +1D6D0>03BF +1D6D1>03C0 +1D6D2>03C1 +1D6D3>03C2 +1D6D4>03C3 +1D6D5>03C4 +1D6D6>03C5 +1D6D7>03C6 +1D6D8>03C7 +1D6D9>03C8 +1D6DA>03C9 +1D6DB>2202 +1D6DC>03F5 +1D6DD>03D1 +1D6DE>03F0 +1D6DF>03D5 +1D6E0>03F1 +1D6E1>03D6 +1D6E2>0391 +1D6E3>0392 +1D6E4>0393 +1D6E5>0394 +1D6E6>0395 +1D6E7>0396 +1D6E8>0397 +1D6E9>0398 +1D6EA>0399 +1D6EB>039A +1D6EC>039B +1D6ED>039C +1D6EE>039D +1D6EF>039E +1D6F0>039F +1D6F1>03A0 +1D6F2>03A1 +1D6F3>03F4 +1D6F4>03A3 +1D6F5>03A4 +1D6F6>03A5 +1D6F7>03A6 +1D6F8>03A7 +1D6F9>03A8 +1D6FA>03A9 +1D6FB>2207 +1D6FC>03B1 +1D6FD>03B2 +1D6FE>03B3 +1D6FF>03B4 +1D700>03B5 +1D701>03B6 +1D702>03B7 +1D703>03B8 +1D704>03B9 +1D705>03BA +1D706>03BB +1D707>03BC +1D708>03BD +1D709>03BE +1D70A>03BF +1D70B>03C0 +1D70C>03C1 +1D70D>03C2 +1D70E>03C3 +1D70F>03C4 +1D710>03C5 +1D711>03C6 +1D712>03C7 +1D713>03C8 +1D714>03C9 +1D715>2202 +1D716>03F5 +1D717>03D1 +1D718>03F0 +1D719>03D5 +1D71A>03F1 +1D71B>03D6 +1D71C>0391 +1D71D>0392 +1D71E>0393 +1D71F>0394 +1D720>0395 +1D721>0396 +1D722>0397 +1D723>0398 +1D724>0399 +1D725>039A +1D726>039B +1D727>039C +1D728>039D +1D729>039E +1D72A>039F +1D72B>03A0 +1D72C>03A1 +1D72D>03F4 +1D72E>03A3 +1D72F>03A4 +1D730>03A5 +1D731>03A6 +1D732>03A7 +1D733>03A8 +1D734>03A9 +1D735>2207 +1D736>03B1 +1D737>03B2 +1D738>03B3 +1D739>03B4 +1D73A>03B5 +1D73B>03B6 +1D73C>03B7 +1D73D>03B8 +1D73E>03B9 +1D73F>03BA +1D740>03BB +1D741>03BC +1D742>03BD +1D743>03BE +1D744>03BF +1D745>03C0 +1D746>03C1 +1D747>03C2 +1D748>03C3 +1D749>03C4 +1D74A>03C5 +1D74B>03C6 +1D74C>03C7 +1D74D>03C8 +1D74E>03C9 +1D74F>2202 +1D750>03F5 +1D751>03D1 +1D752>03F0 +1D753>03D5 +1D754>03F1 +1D755>03D6 +1D756>0391 +1D757>0392 +1D758>0393 +1D759>0394 +1D75A>0395 +1D75B>0396 +1D75C>0397 +1D75D>0398 +1D75E>0399 +1D75F>039A +1D760>039B +1D761>039C +1D762>039D +1D763>039E +1D764>039F +1D765>03A0 +1D766>03A1 +1D767>03F4 +1D768>03A3 +1D769>03A4 +1D76A>03A5 +1D76B>03A6 +1D76C>03A7 +1D76D>03A8 +1D76E>03A9 +1D76F>2207 +1D770>03B1 +1D771>03B2 +1D772>03B3 +1D773>03B4 +1D774>03B5 +1D775>03B6 +1D776>03B7 +1D777>03B8 +1D778>03B9 +1D779>03BA +1D77A>03BB +1D77B>03BC +1D77C>03BD +1D77D>03BE +1D77E>03BF +1D77F>03C0 +1D780>03C1 +1D781>03C2 +1D782>03C3 +1D783>03C4 +1D784>03C5 +1D785>03C6 +1D786>03C7 +1D787>03C8 +1D788>03C9 +1D789>2202 +1D78A>03F5 +1D78B>03D1 +1D78C>03F0 +1D78D>03D5 +1D78E>03F1 +1D78F>03D6 +1D790>0391 +1D791>0392 +1D792>0393 +1D793>0394 +1D794>0395 +1D795>0396 +1D796>0397 +1D797>0398 +1D798>0399 +1D799>039A +1D79A>039B +1D79B>039C +1D79C>039D +1D79D>039E +1D79E>039F +1D79F>03A0 +1D7A0>03A1 +1D7A1>03F4 +1D7A2>03A3 +1D7A3>03A4 +1D7A4>03A5 +1D7A5>03A6 +1D7A6>03A7 +1D7A7>03A8 +1D7A8>03A9 +1D7A9>2207 +1D7AA>03B1 +1D7AB>03B2 +1D7AC>03B3 +1D7AD>03B4 +1D7AE>03B5 +1D7AF>03B6 +1D7B0>03B7 +1D7B1>03B8 +1D7B2>03B9 +1D7B3>03BA +1D7B4>03BB +1D7B5>03BC +1D7B6>03BD +1D7B7>03BE +1D7B8>03BF +1D7B9>03C0 +1D7BA>03C1 +1D7BB>03C2 +1D7BC>03C3 +1D7BD>03C4 +1D7BE>03C5 +1D7BF>03C6 +1D7C0>03C7 +1D7C1>03C8 +1D7C2>03C9 +1D7C3>2202 +1D7C4>03F5 +1D7C5>03D1 +1D7C6>03F0 +1D7C7>03D5 +1D7C8>03F1 +1D7C9>03D6 +1D7CA>03DC +1D7CB>03DD +1D7CE>0030 +1D7CF>0031 +1D7D0>0032 +1D7D1>0033 +1D7D2>0034 +1D7D3>0035 +1D7D4>0036 +1D7D5>0037 +1D7D6>0038 +1D7D7>0039 +1D7D8>0030 +1D7D9>0031 +1D7DA>0032 +1D7DB>0033 +1D7DC>0034 +1D7DD>0035 +1D7DE>0036 +1D7DF>0037 +1D7E0>0038 +1D7E1>0039 +1D7E2>0030 +1D7E3>0031 +1D7E4>0032 +1D7E5>0033 +1D7E6>0034 +1D7E7>0035 +1D7E8>0036 +1D7E9>0037 +1D7EA>0038 +1D7EB>0039 +1D7EC>0030 +1D7ED>0031 +1D7EE>0032 +1D7EF>0033 +1D7F0>0034 +1D7F1>0035 +1D7F2>0036 +1D7F3>0037 +1D7F4>0038 +1D7F5>0039 +1D7F6>0030 +1D7F7>0031 +1D7F8>0032 +1D7F9>0033 +1D7FA>0034 +1D7FB>0035 +1D7FC>0036 +1D7FD>0037 +1D7FE>0038 +1D7FF>0039 +1F100>0030 002E +1F101>0030 002C +1F102>0031 002C +1F103>0032 002C +1F104>0033 002C +1F105>0034 002C +1F106>0035 002C +1F107>0036 002C +1F108>0037 002C +1F109>0038 002C +1F10A>0039 002C +1F110>0028 0041 0029 +1F111>0028 0042 0029 +1F112>0028 0043 0029 +1F113>0028 0044 0029 +1F114>0028 0045 0029 +1F115>0028 0046 0029 +1F116>0028 0047 0029 +1F117>0028 0048 0029 +1F118>0028 0049 0029 +1F119>0028 004A 0029 +1F11A>0028 004B 0029 +1F11B>0028 004C 0029 +1F11C>0028 004D 0029 +1F11D>0028 004E 0029 +1F11E>0028 004F 0029 +1F11F>0028 0050 0029 +1F120>0028 0051 0029 +1F121>0028 0052 0029 +1F122>0028 0053 0029 +1F123>0028 0054 0029 +1F124>0028 0055 0029 +1F125>0028 0056 0029 +1F126>0028 0057 0029 +1F127>0028 0058 0029 +1F128>0028 0059 0029 +1F129>0028 005A 0029 +1F12A>3014 0053 3015 +1F12B>0043 +1F12C>0052 +1F12D>0043 0044 +1F12E>0057 005A +1F131>0042 +1F13D>004E +1F13F>0050 +1F142>0053 +1F146>0057 +1F14A>0048 0056 +1F14B>004D 0056 +1F14C>0053 0044 +1F14D>0053 0053 +1F14E>0050 0050 0056 +1F190>0044 004A +1F200>307B 304B +1F210>624B +1F211>5B57 +1F212>53CC +1F213>30C7 +1F214>4E8C +1F215>591A +1F216>89E3 +1F217>5929 +1F218>4EA4 +1F219>6620 +1F21A>7121 +1F21B>6599 +1F21C>524D +1F21D>5F8C +1F21E>518D +1F21F>65B0 +1F220>521D +1F221>7D42 +1F222>751F +1F223>8CA9 +1F224>58F0 +1F225>5439 +1F226>6F14 +1F227>6295 +1F228>6355 +1F229>4E00 +1F22A>4E09 +1F22B>904A +1F22C>5DE6 +1F22D>4E2D +1F22E>53F3 +1F22F>6307 +1F230>8D70 +1F231>6253 +1F240>3014 672C 3015 +1F241>3014 4E09 3015 +1F242>3014 4E8C 3015 +1F243>3014 5B89 3015 +1F244>3014 70B9 3015 +1F245>3014 6253 3015 +1F246>3014 76D7 3015 +1F247>3014 52DD 3015 +1F248>3014 6557 3015 +2F800>4E3D +2F801>4E38 +2F802>4E41 +2F803>20122 +2F804>4F60 +2F805>4FAE +2F806>4FBB +2F807>5002 +2F808>507A +2F809>5099 +2F80A>50E7 +2F80B>50CF +2F80C>349E +2F80D>2063A +2F80E>514D +2F80F>5154 +2F810>5164 +2F811>5177 +2F812>2051C +2F813>34B9 +2F814>5167 +2F815>518D +2F816>2054B +2F817>5197 +2F818>51A4 +2F819>4ECC +2F81A>51AC +2F81B>51B5 +2F81C>291DF +2F81D>51F5 +2F81E>5203 +2F81F>34DF +2F820>523B +2F821>5246 +2F822>5272 +2F823>5277 +2F824>3515 +2F825>52C7 +2F826>52C9 +2F827>52E4 +2F828>52FA +2F829>5305 +2F82A>5306 +2F82B>5317 +2F82C>5349 +2F82D>5351 +2F82E>535A +2F82F>5373 +2F830>537D +2F831>537F +2F832>537F +2F833>537F +2F834>20A2C +2F835>7070 +2F836>53CA +2F837>53DF +2F838>20B63 +2F839>53EB +2F83A>53F1 +2F83B>5406 +2F83C>549E +2F83D>5438 +2F83E>5448 +2F83F>5468 +2F840>54A2 +2F841>54F6 +2F842>5510 +2F843>5553 +2F844>5563 +2F845>5584 +2F846>5584 +2F847>5599 +2F848>55AB +2F849>55B3 +2F84A>55C2 +2F84B>5716 +2F84C>5606 +2F84D>5717 +2F84E>5651 +2F84F>5674 +2F850>5207 +2F851>58EE +2F852>57CE +2F853>57F4 +2F854>580D +2F855>578B +2F856>5832 +2F857>5831 +2F858>58AC +2F859>214E4 +2F85A>58F2 +2F85B>58F7 +2F85C>5906 +2F85D>591A +2F85E>5922 +2F85F>5962 +2F860>216A8 +2F861>216EA +2F862>59EC +2F863>5A1B +2F864>5A27 +2F865>59D8 +2F866>5A66 +2F867>36EE +2F868>36FC +2F869>5B08 +2F86A>5B3E +2F86B>5B3E +2F86C>219C8 +2F86D>5BC3 +2F86E>5BD8 +2F86F>5BE7 +2F870>5BF3 +2F871>21B18 +2F872>5BFF +2F873>5C06 +2F874>5F53 +2F875>5C22 +2F876>3781 +2F877>5C60 +2F878>5C6E +2F879>5CC0 +2F87A>5C8D +2F87B>21DE4 +2F87C>5D43 +2F87D>21DE6 +2F87E>5D6E +2F87F>5D6B +2F880>5D7C +2F881>5DE1 +2F882>5DE2 +2F883>382F +2F884>5DFD +2F885>5E28 +2F886>5E3D +2F887>5E69 +2F888>3862 +2F889>22183 +2F88A>387C +2F88B>5EB0 +2F88C>5EB3 +2F88D>5EB6 +2F88E>5ECA +2F88F>2A392 +2F890>5EFE +2F891>22331 +2F892>22331 +2F893>8201 +2F894>5F22 +2F895>5F22 +2F896>38C7 +2F897>232B8 +2F898>261DA +2F899>5F62 +2F89A>5F6B +2F89B>38E3 +2F89C>5F9A +2F89D>5FCD +2F89E>5FD7 +2F89F>5FF9 +2F8A0>6081 +2F8A1>393A +2F8A2>391C +2F8A3>6094 +2F8A4>226D4 +2F8A5>60C7 +2F8A6>6148 +2F8A7>614C +2F8A8>614E +2F8A9>614C +2F8AA>617A +2F8AB>618E +2F8AC>61B2 +2F8AD>61A4 +2F8AE>61AF +2F8AF>61DE +2F8B0>61F2 +2F8B1>61F6 +2F8B2>6210 +2F8B3>621B +2F8B4>625D +2F8B5>62B1 +2F8B6>62D4 +2F8B7>6350 +2F8B8>22B0C +2F8B9>633D +2F8BA>62FC +2F8BB>6368 +2F8BC>6383 +2F8BD>63E4 +2F8BE>22BF1 +2F8BF>6422 +2F8C0>63C5 +2F8C1>63A9 +2F8C2>3A2E +2F8C3>6469 +2F8C4>647E +2F8C5>649D +2F8C6>6477 +2F8C7>3A6C +2F8C8>654F +2F8C9>656C +2F8CA>2300A +2F8CB>65E3 +2F8CC>66F8 +2F8CD>6649 +2F8CE>3B19 +2F8CF>6691 +2F8D0>3B08 +2F8D1>3AE4 +2F8D2>5192 +2F8D3>5195 +2F8D4>6700 +2F8D5>669C +2F8D6>80AD +2F8D7>43D9 +2F8D8>6717 +2F8D9>671B +2F8DA>6721 +2F8DB>675E +2F8DC>6753 +2F8DD>233C3 +2F8DE>3B49 +2F8DF>67FA +2F8E0>6785 +2F8E1>6852 +2F8E2>6885 +2F8E3>2346D +2F8E4>688E +2F8E5>681F +2F8E6>6914 +2F8E7>3B9D +2F8E8>6942 +2F8E9>69A3 +2F8EA>69EA +2F8EB>6AA8 +2F8EC>236A3 +2F8ED>6ADB +2F8EE>3C18 +2F8EF>6B21 +2F8F0>238A7 +2F8F1>6B54 +2F8F2>3C4E +2F8F3>6B72 +2F8F4>6B9F +2F8F5>6BBA +2F8F6>6BBB +2F8F7>23A8D +2F8F8>21D0B +2F8F9>23AFA +2F8FA>6C4E +2F8FB>23CBC +2F8FC>6CBF +2F8FD>6CCD +2F8FE>6C67 +2F8FF>6D16 +2F900>6D3E +2F901>6D77 +2F902>6D41 +2F903>6D69 +2F904>6D78 +2F905>6D85 +2F906>23D1E +2F907>6D34 +2F908>6E2F +2F909>6E6E +2F90A>3D33 +2F90B>6ECB +2F90C>6EC7 +2F90D>23ED1 +2F90E>6DF9 +2F90F>6F6E +2F910>23F5E +2F911>23F8E +2F912>6FC6 +2F913>7039 +2F914>701E +2F915>701B +2F916>3D96 +2F917>704A +2F918>707D +2F919>7077 +2F91A>70AD +2F91B>20525 +2F91C>7145 +2F91D>24263 +2F91E>719C +2F91F>243AB +2F920>7228 +2F921>7235 +2F922>7250 +2F923>24608 +2F924>7280 +2F925>7295 +2F926>24735 +2F927>24814 +2F928>737A +2F929>738B +2F92A>3EAC +2F92B>73A5 +2F92C>3EB8 +2F92D>3EB8 +2F92E>7447 +2F92F>745C +2F930>7471 +2F931>7485 +2F932>74CA +2F933>3F1B +2F934>7524 +2F935>24C36 +2F936>753E +2F937>24C92 +2F938>7570 +2F939>2219F +2F93A>7610 +2F93B>24FA1 +2F93C>24FB8 +2F93D>25044 +2F93E>3FFC +2F93F>4008 +2F940>76F4 +2F941>250F3 +2F942>250F2 +2F943>25119 +2F944>25133 +2F945>771E +2F946>771F +2F947>771F +2F948>774A +2F949>4039 +2F94A>778B +2F94B>4046 +2F94C>4096 +2F94D>2541D +2F94E>784E +2F94F>788C +2F950>78CC +2F951>40E3 +2F952>25626 +2F953>7956 +2F954>2569A +2F955>256C5 +2F956>798F +2F957>79EB +2F958>412F +2F959>7A40 +2F95A>7A4A +2F95B>7A4F +2F95C>2597C +2F95D>25AA7 +2F95E>25AA7 +2F95F>7AEE +2F960>4202 +2F961>25BAB +2F962>7BC6 +2F963>7BC9 +2F964>4227 +2F965>25C80 +2F966>7CD2 +2F967>42A0 +2F968>7CE8 +2F969>7CE3 +2F96A>7D00 +2F96B>25F86 +2F96C>7D63 +2F96D>4301 +2F96E>7DC7 +2F96F>7E02 +2F970>7E45 +2F971>4334 +2F972>26228 +2F973>26247 +2F974>4359 +2F975>262D9 +2F976>7F7A +2F977>2633E +2F978>7F95 +2F979>7FFA +2F97A>8005 +2F97B>264DA +2F97C>26523 +2F97D>8060 +2F97E>265A8 +2F97F>8070 +2F980>2335F +2F981>43D5 +2F982>80B2 +2F983>8103 +2F984>440B +2F985>813E +2F986>5AB5 +2F987>267A7 +2F988>267B5 +2F989>23393 +2F98A>2339C +2F98B>8201 +2F98C>8204 +2F98D>8F9E +2F98E>446B +2F98F>8291 +2F990>828B +2F991>829D +2F992>52B3 +2F993>82B1 +2F994>82B3 +2F995>82BD +2F996>82E6 +2F997>26B3C +2F998>82E5 +2F999>831D +2F99A>8363 +2F99B>83AD +2F99C>8323 +2F99D>83BD +2F99E>83E7 +2F99F>8457 +2F9A0>8353 +2F9A1>83CA +2F9A2>83CC +2F9A3>83DC +2F9A4>26C36 +2F9A5>26D6B +2F9A6>26CD5 +2F9A7>452B +2F9A8>84F1 +2F9A9>84F3 +2F9AA>8516 +2F9AB>273CA +2F9AC>8564 +2F9AD>26F2C +2F9AE>455D +2F9AF>4561 +2F9B0>26FB1 +2F9B1>270D2 +2F9B2>456B +2F9B3>8650 +2F9B4>865C +2F9B5>8667 +2F9B6>8669 +2F9B7>86A9 +2F9B8>8688 +2F9B9>870E +2F9BA>86E2 +2F9BB>8779 +2F9BC>8728 +2F9BD>876B +2F9BE>8786 +2F9BF>45D7 +2F9C0>87E1 +2F9C1>8801 +2F9C2>45F9 +2F9C3>8860 +2F9C4>8863 +2F9C5>27667 +2F9C6>88D7 +2F9C7>88DE +2F9C8>4635 +2F9C9>88FA +2F9CA>34BB +2F9CB>278AE +2F9CC>27966 +2F9CD>46BE +2F9CE>46C7 +2F9CF>8AA0 +2F9D0>8AED +2F9D1>8B8A +2F9D2>8C55 +2F9D3>27CA8 +2F9D4>8CAB +2F9D5>8CC1 +2F9D6>8D1B +2F9D7>8D77 +2F9D8>27F2F +2F9D9>20804 +2F9DA>8DCB +2F9DB>8DBC +2F9DC>8DF0 +2F9DD>208DE +2F9DE>8ED4 +2F9DF>8F38 +2F9E0>285D2 +2F9E1>285ED +2F9E2>9094 +2F9E3>90F1 +2F9E4>9111 +2F9E5>2872E +2F9E6>911B +2F9E7>9238 +2F9E8>92D7 +2F9E9>92D8 +2F9EA>927C +2F9EB>93F9 +2F9EC>9415 +2F9ED>28BFA +2F9EE>958B +2F9EF>4995 +2F9F0>95B7 +2F9F1>28D77 +2F9F2>49E6 +2F9F3>96C3 +2F9F4>5DB2 +2F9F5>9723 +2F9F6>29145 +2F9F7>2921A +2F9F8>4A6E +2F9F9>4A76 +2F9FA>97E0 +2F9FB>2940A +2F9FC>4AB2 +2F9FD>29496 +2F9FE>980B +2F9FF>980B +2FA00>9829 +2FA01>295B6 +2FA02>98E2 +2FA03>4B33 +2FA04>9929 +2FA05>99A7 +2FA06>99C2 +2FA07>99FE +2FA08>4BCE +2FA09>29B30 +2FA0A>9B12 +2FA0B>9C40 +2FA0C>9CFD +2FA0D>4CCE +2FA0E>4CED +2FA0F>9D67 +2FA10>2A0CE +2FA11>4CF8 +2FA12>2A105 +2FA13>2A20E +2FA14>2A291 +2FA15>9EBB +2FA16>4D56 +2FA17>9EF9 +2FA18>9EFE +2FA19>9F05 +2FA1A>9F0F +2FA1B>9F16 +2FA1C>9F3B +2FA1D>2A600 diff --git a/comm/mailnews/extensions/fts3/data/nfkc_cf.txt b/comm/mailnews/extensions/fts3/data/nfkc_cf.txt new file mode 100644 index 0000000000..becabbbf34 --- /dev/null +++ b/comm/mailnews/extensions/fts3/data/nfkc_cf.txt @@ -0,0 +1,5376 @@ +# Extracted from: +# DerivedNormalizationProps-5.2.0.txt +# Date: 2009-08-26, 18:18:50 GMT [MD] +# +# Unicode Character Database +# Copyright (c) 1991-2009 Unicode, Inc. +# For terms of use, see http://www.unicode.org/terms_of_use.html +# For documentation, see http://www.unicode.org/reports/tr44/ + +# ================================================ +# This file has been reformatted into syntax for the +# gennorm2 Normalizer2 data generator tool. +# Only the NFKC_CF mappings are retained and reformatted. +# Reformatting via regular expression: s/ *; NFKC_CF; */>/ +# Use this file as the second gennorm2 input file after nfkc.txt. +# ================================================ + +# Derived Property: NFKC_Casefold (NFKC_CF) +# This property removes certain variations from characters: case, compatibility, and default-ignorables. +# It is used for loose matching and certain types of identifiers. +# It is constructed by applying NFKC, CaseFolding, and removal of Default_Ignorable_Code_Points. +# The process of applying these transformations is repeated until a stable result is produced. +# WARNING: Application to STRINGS must apply NFC after mapping each character, because characters may interact. +# For more information, see [http://www.unicode.org/reports/tr44/] +# Omitted code points are unchanged by this mapping. +# @missing: 0000..10FFFF><code point> + +# All code points not explicitly listed for NFKC_Casefold +# have the value <codepoint>. + +0041>0061 +0042>0062 +0043>0063 +0044>0064 +0045>0065 +0046>0066 +0047>0067 +0048>0068 +0049>0069 +004A>006A +004B>006B +004C>006C +004D>006D +004E>006E +004F>006F +0050>0070 +0051>0071 +0052>0072 +0053>0073 +0054>0074 +0055>0075 +0056>0076 +0057>0077 +0058>0078 +0059>0079 +005A>007A +00A0>0020 +00A8>0020 0308 +00AA>0061 +00AD> +00AF>0020 0304 +00B2>0032 +00B3>0033 +00B4>0020 0301 +00B5>03BC +00B8>0020 0327 +00B9>0031 +00BA>006F +00BC>0031 2044 0034 +00BD>0031 2044 0032 +00BE>0033 2044 0034 +00C0>00E0 +00C1>00E1 +00C2>00E2 +00C3>00E3 +00C4>00E4 +00C5>00E5 +00C6>00E6 +00C7>00E7 +00C8>00E8 +00C9>00E9 +00CA>00EA +00CB>00EB +00CC>00EC +00CD>00ED +00CE>00EE +00CF>00EF +00D0>00F0 +00D1>00F1 +00D2>00F2 +00D3>00F3 +00D4>00F4 +00D5>00F5 +00D6>00F6 +00D8>00F8 +00D9>00F9 +00DA>00FA +00DB>00FB +00DC>00FC +00DD>00FD +00DE>00FE +00DF>0073 0073 +0100>0101 +0102>0103 +0104>0105 +0106>0107 +0108>0109 +010A>010B +010C>010D +010E>010F +0110>0111 +0112>0113 +0114>0115 +0116>0117 +0118>0119 +011A>011B +011C>011D +011E>011F +0120>0121 +0122>0123 +0124>0125 +0126>0127 +0128>0129 +012A>012B +012C>012D +012E>012F +0130>0069 0307 +0132..0133>0069 006A +0134>0135 +0136>0137 +0139>013A +013B>013C +013D>013E +013F..0140>006C 00B7 +0141>0142 +0143>0144 +0145>0146 +0147>0148 +0149>02BC 006E +014A>014B +014C>014D +014E>014F +0150>0151 +0152>0153 +0154>0155 +0156>0157 +0158>0159 +015A>015B +015C>015D +015E>015F +0160>0161 +0162>0163 +0164>0165 +0166>0167 +0168>0169 +016A>016B +016C>016D +016E>016F +0170>0171 +0172>0173 +0174>0175 +0176>0177 +0178>00FF +0179>017A +017B>017C +017D>017E +017F>0073 +0181>0253 +0182>0183 +0184>0185 +0186>0254 +0187>0188 +0189>0256 +018A>0257 +018B>018C +018E>01DD +018F>0259 +0190>025B +0191>0192 +0193>0260 +0194>0263 +0196>0269 +0197>0268 +0198>0199 +019C>026F +019D>0272 +019F>0275 +01A0>01A1 +01A2>01A3 +01A4>01A5 +01A6>0280 +01A7>01A8 +01A9>0283 +01AC>01AD +01AE>0288 +01AF>01B0 +01B1>028A +01B2>028B +01B3>01B4 +01B5>01B6 +01B7>0292 +01B8>01B9 +01BC>01BD +01C4..01C6>0064 017E +01C7..01C9>006C 006A +01CA..01CC>006E 006A +01CD>01CE +01CF>01D0 +01D1>01D2 +01D3>01D4 +01D5>01D6 +01D7>01D8 +01D9>01DA +01DB>01DC +01DE>01DF +01E0>01E1 +01E2>01E3 +01E4>01E5 +01E6>01E7 +01E8>01E9 +01EA>01EB +01EC>01ED +01EE>01EF +01F1..01F3>0064 007A +01F4>01F5 +01F6>0195 +01F7>01BF +01F8>01F9 +01FA>01FB +01FC>01FD +01FE>01FF +0200>0201 +0202>0203 +0204>0205 +0206>0207 +0208>0209 +020A>020B +020C>020D +020E>020F +0210>0211 +0212>0213 +0214>0215 +0216>0217 +0218>0219 +021A>021B +021C>021D +021E>021F +0220>019E +0222>0223 +0224>0225 +0226>0227 +0228>0229 +022A>022B +022C>022D +022E>022F +0230>0231 +0232>0233 +023A>2C65 +023B>023C +023D>019A +023E>2C66 +0241>0242 +0243>0180 +0244>0289 +0245>028C +0246>0247 +0248>0249 +024A>024B +024C>024D +024E>024F +02B0>0068 +02B1>0266 +02B2>006A +02B3>0072 +02B4>0279 +02B5>027B +02B6>0281 +02B7>0077 +02B8>0079 +02D8>0020 0306 +02D9>0020 0307 +02DA>0020 030A +02DB>0020 0328 +02DC>0020 0303 +02DD>0020 030B +02E0>0263 +02E1>006C +02E2>0073 +02E3>0078 +02E4>0295 +0340>0300 +0341>0301 +0343>0313 +0344>0308 0301 +0345>03B9 +034F> +0370>0371 +0372>0373 +0374>02B9 +0376>0377 +037A>0020 03B9 +037E>003B +0384>0020 0301 +0385>0020 0308 0301 +0386>03AC +0387>00B7 +0388>03AD +0389>03AE +038A>03AF +038C>03CC +038E>03CD +038F>03CE +0391>03B1 +0392>03B2 +0393>03B3 +0394>03B4 +0395>03B5 +0396>03B6 +0397>03B7 +0398>03B8 +0399>03B9 +039A>03BA +039B>03BB +039C>03BC +039D>03BD +039E>03BE +039F>03BF +03A0>03C0 +03A1>03C1 +03A3>03C3 +03A4>03C4 +03A5>03C5 +03A6>03C6 +03A7>03C7 +03A8>03C8 +03A9>03C9 +03AA>03CA +03AB>03CB +03C2>03C3 +03CF>03D7 +03D0>03B2 +03D1>03B8 +03D2>03C5 +03D3>03CD +03D4>03CB +03D5>03C6 +03D6>03C0 +03D8>03D9 +03DA>03DB +03DC>03DD +03DE>03DF +03E0>03E1 +03E2>03E3 +03E4>03E5 +03E6>03E7 +03E8>03E9 +03EA>03EB +03EC>03ED +03EE>03EF +03F0>03BA +03F1>03C1 +03F2>03C3 +03F4>03B8 +03F5>03B5 +03F7>03F8 +03F9>03C3 +03FA>03FB +03FD>037B +03FE>037C +03FF>037D +0400>0450 +0401>0451 +0402>0452 +0403>0453 +0404>0454 +0405>0455 +0406>0456 +0407>0457 +0408>0458 +0409>0459 +040A>045A +040B>045B +040C>045C +040D>045D +040E>045E +040F>045F +0410>0430 +0411>0431 +0412>0432 +0413>0433 +0414>0434 +0415>0435 +0416>0436 +0417>0437 +0418>0438 +0419>0439 +041A>043A +041B>043B +041C>043C +041D>043D +041E>043E +041F>043F +0420>0440 +0421>0441 +0422>0442 +0423>0443 +0424>0444 +0425>0445 +0426>0446 +0427>0447 +0428>0448 +0429>0449 +042A>044A +042B>044B +042C>044C +042D>044D +042E>044E +042F>044F +0460>0461 +0462>0463 +0464>0465 +0466>0467 +0468>0469 +046A>046B +046C>046D +046E>046F +0470>0471 +0472>0473 +0474>0475 +0476>0477 +0478>0479 +047A>047B +047C>047D +047E>047F +0480>0481 +048A>048B +048C>048D +048E>048F +0490>0491 +0492>0493 +0494>0495 +0496>0497 +0498>0499 +049A>049B +049C>049D +049E>049F +04A0>04A1 +04A2>04A3 +04A4>04A5 +04A6>04A7 +04A8>04A9 +04AA>04AB +04AC>04AD +04AE>04AF +04B0>04B1 +04B2>04B3 +04B4>04B5 +04B6>04B7 +04B8>04B9 +04BA>04BB +04BC>04BD +04BE>04BF +04C0>04CF +04C1>04C2 +04C3>04C4 +04C5>04C6 +04C7>04C8 +04C9>04CA +04CB>04CC +04CD>04CE +04D0>04D1 +04D2>04D3 +04D4>04D5 +04D6>04D7 +04D8>04D9 +04DA>04DB +04DC>04DD +04DE>04DF +04E0>04E1 +04E2>04E3 +04E4>04E5 +04E6>04E7 +04E8>04E9 +04EA>04EB +04EC>04ED +04EE>04EF +04F0>04F1 +04F2>04F3 +04F4>04F5 +04F6>04F7 +04F8>04F9 +04FA>04FB +04FC>04FD +04FE>04FF +0500>0501 +0502>0503 +0504>0505 +0506>0507 +0508>0509 +050A>050B +050C>050D +050E>050F +0510>0511 +0512>0513 +0514>0515 +0516>0517 +0518>0519 +051A>051B +051C>051D +051E>051F +0520>0521 +0522>0523 +0524>0525 +0531>0561 +0532>0562 +0533>0563 +0534>0564 +0535>0565 +0536>0566 +0537>0567 +0538>0568 +0539>0569 +053A>056A +053B>056B +053C>056C +053D>056D +053E>056E +053F>056F +0540>0570 +0541>0571 +0542>0572 +0543>0573 +0544>0574 +0545>0575 +0546>0576 +0547>0577 +0548>0578 +0549>0579 +054A>057A +054B>057B +054C>057C +054D>057D +054E>057E +054F>057F +0550>0580 +0551>0581 +0552>0582 +0553>0583 +0554>0584 +0555>0585 +0556>0586 +0587>0565 0582 +0675>0627 0674 +0676>0648 0674 +0677>06C7 0674 +0678>064A 0674 +0958>0915 093C +0959>0916 093C +095A>0917 093C +095B>091C 093C +095C>0921 093C +095D>0922 093C +095E>092B 093C +095F>092F 093C +09DC>09A1 09BC +09DD>09A2 09BC +09DF>09AF 09BC +0A33>0A32 0A3C +0A36>0A38 0A3C +0A59>0A16 0A3C +0A5A>0A17 0A3C +0A5B>0A1C 0A3C +0A5E>0A2B 0A3C +0B5C>0B21 0B3C +0B5D>0B22 0B3C +0E33>0E4D 0E32 +0EB3>0ECD 0EB2 +0EDC>0EAB 0E99 +0EDD>0EAB 0EA1 +0F0C>0F0B +0F43>0F42 0FB7 +0F4D>0F4C 0FB7 +0F52>0F51 0FB7 +0F57>0F56 0FB7 +0F5C>0F5B 0FB7 +0F69>0F40 0FB5 +0F73>0F71 0F72 +0F75>0F71 0F74 +0F76>0FB2 0F80 +0F77>0FB2 0F71 0F80 +0F78>0FB3 0F80 +0F79>0FB3 0F71 0F80 +0F81>0F71 0F80 +0F93>0F92 0FB7 +0F9D>0F9C 0FB7 +0FA2>0FA1 0FB7 +0FA7>0FA6 0FB7 +0FAC>0FAB 0FB7 +0FB9>0F90 0FB5 +10A0>2D00 +10A1>2D01 +10A2>2D02 +10A3>2D03 +10A4>2D04 +10A5>2D05 +10A6>2D06 +10A7>2D07 +10A8>2D08 +10A9>2D09 +10AA>2D0A +10AB>2D0B +10AC>2D0C +10AD>2D0D +10AE>2D0E +10AF>2D0F +10B0>2D10 +10B1>2D11 +10B2>2D12 +10B3>2D13 +10B4>2D14 +10B5>2D15 +10B6>2D16 +10B7>2D17 +10B8>2D18 +10B9>2D19 +10BA>2D1A +10BB>2D1B +10BC>2D1C +10BD>2D1D +10BE>2D1E +10BF>2D1F +10C0>2D20 +10C1>2D21 +10C2>2D22 +10C3>2D23 +10C4>2D24 +10C5>2D25 +10FC>10DC +115F..1160> +17B4..17B5> +180B..180D> +1D2C>0061 +1D2D>00E6 +1D2E>0062 +1D30>0064 +1D31>0065 +1D32>01DD +1D33>0067 +1D34>0068 +1D35>0069 +1D36>006A +1D37>006B +1D38>006C +1D39>006D +1D3A>006E +1D3C>006F +1D3D>0223 +1D3E>0070 +1D3F>0072 +1D40>0074 +1D41>0075 +1D42>0077 +1D43>0061 +1D44>0250 +1D45>0251 +1D46>1D02 +1D47>0062 +1D48>0064 +1D49>0065 +1D4A>0259 +1D4B>025B +1D4C>025C +1D4D>0067 +1D4F>006B +1D50>006D +1D51>014B +1D52>006F +1D53>0254 +1D54>1D16 +1D55>1D17 +1D56>0070 +1D57>0074 +1D58>0075 +1D59>1D1D +1D5A>026F +1D5B>0076 +1D5C>1D25 +1D5D>03B2 +1D5E>03B3 +1D5F>03B4 +1D60>03C6 +1D61>03C7 +1D62>0069 +1D63>0072 +1D64>0075 +1D65>0076 +1D66>03B2 +1D67>03B3 +1D68>03C1 +1D69>03C6 +1D6A>03C7 +1D78>043D +1D9B>0252 +1D9C>0063 +1D9D>0255 +1D9E>00F0 +1D9F>025C +1DA0>0066 +1DA1>025F +1DA2>0261 +1DA3>0265 +1DA4>0268 +1DA5>0269 +1DA6>026A +1DA7>1D7B +1DA8>029D +1DA9>026D +1DAA>1D85 +1DAB>029F +1DAC>0271 +1DAD>0270 +1DAE>0272 +1DAF>0273 +1DB0>0274 +1DB1>0275 +1DB2>0278 +1DB3>0282 +1DB4>0283 +1DB5>01AB +1DB6>0289 +1DB7>028A +1DB8>1D1C +1DB9>028B +1DBA>028C +1DBB>007A +1DBC>0290 +1DBD>0291 +1DBE>0292 +1DBF>03B8 +1E00>1E01 +1E02>1E03 +1E04>1E05 +1E06>1E07 +1E08>1E09 +1E0A>1E0B +1E0C>1E0D +1E0E>1E0F +1E10>1E11 +1E12>1E13 +1E14>1E15 +1E16>1E17 +1E18>1E19 +1E1A>1E1B +1E1C>1E1D +1E1E>1E1F +1E20>1E21 +1E22>1E23 +1E24>1E25 +1E26>1E27 +1E28>1E29 +1E2A>1E2B +1E2C>1E2D +1E2E>1E2F +1E30>1E31 +1E32>1E33 +1E34>1E35 +1E36>1E37 +1E38>1E39 +1E3A>1E3B +1E3C>1E3D +1E3E>1E3F +1E40>1E41 +1E42>1E43 +1E44>1E45 +1E46>1E47 +1E48>1E49 +1E4A>1E4B +1E4C>1E4D +1E4E>1E4F +1E50>1E51 +1E52>1E53 +1E54>1E55 +1E56>1E57 +1E58>1E59 +1E5A>1E5B +1E5C>1E5D +1E5E>1E5F +1E60>1E61 +1E62>1E63 +1E64>1E65 +1E66>1E67 +1E68>1E69 +1E6A>1E6B +1E6C>1E6D +1E6E>1E6F +1E70>1E71 +1E72>1E73 +1E74>1E75 +1E76>1E77 +1E78>1E79 +1E7A>1E7B +1E7C>1E7D +1E7E>1E7F +1E80>1E81 +1E82>1E83 +1E84>1E85 +1E86>1E87 +1E88>1E89 +1E8A>1E8B +1E8C>1E8D +1E8E>1E8F +1E90>1E91 +1E92>1E93 +1E94>1E95 +1E9A>0061 02BE +1E9B>1E61 +1E9E>0073 0073 +1EA0>1EA1 +1EA2>1EA3 +1EA4>1EA5 +1EA6>1EA7 +1EA8>1EA9 +1EAA>1EAB +1EAC>1EAD +1EAE>1EAF +1EB0>1EB1 +1EB2>1EB3 +1EB4>1EB5 +1EB6>1EB7 +1EB8>1EB9 +1EBA>1EBB +1EBC>1EBD +1EBE>1EBF +1EC0>1EC1 +1EC2>1EC3 +1EC4>1EC5 +1EC6>1EC7 +1EC8>1EC9 +1ECA>1ECB +1ECC>1ECD +1ECE>1ECF +1ED0>1ED1 +1ED2>1ED3 +1ED4>1ED5 +1ED6>1ED7 +1ED8>1ED9 +1EDA>1EDB +1EDC>1EDD +1EDE>1EDF +1EE0>1EE1 +1EE2>1EE3 +1EE4>1EE5 +1EE6>1EE7 +1EE8>1EE9 +1EEA>1EEB +1EEC>1EED +1EEE>1EEF +1EF0>1EF1 +1EF2>1EF3 +1EF4>1EF5 +1EF6>1EF7 +1EF8>1EF9 +1EFA>1EFB +1EFC>1EFD +1EFE>1EFF +1F08>1F00 +1F09>1F01 +1F0A>1F02 +1F0B>1F03 +1F0C>1F04 +1F0D>1F05 +1F0E>1F06 +1F0F>1F07 +1F18>1F10 +1F19>1F11 +1F1A>1F12 +1F1B>1F13 +1F1C>1F14 +1F1D>1F15 +1F28>1F20 +1F29>1F21 +1F2A>1F22 +1F2B>1F23 +1F2C>1F24 +1F2D>1F25 +1F2E>1F26 +1F2F>1F27 +1F38>1F30 +1F39>1F31 +1F3A>1F32 +1F3B>1F33 +1F3C>1F34 +1F3D>1F35 +1F3E>1F36 +1F3F>1F37 +1F48>1F40 +1F49>1F41 +1F4A>1F42 +1F4B>1F43 +1F4C>1F44 +1F4D>1F45 +1F59>1F51 +1F5B>1F53 +1F5D>1F55 +1F5F>1F57 +1F68>1F60 +1F69>1F61 +1F6A>1F62 +1F6B>1F63 +1F6C>1F64 +1F6D>1F65 +1F6E>1F66 +1F6F>1F67 +1F71>03AC +1F73>03AD +1F75>03AE +1F77>03AF +1F79>03CC +1F7B>03CD +1F7D>03CE +1F80>1F00 03B9 +1F81>1F01 03B9 +1F82>1F02 03B9 +1F83>1F03 03B9 +1F84>1F04 03B9 +1F85>1F05 03B9 +1F86>1F06 03B9 +1F87>1F07 03B9 +1F88>1F00 03B9 +1F89>1F01 03B9 +1F8A>1F02 03B9 +1F8B>1F03 03B9 +1F8C>1F04 03B9 +1F8D>1F05 03B9 +1F8E>1F06 03B9 +1F8F>1F07 03B9 +1F90>1F20 03B9 +1F91>1F21 03B9 +1F92>1F22 03B9 +1F93>1F23 03B9 +1F94>1F24 03B9 +1F95>1F25 03B9 +1F96>1F26 03B9 +1F97>1F27 03B9 +1F98>1F20 03B9 +1F99>1F21 03B9 +1F9A>1F22 03B9 +1F9B>1F23 03B9 +1F9C>1F24 03B9 +1F9D>1F25 03B9 +1F9E>1F26 03B9 +1F9F>1F27 03B9 +1FA0>1F60 03B9 +1FA1>1F61 03B9 +1FA2>1F62 03B9 +1FA3>1F63 03B9 +1FA4>1F64 03B9 +1FA5>1F65 03B9 +1FA6>1F66 03B9 +1FA7>1F67 03B9 +1FA8>1F60 03B9 +1FA9>1F61 03B9 +1FAA>1F62 03B9 +1FAB>1F63 03B9 +1FAC>1F64 03B9 +1FAD>1F65 03B9 +1FAE>1F66 03B9 +1FAF>1F67 03B9 +1FB2>1F70 03B9 +1FB3>03B1 03B9 +1FB4>03AC 03B9 +1FB7>1FB6 03B9 +1FB8>1FB0 +1FB9>1FB1 +1FBA>1F70 +1FBB>03AC +1FBC>03B1 03B9 +1FBD>0020 0313 +1FBE>03B9 +1FBF>0020 0313 +1FC0>0020 0342 +1FC1>0020 0308 0342 +1FC2>1F74 03B9 +1FC3>03B7 03B9 +1FC4>03AE 03B9 +1FC7>1FC6 03B9 +1FC8>1F72 +1FC9>03AD +1FCA>1F74 +1FCB>03AE +1FCC>03B7 03B9 +1FCD>0020 0313 0300 +1FCE>0020 0313 0301 +1FCF>0020 0313 0342 +1FD3>0390 +1FD8>1FD0 +1FD9>1FD1 +1FDA>1F76 +1FDB>03AF +1FDD>0020 0314 0300 +1FDE>0020 0314 0301 +1FDF>0020 0314 0342 +1FE3>03B0 +1FE8>1FE0 +1FE9>1FE1 +1FEA>1F7A +1FEB>03CD +1FEC>1FE5 +1FED>0020 0308 0300 +1FEE>0020 0308 0301 +1FEF>0060 +1FF2>1F7C 03B9 +1FF3>03C9 03B9 +1FF4>03CE 03B9 +1FF7>1FF6 03B9 +1FF8>1F78 +1FF9>03CC +1FFA>1F7C +1FFB>03CE +1FFC>03C9 03B9 +1FFD>0020 0301 +1FFE>0020 0314 +2000..200A>0020 +200B..200F> +2011>2010 +2017>0020 0333 +2024>002E +2025>002E 002E +2026>002E 002E 002E +202A..202E> +202F>0020 +2033>2032 2032 +2034>2032 2032 2032 +2036>2035 2035 +2037>2035 2035 2035 +203C>0021 0021 +203E>0020 0305 +2047>003F 003F +2048>003F 0021 +2049>0021 003F +2057>2032 2032 2032 2032 +205F>0020 +2060..2064> +2065..2069> +206A..206F> +2070>0030 +2071>0069 +2074>0034 +2075>0035 +2076>0036 +2077>0037 +2078>0038 +2079>0039 +207A>002B +207B>2212 +207C>003D +207D>0028 +207E>0029 +207F>006E +2080>0030 +2081>0031 +2082>0032 +2083>0033 +2084>0034 +2085>0035 +2086>0036 +2087>0037 +2088>0038 +2089>0039 +208A>002B +208B>2212 +208C>003D +208D>0028 +208E>0029 +2090>0061 +2091>0065 +2092>006F +2093>0078 +2094>0259 +20A8>0072 0073 +2100>0061 002F 0063 +2101>0061 002F 0073 +2102>0063 +2103>00B0 0063 +2105>0063 002F 006F +2106>0063 002F 0075 +2107>025B +2109>00B0 0066 +210A>0067 +210B..210E>0068 +210F>0127 +2110..2111>0069 +2112..2113>006C +2115>006E +2116>006E 006F +2119>0070 +211A>0071 +211B..211D>0072 +2120>0073 006D +2121>0074 0065 006C +2122>0074 006D +2124>007A +2126>03C9 +2128>007A +212A>006B +212B>00E5 +212C>0062 +212D>0063 +212F..2130>0065 +2131>0066 +2132>214E +2133>006D +2134>006F +2135>05D0 +2136>05D1 +2137>05D2 +2138>05D3 +2139>0069 +213B>0066 0061 0078 +213C>03C0 +213D..213E>03B3 +213F>03C0 +2140>2211 +2145..2146>0064 +2147>0065 +2148>0069 +2149>006A +2150>0031 2044 0037 +2151>0031 2044 0039 +2152>0031 2044 0031 0030 +2153>0031 2044 0033 +2154>0032 2044 0033 +2155>0031 2044 0035 +2156>0032 2044 0035 +2157>0033 2044 0035 +2158>0034 2044 0035 +2159>0031 2044 0036 +215A>0035 2044 0036 +215B>0031 2044 0038 +215C>0033 2044 0038 +215D>0035 2044 0038 +215E>0037 2044 0038 +215F>0031 2044 +2160>0069 +2161>0069 0069 +2162>0069 0069 0069 +2163>0069 0076 +2164>0076 +2165>0076 0069 +2166>0076 0069 0069 +2167>0076 0069 0069 0069 +2168>0069 0078 +2169>0078 +216A>0078 0069 +216B>0078 0069 0069 +216C>006C +216D>0063 +216E>0064 +216F>006D +2170>0069 +2171>0069 0069 +2172>0069 0069 0069 +2173>0069 0076 +2174>0076 +2175>0076 0069 +2176>0076 0069 0069 +2177>0076 0069 0069 0069 +2178>0069 0078 +2179>0078 +217A>0078 0069 +217B>0078 0069 0069 +217C>006C +217D>0063 +217E>0064 +217F>006D +2183>2184 +2189>0030 2044 0033 +222C>222B 222B +222D>222B 222B 222B +222F>222E 222E +2230>222E 222E 222E +2329>3008 +232A>3009 +2460>0031 +2461>0032 +2462>0033 +2463>0034 +2464>0035 +2465>0036 +2466>0037 +2467>0038 +2468>0039 +2469>0031 0030 +246A>0031 0031 +246B>0031 0032 +246C>0031 0033 +246D>0031 0034 +246E>0031 0035 +246F>0031 0036 +2470>0031 0037 +2471>0031 0038 +2472>0031 0039 +2473>0032 0030 +2474>0028 0031 0029 +2475>0028 0032 0029 +2476>0028 0033 0029 +2477>0028 0034 0029 +2478>0028 0035 0029 +2479>0028 0036 0029 +247A>0028 0037 0029 +247B>0028 0038 0029 +247C>0028 0039 0029 +247D>0028 0031 0030 0029 +247E>0028 0031 0031 0029 +247F>0028 0031 0032 0029 +2480>0028 0031 0033 0029 +2481>0028 0031 0034 0029 +2482>0028 0031 0035 0029 +2483>0028 0031 0036 0029 +2484>0028 0031 0037 0029 +2485>0028 0031 0038 0029 +2486>0028 0031 0039 0029 +2487>0028 0032 0030 0029 +2488>0031 002E +2489>0032 002E +248A>0033 002E +248B>0034 002E +248C>0035 002E +248D>0036 002E +248E>0037 002E +248F>0038 002E +2490>0039 002E +2491>0031 0030 002E +2492>0031 0031 002E +2493>0031 0032 002E +2494>0031 0033 002E +2495>0031 0034 002E +2496>0031 0035 002E +2497>0031 0036 002E +2498>0031 0037 002E +2499>0031 0038 002E +249A>0031 0039 002E +249B>0032 0030 002E +249C>0028 0061 0029 +249D>0028 0062 0029 +249E>0028 0063 0029 +249F>0028 0064 0029 +24A0>0028 0065 0029 +24A1>0028 0066 0029 +24A2>0028 0067 0029 +24A3>0028 0068 0029 +24A4>0028 0069 0029 +24A5>0028 006A 0029 +24A6>0028 006B 0029 +24A7>0028 006C 0029 +24A8>0028 006D 0029 +24A9>0028 006E 0029 +24AA>0028 006F 0029 +24AB>0028 0070 0029 +24AC>0028 0071 0029 +24AD>0028 0072 0029 +24AE>0028 0073 0029 +24AF>0028 0074 0029 +24B0>0028 0075 0029 +24B1>0028 0076 0029 +24B2>0028 0077 0029 +24B3>0028 0078 0029 +24B4>0028 0079 0029 +24B5>0028 007A 0029 +24B6>0061 +24B7>0062 +24B8>0063 +24B9>0064 +24BA>0065 +24BB>0066 +24BC>0067 +24BD>0068 +24BE>0069 +24BF>006A +24C0>006B +24C1>006C +24C2>006D +24C3>006E +24C4>006F +24C5>0070 +24C6>0071 +24C7>0072 +24C8>0073 +24C9>0074 +24CA>0075 +24CB>0076 +24CC>0077 +24CD>0078 +24CE>0079 +24CF>007A +24D0>0061 +24D1>0062 +24D2>0063 +24D3>0064 +24D4>0065 +24D5>0066 +24D6>0067 +24D7>0068 +24D8>0069 +24D9>006A +24DA>006B +24DB>006C +24DC>006D +24DD>006E +24DE>006F +24DF>0070 +24E0>0071 +24E1>0072 +24E2>0073 +24E3>0074 +24E4>0075 +24E5>0076 +24E6>0077 +24E7>0078 +24E8>0079 +24E9>007A +24EA>0030 +2A0C>222B 222B 222B 222B +2A74>003A 003A 003D +2A75>003D 003D +2A76>003D 003D 003D +2ADC>2ADD 0338 +2C00>2C30 +2C01>2C31 +2C02>2C32 +2C03>2C33 +2C04>2C34 +2C05>2C35 +2C06>2C36 +2C07>2C37 +2C08>2C38 +2C09>2C39 +2C0A>2C3A +2C0B>2C3B +2C0C>2C3C +2C0D>2C3D +2C0E>2C3E +2C0F>2C3F +2C10>2C40 +2C11>2C41 +2C12>2C42 +2C13>2C43 +2C14>2C44 +2C15>2C45 +2C16>2C46 +2C17>2C47 +2C18>2C48 +2C19>2C49 +2C1A>2C4A +2C1B>2C4B +2C1C>2C4C +2C1D>2C4D +2C1E>2C4E +2C1F>2C4F +2C20>2C50 +2C21>2C51 +2C22>2C52 +2C23>2C53 +2C24>2C54 +2C25>2C55 +2C26>2C56 +2C27>2C57 +2C28>2C58 +2C29>2C59 +2C2A>2C5A +2C2B>2C5B +2C2C>2C5C +2C2D>2C5D +2C2E>2C5E +2C60>2C61 +2C62>026B +2C63>1D7D +2C64>027D +2C67>2C68 +2C69>2C6A +2C6B>2C6C +2C6D>0251 +2C6E>0271 +2C6F>0250 +2C70>0252 +2C72>2C73 +2C75>2C76 +2C7C>006A +2C7D>0076 +2C7E>023F +2C7F>0240 +2C80>2C81 +2C82>2C83 +2C84>2C85 +2C86>2C87 +2C88>2C89 +2C8A>2C8B +2C8C>2C8D +2C8E>2C8F +2C90>2C91 +2C92>2C93 +2C94>2C95 +2C96>2C97 +2C98>2C99 +2C9A>2C9B +2C9C>2C9D +2C9E>2C9F +2CA0>2CA1 +2CA2>2CA3 +2CA4>2CA5 +2CA6>2CA7 +2CA8>2CA9 +2CAA>2CAB +2CAC>2CAD +2CAE>2CAF +2CB0>2CB1 +2CB2>2CB3 +2CB4>2CB5 +2CB6>2CB7 +2CB8>2CB9 +2CBA>2CBB +2CBC>2CBD +2CBE>2CBF +2CC0>2CC1 +2CC2>2CC3 +2CC4>2CC5 +2CC6>2CC7 +2CC8>2CC9 +2CCA>2CCB +2CCC>2CCD +2CCE>2CCF +2CD0>2CD1 +2CD2>2CD3 +2CD4>2CD5 +2CD6>2CD7 +2CD8>2CD9 +2CDA>2CDB +2CDC>2CDD +2CDE>2CDF +2CE0>2CE1 +2CE2>2CE3 +2CEB>2CEC +2CED>2CEE +2D6F>2D61 +2E9F>6BCD +2EF3>9F9F +2F00>4E00 +2F01>4E28 +2F02>4E36 +2F03>4E3F +2F04>4E59 +2F05>4E85 +2F06>4E8C +2F07>4EA0 +2F08>4EBA +2F09>513F +2F0A>5165 +2F0B>516B +2F0C>5182 +2F0D>5196 +2F0E>51AB +2F0F>51E0 +2F10>51F5 +2F11>5200 +2F12>529B +2F13>52F9 +2F14>5315 +2F15>531A +2F16>5338 +2F17>5341 +2F18>535C +2F19>5369 +2F1A>5382 +2F1B>53B6 +2F1C>53C8 +2F1D>53E3 +2F1E>56D7 +2F1F>571F +2F20>58EB +2F21>5902 +2F22>590A +2F23>5915 +2F24>5927 +2F25>5973 +2F26>5B50 +2F27>5B80 +2F28>5BF8 +2F29>5C0F +2F2A>5C22 +2F2B>5C38 +2F2C>5C6E +2F2D>5C71 +2F2E>5DDB +2F2F>5DE5 +2F30>5DF1 +2F31>5DFE +2F32>5E72 +2F33>5E7A +2F34>5E7F +2F35>5EF4 +2F36>5EFE +2F37>5F0B +2F38>5F13 +2F39>5F50 +2F3A>5F61 +2F3B>5F73 +2F3C>5FC3 +2F3D>6208 +2F3E>6236 +2F3F>624B +2F40>652F +2F41>6534 +2F42>6587 +2F43>6597 +2F44>65A4 +2F45>65B9 +2F46>65E0 +2F47>65E5 +2F48>66F0 +2F49>6708 +2F4A>6728 +2F4B>6B20 +2F4C>6B62 +2F4D>6B79 +2F4E>6BB3 +2F4F>6BCB +2F50>6BD4 +2F51>6BDB +2F52>6C0F +2F53>6C14 +2F54>6C34 +2F55>706B +2F56>722A +2F57>7236 +2F58>723B +2F59>723F +2F5A>7247 +2F5B>7259 +2F5C>725B +2F5D>72AC +2F5E>7384 +2F5F>7389 +2F60>74DC +2F61>74E6 +2F62>7518 +2F63>751F +2F64>7528 +2F65>7530 +2F66>758B +2F67>7592 +2F68>7676 +2F69>767D +2F6A>76AE +2F6B>76BF +2F6C>76EE +2F6D>77DB +2F6E>77E2 +2F6F>77F3 +2F70>793A +2F71>79B8 +2F72>79BE +2F73>7A74 +2F74>7ACB +2F75>7AF9 +2F76>7C73 +2F77>7CF8 +2F78>7F36 +2F79>7F51 +2F7A>7F8A +2F7B>7FBD +2F7C>8001 +2F7D>800C +2F7E>8012 +2F7F>8033 +2F80>807F +2F81>8089 +2F82>81E3 +2F83>81EA +2F84>81F3 +2F85>81FC +2F86>820C +2F87>821B +2F88>821F +2F89>826E +2F8A>8272 +2F8B>8278 +2F8C>864D +2F8D>866B +2F8E>8840 +2F8F>884C +2F90>8863 +2F91>897E +2F92>898B +2F93>89D2 +2F94>8A00 +2F95>8C37 +2F96>8C46 +2F97>8C55 +2F98>8C78 +2F99>8C9D +2F9A>8D64 +2F9B>8D70 +2F9C>8DB3 +2F9D>8EAB +2F9E>8ECA +2F9F>8F9B +2FA0>8FB0 +2FA1>8FB5 +2FA2>9091 +2FA3>9149 +2FA4>91C6 +2FA5>91CC +2FA6>91D1 +2FA7>9577 +2FA8>9580 +2FA9>961C +2FAA>96B6 +2FAB>96B9 +2FAC>96E8 +2FAD>9751 +2FAE>975E +2FAF>9762 +2FB0>9769 +2FB1>97CB +2FB2>97ED +2FB3>97F3 +2FB4>9801 +2FB5>98A8 +2FB6>98DB +2FB7>98DF +2FB8>9996 +2FB9>9999 +2FBA>99AC +2FBB>9AA8 +2FBC>9AD8 +2FBD>9ADF +2FBE>9B25 +2FBF>9B2F +2FC0>9B32 +2FC1>9B3C +2FC2>9B5A +2FC3>9CE5 +2FC4>9E75 +2FC5>9E7F +2FC6>9EA5 +2FC7>9EBB +2FC8>9EC3 +2FC9>9ECD +2FCA>9ED1 +2FCB>9EF9 +2FCC>9EFD +2FCD>9F0E +2FCE>9F13 +2FCF>9F20 +2FD0>9F3B +2FD1>9F4A +2FD2>9F52 +2FD3>9F8D +2FD4>9F9C +2FD5>9FA0 +3000>0020 +3036>3012 +3038>5341 +3039>5344 +303A>5345 +309B>0020 3099 +309C>0020 309A +309F>3088 308A +30FF>30B3 30C8 +3131>1100 +3132>1101 +3133>11AA +3134>1102 +3135>11AC +3136>11AD +3137>1103 +3138>1104 +3139>1105 +313A>11B0 +313B>11B1 +313C>11B2 +313D>11B3 +313E>11B4 +313F>11B5 +3140>111A +3141>1106 +3142>1107 +3143>1108 +3144>1121 +3145>1109 +3146>110A +3147>110B +3148>110C +3149>110D +314A>110E +314B>110F +314C>1110 +314D>1111 +314E>1112 +314F>1161 +3150>1162 +3151>1163 +3152>1164 +3153>1165 +3154>1166 +3155>1167 +3156>1168 +3157>1169 +3158>116A +3159>116B +315A>116C +315B>116D +315C>116E +315D>116F +315E>1170 +315F>1171 +3160>1172 +3161>1173 +3162>1174 +3163>1175 +3164> +3165>1114 +3166>1115 +3167>11C7 +3168>11C8 +3169>11CC +316A>11CE +316B>11D3 +316C>11D7 +316D>11D9 +316E>111C +316F>11DD +3170>11DF +3171>111D +3172>111E +3173>1120 +3174>1122 +3175>1123 +3176>1127 +3177>1129 +3178>112B +3179>112C +317A>112D +317B>112E +317C>112F +317D>1132 +317E>1136 +317F>1140 +3180>1147 +3181>114C +3182>11F1 +3183>11F2 +3184>1157 +3185>1158 +3186>1159 +3187>1184 +3188>1185 +3189>1188 +318A>1191 +318B>1192 +318C>1194 +318D>119E +318E>11A1 +3192>4E00 +3193>4E8C +3194>4E09 +3195>56DB +3196>4E0A +3197>4E2D +3198>4E0B +3199>7532 +319A>4E59 +319B>4E19 +319C>4E01 +319D>5929 +319E>5730 +319F>4EBA +3200>0028 1100 0029 +3201>0028 1102 0029 +3202>0028 1103 0029 +3203>0028 1105 0029 +3204>0028 1106 0029 +3205>0028 1107 0029 +3206>0028 1109 0029 +3207>0028 110B 0029 +3208>0028 110C 0029 +3209>0028 110E 0029 +320A>0028 110F 0029 +320B>0028 1110 0029 +320C>0028 1111 0029 +320D>0028 1112 0029 +320E>0028 AC00 0029 +320F>0028 B098 0029 +3210>0028 B2E4 0029 +3211>0028 B77C 0029 +3212>0028 B9C8 0029 +3213>0028 BC14 0029 +3214>0028 C0AC 0029 +3215>0028 C544 0029 +3216>0028 C790 0029 +3217>0028 CC28 0029 +3218>0028 CE74 0029 +3219>0028 D0C0 0029 +321A>0028 D30C 0029 +321B>0028 D558 0029 +321C>0028 C8FC 0029 +321D>0028 C624 C804 0029 +321E>0028 C624 D6C4 0029 +3220>0028 4E00 0029 +3221>0028 4E8C 0029 +3222>0028 4E09 0029 +3223>0028 56DB 0029 +3224>0028 4E94 0029 +3225>0028 516D 0029 +3226>0028 4E03 0029 +3227>0028 516B 0029 +3228>0028 4E5D 0029 +3229>0028 5341 0029 +322A>0028 6708 0029 +322B>0028 706B 0029 +322C>0028 6C34 0029 +322D>0028 6728 0029 +322E>0028 91D1 0029 +322F>0028 571F 0029 +3230>0028 65E5 0029 +3231>0028 682A 0029 +3232>0028 6709 0029 +3233>0028 793E 0029 +3234>0028 540D 0029 +3235>0028 7279 0029 +3236>0028 8CA1 0029 +3237>0028 795D 0029 +3238>0028 52B4 0029 +3239>0028 4EE3 0029 +323A>0028 547C 0029 +323B>0028 5B66 0029 +323C>0028 76E3 0029 +323D>0028 4F01 0029 +323E>0028 8CC7 0029 +323F>0028 5354 0029 +3240>0028 796D 0029 +3241>0028 4F11 0029 +3242>0028 81EA 0029 +3243>0028 81F3 0029 +3244>554F +3245>5E7C +3246>6587 +3247>7B8F +3250>0070 0074 0065 +3251>0032 0031 +3252>0032 0032 +3253>0032 0033 +3254>0032 0034 +3255>0032 0035 +3256>0032 0036 +3257>0032 0037 +3258>0032 0038 +3259>0032 0039 +325A>0033 0030 +325B>0033 0031 +325C>0033 0032 +325D>0033 0033 +325E>0033 0034 +325F>0033 0035 +3260>1100 +3261>1102 +3262>1103 +3263>1105 +3264>1106 +3265>1107 +3266>1109 +3267>110B +3268>110C +3269>110E +326A>110F +326B>1110 +326C>1111 +326D>1112 +326E>AC00 +326F>B098 +3270>B2E4 +3271>B77C +3272>B9C8 +3273>BC14 +3274>C0AC +3275>C544 +3276>C790 +3277>CC28 +3278>CE74 +3279>D0C0 +327A>D30C +327B>D558 +327C>CC38 ACE0 +327D>C8FC C758 +327E>C6B0 +3280>4E00 +3281>4E8C +3282>4E09 +3283>56DB +3284>4E94 +3285>516D +3286>4E03 +3287>516B +3288>4E5D +3289>5341 +328A>6708 +328B>706B +328C>6C34 +328D>6728 +328E>91D1 +328F>571F +3290>65E5 +3291>682A +3292>6709 +3293>793E +3294>540D +3295>7279 +3296>8CA1 +3297>795D +3298>52B4 +3299>79D8 +329A>7537 +329B>5973 +329C>9069 +329D>512A +329E>5370 +329F>6CE8 +32A0>9805 +32A1>4F11 +32A2>5199 +32A3>6B63 +32A4>4E0A +32A5>4E2D +32A6>4E0B +32A7>5DE6 +32A8>53F3 +32A9>533B +32AA>5B97 +32AB>5B66 +32AC>76E3 +32AD>4F01 +32AE>8CC7 +32AF>5354 +32B0>591C +32B1>0033 0036 +32B2>0033 0037 +32B3>0033 0038 +32B4>0033 0039 +32B5>0034 0030 +32B6>0034 0031 +32B7>0034 0032 +32B8>0034 0033 +32B9>0034 0034 +32BA>0034 0035 +32BB>0034 0036 +32BC>0034 0037 +32BD>0034 0038 +32BE>0034 0039 +32BF>0035 0030 +32C0>0031 6708 +32C1>0032 6708 +32C2>0033 6708 +32C3>0034 6708 +32C4>0035 6708 +32C5>0036 6708 +32C6>0037 6708 +32C7>0038 6708 +32C8>0039 6708 +32C9>0031 0030 6708 +32CA>0031 0031 6708 +32CB>0031 0032 6708 +32CC>0068 0067 +32CD>0065 0072 0067 +32CE>0065 0076 +32CF>006C 0074 0064 +32D0>30A2 +32D1>30A4 +32D2>30A6 +32D3>30A8 +32D4>30AA +32D5>30AB +32D6>30AD +32D7>30AF +32D8>30B1 +32D9>30B3 +32DA>30B5 +32DB>30B7 +32DC>30B9 +32DD>30BB +32DE>30BD +32DF>30BF +32E0>30C1 +32E1>30C4 +32E2>30C6 +32E3>30C8 +32E4>30CA +32E5>30CB +32E6>30CC +32E7>30CD +32E8>30CE +32E9>30CF +32EA>30D2 +32EB>30D5 +32EC>30D8 +32ED>30DB +32EE>30DE +32EF>30DF +32F0>30E0 +32F1>30E1 +32F2>30E2 +32F3>30E4 +32F4>30E6 +32F5>30E8 +32F6>30E9 +32F7>30EA +32F8>30EB +32F9>30EC +32FA>30ED +32FB>30EF +32FC>30F0 +32FD>30F1 +32FE>30F2 +3300>30A2 30D1 30FC 30C8 +3301>30A2 30EB 30D5 30A1 +3302>30A2 30F3 30DA 30A2 +3303>30A2 30FC 30EB +3304>30A4 30CB 30F3 30B0 +3305>30A4 30F3 30C1 +3306>30A6 30A9 30F3 +3307>30A8 30B9 30AF 30FC 30C9 +3308>30A8 30FC 30AB 30FC +3309>30AA 30F3 30B9 +330A>30AA 30FC 30E0 +330B>30AB 30A4 30EA +330C>30AB 30E9 30C3 30C8 +330D>30AB 30ED 30EA 30FC +330E>30AC 30ED 30F3 +330F>30AC 30F3 30DE +3310>30AE 30AC +3311>30AE 30CB 30FC +3312>30AD 30E5 30EA 30FC +3313>30AE 30EB 30C0 30FC +3314>30AD 30ED +3315>30AD 30ED 30B0 30E9 30E0 +3316>30AD 30ED 30E1 30FC 30C8 30EB +3317>30AD 30ED 30EF 30C3 30C8 +3318>30B0 30E9 30E0 +3319>30B0 30E9 30E0 30C8 30F3 +331A>30AF 30EB 30BC 30A4 30ED +331B>30AF 30ED 30FC 30CD +331C>30B1 30FC 30B9 +331D>30B3 30EB 30CA +331E>30B3 30FC 30DD +331F>30B5 30A4 30AF 30EB +3320>30B5 30F3 30C1 30FC 30E0 +3321>30B7 30EA 30F3 30B0 +3322>30BB 30F3 30C1 +3323>30BB 30F3 30C8 +3324>30C0 30FC 30B9 +3325>30C7 30B7 +3326>30C9 30EB +3327>30C8 30F3 +3328>30CA 30CE +3329>30CE 30C3 30C8 +332A>30CF 30A4 30C4 +332B>30D1 30FC 30BB 30F3 30C8 +332C>30D1 30FC 30C4 +332D>30D0 30FC 30EC 30EB +332E>30D4 30A2 30B9 30C8 30EB +332F>30D4 30AF 30EB +3330>30D4 30B3 +3331>30D3 30EB +3332>30D5 30A1 30E9 30C3 30C9 +3333>30D5 30A3 30FC 30C8 +3334>30D6 30C3 30B7 30A7 30EB +3335>30D5 30E9 30F3 +3336>30D8 30AF 30BF 30FC 30EB +3337>30DA 30BD +3338>30DA 30CB 30D2 +3339>30D8 30EB 30C4 +333A>30DA 30F3 30B9 +333B>30DA 30FC 30B8 +333C>30D9 30FC 30BF +333D>30DD 30A4 30F3 30C8 +333E>30DC 30EB 30C8 +333F>30DB 30F3 +3340>30DD 30F3 30C9 +3341>30DB 30FC 30EB +3342>30DB 30FC 30F3 +3343>30DE 30A4 30AF 30ED +3344>30DE 30A4 30EB +3345>30DE 30C3 30CF +3346>30DE 30EB 30AF +3347>30DE 30F3 30B7 30E7 30F3 +3348>30DF 30AF 30ED 30F3 +3349>30DF 30EA +334A>30DF 30EA 30D0 30FC 30EB +334B>30E1 30AC +334C>30E1 30AC 30C8 30F3 +334D>30E1 30FC 30C8 30EB +334E>30E4 30FC 30C9 +334F>30E4 30FC 30EB +3350>30E6 30A2 30F3 +3351>30EA 30C3 30C8 30EB +3352>30EA 30E9 +3353>30EB 30D4 30FC +3354>30EB 30FC 30D6 30EB +3355>30EC 30E0 +3356>30EC 30F3 30C8 30B2 30F3 +3357>30EF 30C3 30C8 +3358>0030 70B9 +3359>0031 70B9 +335A>0032 70B9 +335B>0033 70B9 +335C>0034 70B9 +335D>0035 70B9 +335E>0036 70B9 +335F>0037 70B9 +3360>0038 70B9 +3361>0039 70B9 +3362>0031 0030 70B9 +3363>0031 0031 70B9 +3364>0031 0032 70B9 +3365>0031 0033 70B9 +3366>0031 0034 70B9 +3367>0031 0035 70B9 +3368>0031 0036 70B9 +3369>0031 0037 70B9 +336A>0031 0038 70B9 +336B>0031 0039 70B9 +336C>0032 0030 70B9 +336D>0032 0031 70B9 +336E>0032 0032 70B9 +336F>0032 0033 70B9 +3370>0032 0034 70B9 +3371>0068 0070 0061 +3372>0064 0061 +3373>0061 0075 +3374>0062 0061 0072 +3375>006F 0076 +3376>0070 0063 +3377>0064 006D +3378>0064 006D 0032 +3379>0064 006D 0033 +337A>0069 0075 +337B>5E73 6210 +337C>662D 548C +337D>5927 6B63 +337E>660E 6CBB +337F>682A 5F0F 4F1A 793E +3380>0070 0061 +3381>006E 0061 +3382>03BC 0061 +3383>006D 0061 +3384>006B 0061 +3385>006B 0062 +3386>006D 0062 +3387>0067 0062 +3388>0063 0061 006C +3389>006B 0063 0061 006C +338A>0070 0066 +338B>006E 0066 +338C>03BC 0066 +338D>03BC 0067 +338E>006D 0067 +338F>006B 0067 +3390>0068 007A +3391>006B 0068 007A +3392>006D 0068 007A +3393>0067 0068 007A +3394>0074 0068 007A +3395>03BC 006C +3396>006D 006C +3397>0064 006C +3398>006B 006C +3399>0066 006D +339A>006E 006D +339B>03BC 006D +339C>006D 006D +339D>0063 006D +339E>006B 006D +339F>006D 006D 0032 +33A0>0063 006D 0032 +33A1>006D 0032 +33A2>006B 006D 0032 +33A3>006D 006D 0033 +33A4>0063 006D 0033 +33A5>006D 0033 +33A6>006B 006D 0033 +33A7>006D 2215 0073 +33A8>006D 2215 0073 0032 +33A9>0070 0061 +33AA>006B 0070 0061 +33AB>006D 0070 0061 +33AC>0067 0070 0061 +33AD>0072 0061 0064 +33AE>0072 0061 0064 2215 0073 +33AF>0072 0061 0064 2215 0073 0032 +33B0>0070 0073 +33B1>006E 0073 +33B2>03BC 0073 +33B3>006D 0073 +33B4>0070 0076 +33B5>006E 0076 +33B6>03BC 0076 +33B7>006D 0076 +33B8>006B 0076 +33B9>006D 0076 +33BA>0070 0077 +33BB>006E 0077 +33BC>03BC 0077 +33BD>006D 0077 +33BE>006B 0077 +33BF>006D 0077 +33C0>006B 03C9 +33C1>006D 03C9 +33C2>0061 002E 006D 002E +33C3>0062 0071 +33C4>0063 0063 +33C5>0063 0064 +33C6>0063 2215 006B 0067 +33C7>0063 006F 002E +33C8>0064 0062 +33C9>0067 0079 +33CA>0068 0061 +33CB>0068 0070 +33CC>0069 006E +33CD>006B 006B +33CE>006B 006D +33CF>006B 0074 +33D0>006C 006D +33D1>006C 006E +33D2>006C 006F 0067 +33D3>006C 0078 +33D4>006D 0062 +33D5>006D 0069 006C +33D6>006D 006F 006C +33D7>0070 0068 +33D8>0070 002E 006D 002E +33D9>0070 0070 006D +33DA>0070 0072 +33DB>0073 0072 +33DC>0073 0076 +33DD>0077 0062 +33DE>0076 2215 006D +33DF>0061 2215 006D +33E0>0031 65E5 +33E1>0032 65E5 +33E2>0033 65E5 +33E3>0034 65E5 +33E4>0035 65E5 +33E5>0036 65E5 +33E6>0037 65E5 +33E7>0038 65E5 +33E8>0039 65E5 +33E9>0031 0030 65E5 +33EA>0031 0031 65E5 +33EB>0031 0032 65E5 +33EC>0031 0033 65E5 +33ED>0031 0034 65E5 +33EE>0031 0035 65E5 +33EF>0031 0036 65E5 +33F0>0031 0037 65E5 +33F1>0031 0038 65E5 +33F2>0031 0039 65E5 +33F3>0032 0030 65E5 +33F4>0032 0031 65E5 +33F5>0032 0032 65E5 +33F6>0032 0033 65E5 +33F7>0032 0034 65E5 +33F8>0032 0035 65E5 +33F9>0032 0036 65E5 +33FA>0032 0037 65E5 +33FB>0032 0038 65E5 +33FC>0032 0039 65E5 +33FD>0033 0030 65E5 +33FE>0033 0031 65E5 +33FF>0067 0061 006C +A640>A641 +A642>A643 +A644>A645 +A646>A647 +A648>A649 +A64A>A64B +A64C>A64D +A64E>A64F +A650>A651 +A652>A653 +A654>A655 +A656>A657 +A658>A659 +A65A>A65B +A65C>A65D +A65E>A65F +A662>A663 +A664>A665 +A666>A667 +A668>A669 +A66A>A66B +A66C>A66D +A680>A681 +A682>A683 +A684>A685 +A686>A687 +A688>A689 +A68A>A68B +A68C>A68D +A68E>A68F +A690>A691 +A692>A693 +A694>A695 +A696>A697 +A722>A723 +A724>A725 +A726>A727 +A728>A729 +A72A>A72B +A72C>A72D +A72E>A72F +A732>A733 +A734>A735 +A736>A737 +A738>A739 +A73A>A73B +A73C>A73D +A73E>A73F +A740>A741 +A742>A743 +A744>A745 +A746>A747 +A748>A749 +A74A>A74B +A74C>A74D +A74E>A74F +A750>A751 +A752>A753 +A754>A755 +A756>A757 +A758>A759 +A75A>A75B +A75C>A75D +A75E>A75F +A760>A761 +A762>A763 +A764>A765 +A766>A767 +A768>A769 +A76A>A76B +A76C>A76D +A76E>A76F +A770>A76F +A779>A77A +A77B>A77C +A77D>1D79 +A77E>A77F +A780>A781 +A782>A783 +A784>A785 +A786>A787 +A78B>A78C +F900>8C48 +F901>66F4 +F902>8ECA +F903>8CC8 +F904>6ED1 +F905>4E32 +F906>53E5 +F907..F908>9F9C +F909>5951 +F90A>91D1 +F90B>5587 +F90C>5948 +F90D>61F6 +F90E>7669 +F90F>7F85 +F910>863F +F911>87BA +F912>88F8 +F913>908F +F914>6A02 +F915>6D1B +F916>70D9 +F917>73DE +F918>843D +F919>916A +F91A>99F1 +F91B>4E82 +F91C>5375 +F91D>6B04 +F91E>721B +F91F>862D +F920>9E1E +F921>5D50 +F922>6FEB +F923>85CD +F924>8964 +F925>62C9 +F926>81D8 +F927>881F +F928>5ECA +F929>6717 +F92A>6D6A +F92B>72FC +F92C>90CE +F92D>4F86 +F92E>51B7 +F92F>52DE +F930>64C4 +F931>6AD3 +F932>7210 +F933>76E7 +F934>8001 +F935>8606 +F936>865C +F937>8DEF +F938>9732 +F939>9B6F +F93A>9DFA +F93B>788C +F93C>797F +F93D>7DA0 +F93E>83C9 +F93F>9304 +F940>9E7F +F941>8AD6 +F942>58DF +F943>5F04 +F944>7C60 +F945>807E +F946>7262 +F947>78CA +F948>8CC2 +F949>96F7 +F94A>58D8 +F94B>5C62 +F94C>6A13 +F94D>6DDA +F94E>6F0F +F94F>7D2F +F950>7E37 +F951>964B +F952>52D2 +F953>808B +F954>51DC +F955>51CC +F956>7A1C +F957>7DBE +F958>83F1 +F959>9675 +F95A>8B80 +F95B>62CF +F95C>6A02 +F95D>8AFE +F95E>4E39 +F95F>5BE7 +F960>6012 +F961>7387 +F962>7570 +F963>5317 +F964>78FB +F965>4FBF +F966>5FA9 +F967>4E0D +F968>6CCC +F969>6578 +F96A>7D22 +F96B>53C3 +F96C>585E +F96D>7701 +F96E>8449 +F96F>8AAA +F970>6BBA +F971>8FB0 +F972>6C88 +F973>62FE +F974>82E5 +F975>63A0 +F976>7565 +F977>4EAE +F978>5169 +F979>51C9 +F97A>6881 +F97B>7CE7 +F97C>826F +F97D>8AD2 +F97E>91CF +F97F>52F5 +F980>5442 +F981>5973 +F982>5EEC +F983>65C5 +F984>6FFE +F985>792A +F986>95AD +F987>9A6A +F988>9E97 +F989>9ECE +F98A>529B +F98B>66C6 +F98C>6B77 +F98D>8F62 +F98E>5E74 +F98F>6190 +F990>6200 +F991>649A +F992>6F23 +F993>7149 +F994>7489 +F995>79CA +F996>7DF4 +F997>806F +F998>8F26 +F999>84EE +F99A>9023 +F99B>934A +F99C>5217 +F99D>52A3 +F99E>54BD +F99F>70C8 +F9A0>88C2 +F9A1>8AAA +F9A2>5EC9 +F9A3>5FF5 +F9A4>637B +F9A5>6BAE +F9A6>7C3E +F9A7>7375 +F9A8>4EE4 +F9A9>56F9 +F9AA>5BE7 +F9AB>5DBA +F9AC>601C +F9AD>73B2 +F9AE>7469 +F9AF>7F9A +F9B0>8046 +F9B1>9234 +F9B2>96F6 +F9B3>9748 +F9B4>9818 +F9B5>4F8B +F9B6>79AE +F9B7>91B4 +F9B8>96B8 +F9B9>60E1 +F9BA>4E86 +F9BB>50DA +F9BC>5BEE +F9BD>5C3F +F9BE>6599 +F9BF>6A02 +F9C0>71CE +F9C1>7642 +F9C2>84FC +F9C3>907C +F9C4>9F8D +F9C5>6688 +F9C6>962E +F9C7>5289 +F9C8>677B +F9C9>67F3 +F9CA>6D41 +F9CB>6E9C +F9CC>7409 +F9CD>7559 +F9CE>786B +F9CF>7D10 +F9D0>985E +F9D1>516D +F9D2>622E +F9D3>9678 +F9D4>502B +F9D5>5D19 +F9D6>6DEA +F9D7>8F2A +F9D8>5F8B +F9D9>6144 +F9DA>6817 +F9DB>7387 +F9DC>9686 +F9DD>5229 +F9DE>540F +F9DF>5C65 +F9E0>6613 +F9E1>674E +F9E2>68A8 +F9E3>6CE5 +F9E4>7406 +F9E5>75E2 +F9E6>7F79 +F9E7>88CF +F9E8>88E1 +F9E9>91CC +F9EA>96E2 +F9EB>533F +F9EC>6EBA +F9ED>541D +F9EE>71D0 +F9EF>7498 +F9F0>85FA +F9F1>96A3 +F9F2>9C57 +F9F3>9E9F +F9F4>6797 +F9F5>6DCB +F9F6>81E8 +F9F7>7ACB +F9F8>7B20 +F9F9>7C92 +F9FA>72C0 +F9FB>7099 +F9FC>8B58 +F9FD>4EC0 +F9FE>8336 +F9FF>523A +FA00>5207 +FA01>5EA6 +FA02>62D3 +FA03>7CD6 +FA04>5B85 +FA05>6D1E +FA06>66B4 +FA07>8F3B +FA08>884C +FA09>964D +FA0A>898B +FA0B>5ED3 +FA0C>5140 +FA0D>55C0 +FA10>585A +FA12>6674 +FA15>51DE +FA16>732A +FA17>76CA +FA18>793C +FA19>795E +FA1A>7965 +FA1B>798F +FA1C>9756 +FA1D>7CBE +FA1E>7FBD +FA20>8612 +FA22>8AF8 +FA25>9038 +FA26>90FD +FA2A>98EF +FA2B>98FC +FA2C>9928 +FA2D>9DB4 +FA30>4FAE +FA31>50E7 +FA32>514D +FA33>52C9 +FA34>52E4 +FA35>5351 +FA36>559D +FA37>5606 +FA38>5668 +FA39>5840 +FA3A>58A8 +FA3B>5C64 +FA3C>5C6E +FA3D>6094 +FA3E>6168 +FA3F>618E +FA40>61F2 +FA41>654F +FA42>65E2 +FA43>6691 +FA44>6885 +FA45>6D77 +FA46>6E1A +FA47>6F22 +FA48>716E +FA49>722B +FA4A>7422 +FA4B>7891 +FA4C>793E +FA4D>7949 +FA4E>7948 +FA4F>7950 +FA50>7956 +FA51>795D +FA52>798D +FA53>798E +FA54>7A40 +FA55>7A81 +FA56>7BC0 +FA57>7DF4 +FA58>7E09 +FA59>7E41 +FA5A>7F72 +FA5B>8005 +FA5C>81ED +FA5D..FA5E>8279 +FA5F>8457 +FA60>8910 +FA61>8996 +FA62>8B01 +FA63>8B39 +FA64>8CD3 +FA65>8D08 +FA66>8FB6 +FA67>9038 +FA68>96E3 +FA69>97FF +FA6A>983B +FA6B>6075 +FA6C>242EE +FA6D>8218 +FA70>4E26 +FA71>51B5 +FA72>5168 +FA73>4F80 +FA74>5145 +FA75>5180 +FA76>52C7 +FA77>52FA +FA78>559D +FA79>5555 +FA7A>5599 +FA7B>55E2 +FA7C>585A +FA7D>58B3 +FA7E>5944 +FA7F>5954 +FA80>5A62 +FA81>5B28 +FA82>5ED2 +FA83>5ED9 +FA84>5F69 +FA85>5FAD +FA86>60D8 +FA87>614E +FA88>6108 +FA89>618E +FA8A>6160 +FA8B>61F2 +FA8C>6234 +FA8D>63C4 +FA8E>641C +FA8F>6452 +FA90>6556 +FA91>6674 +FA92>6717 +FA93>671B +FA94>6756 +FA95>6B79 +FA96>6BBA +FA97>6D41 +FA98>6EDB +FA99>6ECB +FA9A>6F22 +FA9B>701E +FA9C>716E +FA9D>77A7 +FA9E>7235 +FA9F>72AF +FAA0>732A +FAA1>7471 +FAA2>7506 +FAA3>753B +FAA4>761D +FAA5>761F +FAA6>76CA +FAA7>76DB +FAA8>76F4 +FAA9>774A +FAAA>7740 +FAAB>78CC +FAAC>7AB1 +FAAD>7BC0 +FAAE>7C7B +FAAF>7D5B +FAB0>7DF4 +FAB1>7F3E +FAB2>8005 +FAB3>8352 +FAB4>83EF +FAB5>8779 +FAB6>8941 +FAB7>8986 +FAB8>8996 +FAB9>8ABF +FABA>8AF8 +FABB>8ACB +FABC>8B01 +FABD>8AFE +FABE>8AED +FABF>8B39 +FAC0>8B8A +FAC1>8D08 +FAC2>8F38 +FAC3>9072 +FAC4>9199 +FAC5>9276 +FAC6>967C +FAC7>96E3 +FAC8>9756 +FAC9>97DB +FACA>97FF +FACB>980B +FACC>983B +FACD>9B12 +FACE>9F9C +FACF>2284A +FAD0>22844 +FAD1>233D5 +FAD2>3B9D +FAD3>4018 +FAD4>4039 +FAD5>25249 +FAD6>25CD0 +FAD7>27ED3 +FAD8>9F43 +FAD9>9F8E +FB00>0066 0066 +FB01>0066 0069 +FB02>0066 006C +FB03>0066 0066 0069 +FB04>0066 0066 006C +FB05..FB06>0073 0074 +FB13>0574 0576 +FB14>0574 0565 +FB15>0574 056B +FB16>057E 0576 +FB17>0574 056D +FB1D>05D9 05B4 +FB1F>05F2 05B7 +FB20>05E2 +FB21>05D0 +FB22>05D3 +FB23>05D4 +FB24>05DB +FB25>05DC +FB26>05DD +FB27>05E8 +FB28>05EA +FB29>002B +FB2A>05E9 05C1 +FB2B>05E9 05C2 +FB2C>05E9 05BC 05C1 +FB2D>05E9 05BC 05C2 +FB2E>05D0 05B7 +FB2F>05D0 05B8 +FB30>05D0 05BC +FB31>05D1 05BC +FB32>05D2 05BC +FB33>05D3 05BC +FB34>05D4 05BC +FB35>05D5 05BC +FB36>05D6 05BC +FB38>05D8 05BC +FB39>05D9 05BC +FB3A>05DA 05BC +FB3B>05DB 05BC +FB3C>05DC 05BC +FB3E>05DE 05BC +FB40>05E0 05BC +FB41>05E1 05BC +FB43>05E3 05BC +FB44>05E4 05BC +FB46>05E6 05BC +FB47>05E7 05BC +FB48>05E8 05BC +FB49>05E9 05BC +FB4A>05EA 05BC +FB4B>05D5 05B9 +FB4C>05D1 05BF +FB4D>05DB 05BF +FB4E>05E4 05BF +FB4F>05D0 05DC +FB50..FB51>0671 +FB52..FB55>067B +FB56..FB59>067E +FB5A..FB5D>0680 +FB5E..FB61>067A +FB62..FB65>067F +FB66..FB69>0679 +FB6A..FB6D>06A4 +FB6E..FB71>06A6 +FB72..FB75>0684 +FB76..FB79>0683 +FB7A..FB7D>0686 +FB7E..FB81>0687 +FB82..FB83>068D +FB84..FB85>068C +FB86..FB87>068E +FB88..FB89>0688 +FB8A..FB8B>0698 +FB8C..FB8D>0691 +FB8E..FB91>06A9 +FB92..FB95>06AF +FB96..FB99>06B3 +FB9A..FB9D>06B1 +FB9E..FB9F>06BA +FBA0..FBA3>06BB +FBA4..FBA5>06C0 +FBA6..FBA9>06C1 +FBAA..FBAD>06BE +FBAE..FBAF>06D2 +FBB0..FBB1>06D3 +FBD3..FBD6>06AD +FBD7..FBD8>06C7 +FBD9..FBDA>06C6 +FBDB..FBDC>06C8 +FBDD>06C7 0674 +FBDE..FBDF>06CB +FBE0..FBE1>06C5 +FBE2..FBE3>06C9 +FBE4..FBE7>06D0 +FBE8..FBE9>0649 +FBEA..FBEB>0626 0627 +FBEC..FBED>0626 06D5 +FBEE..FBEF>0626 0648 +FBF0..FBF1>0626 06C7 +FBF2..FBF3>0626 06C6 +FBF4..FBF5>0626 06C8 +FBF6..FBF8>0626 06D0 +FBF9..FBFB>0626 0649 +FBFC..FBFF>06CC +FC00>0626 062C +FC01>0626 062D +FC02>0626 0645 +FC03>0626 0649 +FC04>0626 064A +FC05>0628 062C +FC06>0628 062D +FC07>0628 062E +FC08>0628 0645 +FC09>0628 0649 +FC0A>0628 064A +FC0B>062A 062C +FC0C>062A 062D +FC0D>062A 062E +FC0E>062A 0645 +FC0F>062A 0649 +FC10>062A 064A +FC11>062B 062C +FC12>062B 0645 +FC13>062B 0649 +FC14>062B 064A +FC15>062C 062D +FC16>062C 0645 +FC17>062D 062C +FC18>062D 0645 +FC19>062E 062C +FC1A>062E 062D +FC1B>062E 0645 +FC1C>0633 062C +FC1D>0633 062D +FC1E>0633 062E +FC1F>0633 0645 +FC20>0635 062D +FC21>0635 0645 +FC22>0636 062C +FC23>0636 062D +FC24>0636 062E +FC25>0636 0645 +FC26>0637 062D +FC27>0637 0645 +FC28>0638 0645 +FC29>0639 062C +FC2A>0639 0645 +FC2B>063A 062C +FC2C>063A 0645 +FC2D>0641 062C +FC2E>0641 062D +FC2F>0641 062E +FC30>0641 0645 +FC31>0641 0649 +FC32>0641 064A +FC33>0642 062D +FC34>0642 0645 +FC35>0642 0649 +FC36>0642 064A +FC37>0643 0627 +FC38>0643 062C +FC39>0643 062D +FC3A>0643 062E +FC3B>0643 0644 +FC3C>0643 0645 +FC3D>0643 0649 +FC3E>0643 064A +FC3F>0644 062C +FC40>0644 062D +FC41>0644 062E +FC42>0644 0645 +FC43>0644 0649 +FC44>0644 064A +FC45>0645 062C +FC46>0645 062D +FC47>0645 062E +FC48>0645 0645 +FC49>0645 0649 +FC4A>0645 064A +FC4B>0646 062C +FC4C>0646 062D +FC4D>0646 062E +FC4E>0646 0645 +FC4F>0646 0649 +FC50>0646 064A +FC51>0647 062C +FC52>0647 0645 +FC53>0647 0649 +FC54>0647 064A +FC55>064A 062C +FC56>064A 062D +FC57>064A 062E +FC58>064A 0645 +FC59>064A 0649 +FC5A>064A 064A +FC5B>0630 0670 +FC5C>0631 0670 +FC5D>0649 0670 +FC5E>0020 064C 0651 +FC5F>0020 064D 0651 +FC60>0020 064E 0651 +FC61>0020 064F 0651 +FC62>0020 0650 0651 +FC63>0020 0651 0670 +FC64>0626 0631 +FC65>0626 0632 +FC66>0626 0645 +FC67>0626 0646 +FC68>0626 0649 +FC69>0626 064A +FC6A>0628 0631 +FC6B>0628 0632 +FC6C>0628 0645 +FC6D>0628 0646 +FC6E>0628 0649 +FC6F>0628 064A +FC70>062A 0631 +FC71>062A 0632 +FC72>062A 0645 +FC73>062A 0646 +FC74>062A 0649 +FC75>062A 064A +FC76>062B 0631 +FC77>062B 0632 +FC78>062B 0645 +FC79>062B 0646 +FC7A>062B 0649 +FC7B>062B 064A +FC7C>0641 0649 +FC7D>0641 064A +FC7E>0642 0649 +FC7F>0642 064A +FC80>0643 0627 +FC81>0643 0644 +FC82>0643 0645 +FC83>0643 0649 +FC84>0643 064A +FC85>0644 0645 +FC86>0644 0649 +FC87>0644 064A +FC88>0645 0627 +FC89>0645 0645 +FC8A>0646 0631 +FC8B>0646 0632 +FC8C>0646 0645 +FC8D>0646 0646 +FC8E>0646 0649 +FC8F>0646 064A +FC90>0649 0670 +FC91>064A 0631 +FC92>064A 0632 +FC93>064A 0645 +FC94>064A 0646 +FC95>064A 0649 +FC96>064A 064A +FC97>0626 062C +FC98>0626 062D +FC99>0626 062E +FC9A>0626 0645 +FC9B>0626 0647 +FC9C>0628 062C +FC9D>0628 062D +FC9E>0628 062E +FC9F>0628 0645 +FCA0>0628 0647 +FCA1>062A 062C +FCA2>062A 062D +FCA3>062A 062E +FCA4>062A 0645 +FCA5>062A 0647 +FCA6>062B 0645 +FCA7>062C 062D +FCA8>062C 0645 +FCA9>062D 062C +FCAA>062D 0645 +FCAB>062E 062C +FCAC>062E 0645 +FCAD>0633 062C +FCAE>0633 062D +FCAF>0633 062E +FCB0>0633 0645 +FCB1>0635 062D +FCB2>0635 062E +FCB3>0635 0645 +FCB4>0636 062C +FCB5>0636 062D +FCB6>0636 062E +FCB7>0636 0645 +FCB8>0637 062D +FCB9>0638 0645 +FCBA>0639 062C +FCBB>0639 0645 +FCBC>063A 062C +FCBD>063A 0645 +FCBE>0641 062C +FCBF>0641 062D +FCC0>0641 062E +FCC1>0641 0645 +FCC2>0642 062D +FCC3>0642 0645 +FCC4>0643 062C +FCC5>0643 062D +FCC6>0643 062E +FCC7>0643 0644 +FCC8>0643 0645 +FCC9>0644 062C +FCCA>0644 062D +FCCB>0644 062E +FCCC>0644 0645 +FCCD>0644 0647 +FCCE>0645 062C +FCCF>0645 062D +FCD0>0645 062E +FCD1>0645 0645 +FCD2>0646 062C +FCD3>0646 062D +FCD4>0646 062E +FCD5>0646 0645 +FCD6>0646 0647 +FCD7>0647 062C +FCD8>0647 0645 +FCD9>0647 0670 +FCDA>064A 062C +FCDB>064A 062D +FCDC>064A 062E +FCDD>064A 0645 +FCDE>064A 0647 +FCDF>0626 0645 +FCE0>0626 0647 +FCE1>0628 0645 +FCE2>0628 0647 +FCE3>062A 0645 +FCE4>062A 0647 +FCE5>062B 0645 +FCE6>062B 0647 +FCE7>0633 0645 +FCE8>0633 0647 +FCE9>0634 0645 +FCEA>0634 0647 +FCEB>0643 0644 +FCEC>0643 0645 +FCED>0644 0645 +FCEE>0646 0645 +FCEF>0646 0647 +FCF0>064A 0645 +FCF1>064A 0647 +FCF2>0640 064E 0651 +FCF3>0640 064F 0651 +FCF4>0640 0650 0651 +FCF5>0637 0649 +FCF6>0637 064A +FCF7>0639 0649 +FCF8>0639 064A +FCF9>063A 0649 +FCFA>063A 064A +FCFB>0633 0649 +FCFC>0633 064A +FCFD>0634 0649 +FCFE>0634 064A +FCFF>062D 0649 +FD00>062D 064A +FD01>062C 0649 +FD02>062C 064A +FD03>062E 0649 +FD04>062E 064A +FD05>0635 0649 +FD06>0635 064A +FD07>0636 0649 +FD08>0636 064A +FD09>0634 062C +FD0A>0634 062D +FD0B>0634 062E +FD0C>0634 0645 +FD0D>0634 0631 +FD0E>0633 0631 +FD0F>0635 0631 +FD10>0636 0631 +FD11>0637 0649 +FD12>0637 064A +FD13>0639 0649 +FD14>0639 064A +FD15>063A 0649 +FD16>063A 064A +FD17>0633 0649 +FD18>0633 064A +FD19>0634 0649 +FD1A>0634 064A +FD1B>062D 0649 +FD1C>062D 064A +FD1D>062C 0649 +FD1E>062C 064A +FD1F>062E 0649 +FD20>062E 064A +FD21>0635 0649 +FD22>0635 064A +FD23>0636 0649 +FD24>0636 064A +FD25>0634 062C +FD26>0634 062D +FD27>0634 062E +FD28>0634 0645 +FD29>0634 0631 +FD2A>0633 0631 +FD2B>0635 0631 +FD2C>0636 0631 +FD2D>0634 062C +FD2E>0634 062D +FD2F>0634 062E +FD30>0634 0645 +FD31>0633 0647 +FD32>0634 0647 +FD33>0637 0645 +FD34>0633 062C +FD35>0633 062D +FD36>0633 062E +FD37>0634 062C +FD38>0634 062D +FD39>0634 062E +FD3A>0637 0645 +FD3B>0638 0645 +FD3C..FD3D>0627 064B +FD50>062A 062C 0645 +FD51..FD52>062A 062D 062C +FD53>062A 062D 0645 +FD54>062A 062E 0645 +FD55>062A 0645 062C +FD56>062A 0645 062D +FD57>062A 0645 062E +FD58..FD59>062C 0645 062D +FD5A>062D 0645 064A +FD5B>062D 0645 0649 +FD5C>0633 062D 062C +FD5D>0633 062C 062D +FD5E>0633 062C 0649 +FD5F..FD60>0633 0645 062D +FD61>0633 0645 062C +FD62..FD63>0633 0645 0645 +FD64..FD65>0635 062D 062D +FD66>0635 0645 0645 +FD67..FD68>0634 062D 0645 +FD69>0634 062C 064A +FD6A..FD6B>0634 0645 062E +FD6C..FD6D>0634 0645 0645 +FD6E>0636 062D 0649 +FD6F..FD70>0636 062E 0645 +FD71..FD72>0637 0645 062D +FD73>0637 0645 0645 +FD74>0637 0645 064A +FD75>0639 062C 0645 +FD76..FD77>0639 0645 0645 +FD78>0639 0645 0649 +FD79>063A 0645 0645 +FD7A>063A 0645 064A +FD7B>063A 0645 0649 +FD7C..FD7D>0641 062E 0645 +FD7E>0642 0645 062D +FD7F>0642 0645 0645 +FD80>0644 062D 0645 +FD81>0644 062D 064A +FD82>0644 062D 0649 +FD83..FD84>0644 062C 062C +FD85..FD86>0644 062E 0645 +FD87..FD88>0644 0645 062D +FD89>0645 062D 062C +FD8A>0645 062D 0645 +FD8B>0645 062D 064A +FD8C>0645 062C 062D +FD8D>0645 062C 0645 +FD8E>0645 062E 062C +FD8F>0645 062E 0645 +FD92>0645 062C 062E +FD93>0647 0645 062C +FD94>0647 0645 0645 +FD95>0646 062D 0645 +FD96>0646 062D 0649 +FD97..FD98>0646 062C 0645 +FD99>0646 062C 0649 +FD9A>0646 0645 064A +FD9B>0646 0645 0649 +FD9C..FD9D>064A 0645 0645 +FD9E>0628 062E 064A +FD9F>062A 062C 064A +FDA0>062A 062C 0649 +FDA1>062A 062E 064A +FDA2>062A 062E 0649 +FDA3>062A 0645 064A +FDA4>062A 0645 0649 +FDA5>062C 0645 064A +FDA6>062C 062D 0649 +FDA7>062C 0645 0649 +FDA8>0633 062E 0649 +FDA9>0635 062D 064A +FDAA>0634 062D 064A +FDAB>0636 062D 064A +FDAC>0644 062C 064A +FDAD>0644 0645 064A +FDAE>064A 062D 064A +FDAF>064A 062C 064A +FDB0>064A 0645 064A +FDB1>0645 0645 064A +FDB2>0642 0645 064A +FDB3>0646 062D 064A +FDB4>0642 0645 062D +FDB5>0644 062D 0645 +FDB6>0639 0645 064A +FDB7>0643 0645 064A +FDB8>0646 062C 062D +FDB9>0645 062E 064A +FDBA>0644 062C 0645 +FDBB>0643 0645 0645 +FDBC>0644 062C 0645 +FDBD>0646 062C 062D +FDBE>062C 062D 064A +FDBF>062D 062C 064A +FDC0>0645 062C 064A +FDC1>0641 0645 064A +FDC2>0628 062D 064A +FDC3>0643 0645 0645 +FDC4>0639 062C 0645 +FDC5>0635 0645 0645 +FDC6>0633 062E 064A +FDC7>0646 062C 064A +FDF0>0635 0644 06D2 +FDF1>0642 0644 06D2 +FDF2>0627 0644 0644 0647 +FDF3>0627 0643 0628 0631 +FDF4>0645 062D 0645 062F +FDF5>0635 0644 0639 0645 +FDF6>0631 0633 0648 0644 +FDF7>0639 0644 064A 0647 +FDF8>0648 0633 0644 0645 +FDF9>0635 0644 0649 +FDFA>0635 0644 0649 0020 0627 0644 0644 0647 0020 0639 0644 064A 0647 0020 0648 0633 0644 0645 +FDFB>062C 0644 0020 062C 0644 0627 0644 0647 +FDFC>0631 06CC 0627 0644 +FE00..FE0F> +FE10>002C +FE11>3001 +FE12>3002 +FE13>003A +FE14>003B +FE15>0021 +FE16>003F +FE17>3016 +FE18>3017 +FE19>002E 002E 002E +FE30>002E 002E +FE31>2014 +FE32>2013 +FE33..FE34>005F +FE35>0028 +FE36>0029 +FE37>007B +FE38>007D +FE39>3014 +FE3A>3015 +FE3B>3010 +FE3C>3011 +FE3D>300A +FE3E>300B +FE3F>3008 +FE40>3009 +FE41>300C +FE42>300D +FE43>300E +FE44>300F +FE47>005B +FE48>005D +FE49..FE4C>0020 0305 +FE4D..FE4F>005F +FE50>002C +FE51>3001 +FE52>002E +FE54>003B +FE55>003A +FE56>003F +FE57>0021 +FE58>2014 +FE59>0028 +FE5A>0029 +FE5B>007B +FE5C>007D +FE5D>3014 +FE5E>3015 +FE5F>0023 +FE60>0026 +FE61>002A +FE62>002B +FE63>002D +FE64>003C +FE65>003E +FE66>003D +FE68>005C +FE69>0024 +FE6A>0025 +FE6B>0040 +FE70>0020 064B +FE71>0640 064B +FE72>0020 064C +FE74>0020 064D +FE76>0020 064E +FE77>0640 064E +FE78>0020 064F +FE79>0640 064F +FE7A>0020 0650 +FE7B>0640 0650 +FE7C>0020 0651 +FE7D>0640 0651 +FE7E>0020 0652 +FE7F>0640 0652 +FE80>0621 +FE81..FE82>0622 +FE83..FE84>0623 +FE85..FE86>0624 +FE87..FE88>0625 +FE89..FE8C>0626 +FE8D..FE8E>0627 +FE8F..FE92>0628 +FE93..FE94>0629 +FE95..FE98>062A +FE99..FE9C>062B +FE9D..FEA0>062C +FEA1..FEA4>062D +FEA5..FEA8>062E +FEA9..FEAA>062F +FEAB..FEAC>0630 +FEAD..FEAE>0631 +FEAF..FEB0>0632 +FEB1..FEB4>0633 +FEB5..FEB8>0634 +FEB9..FEBC>0635 +FEBD..FEC0>0636 +FEC1..FEC4>0637 +FEC5..FEC8>0638 +FEC9..FECC>0639 +FECD..FED0>063A +FED1..FED4>0641 +FED5..FED8>0642 +FED9..FEDC>0643 +FEDD..FEE0>0644 +FEE1..FEE4>0645 +FEE5..FEE8>0646 +FEE9..FEEC>0647 +FEED..FEEE>0648 +FEEF..FEF0>0649 +FEF1..FEF4>064A +FEF5..FEF6>0644 0622 +FEF7..FEF8>0644 0623 +FEF9..FEFA>0644 0625 +FEFB..FEFC>0644 0627 +FEFF> +FF01>0021 +FF02>0022 +FF03>0023 +FF04>0024 +FF05>0025 +FF06>0026 +FF07>0027 +FF08>0028 +FF09>0029 +FF0A>002A +FF0B>002B +FF0C>002C +FF0D>002D +FF0E>002E +FF0F>002F +FF10>0030 +FF11>0031 +FF12>0032 +FF13>0033 +FF14>0034 +FF15>0035 +FF16>0036 +FF17>0037 +FF18>0038 +FF19>0039 +FF1A>003A +FF1B>003B +FF1C>003C +FF1D>003D +FF1E>003E +FF1F>003F +FF20>0040 +FF21>0061 +FF22>0062 +FF23>0063 +FF24>0064 +FF25>0065 +FF26>0066 +FF27>0067 +FF28>0068 +FF29>0069 +FF2A>006A +FF2B>006B +FF2C>006C +FF2D>006D +FF2E>006E +FF2F>006F +FF30>0070 +FF31>0071 +FF32>0072 +FF33>0073 +FF34>0074 +FF35>0075 +FF36>0076 +FF37>0077 +FF38>0078 +FF39>0079 +FF3A>007A +FF3B>005B +FF3C>005C +FF3D>005D +FF3E>005E +FF3F>005F +FF40>0060 +FF41>0061 +FF42>0062 +FF43>0063 +FF44>0064 +FF45>0065 +FF46>0066 +FF47>0067 +FF48>0068 +FF49>0069 +FF4A>006A +FF4B>006B +FF4C>006C +FF4D>006D +FF4E>006E +FF4F>006F +FF50>0070 +FF51>0071 +FF52>0072 +FF53>0073 +FF54>0074 +FF55>0075 +FF56>0076 +FF57>0077 +FF58>0078 +FF59>0079 +FF5A>007A +FF5B>007B +FF5C>007C +FF5D>007D +FF5E>007E +FF5F>2985 +FF60>2986 +FF61>3002 +FF62>300C +FF63>300D +FF64>3001 +FF65>30FB +FF66>30F2 +FF67>30A1 +FF68>30A3 +FF69>30A5 +FF6A>30A7 +FF6B>30A9 +FF6C>30E3 +FF6D>30E5 +FF6E>30E7 +FF6F>30C3 +FF70>30FC +FF71>30A2 +FF72>30A4 +FF73>30A6 +FF74>30A8 +FF75>30AA +FF76>30AB +FF77>30AD +FF78>30AF +FF79>30B1 +FF7A>30B3 +FF7B>30B5 +FF7C>30B7 +FF7D>30B9 +FF7E>30BB +FF7F>30BD +FF80>30BF +FF81>30C1 +FF82>30C4 +FF83>30C6 +FF84>30C8 +FF85>30CA +FF86>30CB +FF87>30CC +FF88>30CD +FF89>30CE +FF8A>30CF +FF8B>30D2 +FF8C>30D5 +FF8D>30D8 +FF8E>30DB +FF8F>30DE +FF90>30DF +FF91>30E0 +FF92>30E1 +FF93>30E2 +FF94>30E4 +FF95>30E6 +FF96>30E8 +FF97>30E9 +FF98>30EA +FF99>30EB +FF9A>30EC +FF9B>30ED +FF9C>30EF +FF9D>30F3 +FF9E>3099 +FF9F>309A +FFA0> +FFA1>1100 +FFA2>1101 +FFA3>11AA +FFA4>1102 +FFA5>11AC +FFA6>11AD +FFA7>1103 +FFA8>1104 +FFA9>1105 +FFAA>11B0 +FFAB>11B1 +FFAC>11B2 +FFAD>11B3 +FFAE>11B4 +FFAF>11B5 +FFB0>111A +FFB1>1106 +FFB2>1107 +FFB3>1108 +FFB4>1121 +FFB5>1109 +FFB6>110A +FFB7>110B +FFB8>110C +FFB9>110D +FFBA>110E +FFBB>110F +FFBC>1110 +FFBD>1111 +FFBE>1112 +FFC2>1161 +FFC3>1162 +FFC4>1163 +FFC5>1164 +FFC6>1165 +FFC7>1166 +FFCA>1167 +FFCB>1168 +FFCC>1169 +FFCD>116A +FFCE>116B +FFCF>116C +FFD2>116D +FFD3>116E +FFD4>116F +FFD5>1170 +FFD6>1171 +FFD7>1172 +FFDA>1173 +FFDB>1174 +FFDC>1175 +FFE0>00A2 +FFE1>00A3 +FFE2>00AC +FFE3>0020 0304 +FFE4>00A6 +FFE5>00A5 +FFE6>20A9 +FFE8>2502 +FFE9>2190 +FFEA>2191 +FFEB>2192 +FFEC>2193 +FFED>25A0 +FFEE>25CB +FFF0..FFF8> +10400>10428 +10401>10429 +10402>1042A +10403>1042B +10404>1042C +10405>1042D +10406>1042E +10407>1042F +10408>10430 +10409>10431 +1040A>10432 +1040B>10433 +1040C>10434 +1040D>10435 +1040E>10436 +1040F>10437 +10410>10438 +10411>10439 +10412>1043A +10413>1043B +10414>1043C +10415>1043D +10416>1043E +10417>1043F +10418>10440 +10419>10441 +1041A>10442 +1041B>10443 +1041C>10444 +1041D>10445 +1041E>10446 +1041F>10447 +10420>10448 +10421>10449 +10422>1044A +10423>1044B +10424>1044C +10425>1044D +10426>1044E +10427>1044F +1D15E>1D157 1D165 +1D15F>1D158 1D165 +1D160>1D158 1D165 1D16E +1D161>1D158 1D165 1D16F +1D162>1D158 1D165 1D170 +1D163>1D158 1D165 1D171 +1D164>1D158 1D165 1D172 +1D173..1D17A> +1D1BB>1D1B9 1D165 +1D1BC>1D1BA 1D165 +1D1BD>1D1B9 1D165 1D16E +1D1BE>1D1BA 1D165 1D16E +1D1BF>1D1B9 1D165 1D16F +1D1C0>1D1BA 1D165 1D16F +1D400>0061 +1D401>0062 +1D402>0063 +1D403>0064 +1D404>0065 +1D405>0066 +1D406>0067 +1D407>0068 +1D408>0069 +1D409>006A +1D40A>006B +1D40B>006C +1D40C>006D +1D40D>006E +1D40E>006F +1D40F>0070 +1D410>0071 +1D411>0072 +1D412>0073 +1D413>0074 +1D414>0075 +1D415>0076 +1D416>0077 +1D417>0078 +1D418>0079 +1D419>007A +1D41A>0061 +1D41B>0062 +1D41C>0063 +1D41D>0064 +1D41E>0065 +1D41F>0066 +1D420>0067 +1D421>0068 +1D422>0069 +1D423>006A +1D424>006B +1D425>006C +1D426>006D +1D427>006E +1D428>006F +1D429>0070 +1D42A>0071 +1D42B>0072 +1D42C>0073 +1D42D>0074 +1D42E>0075 +1D42F>0076 +1D430>0077 +1D431>0078 +1D432>0079 +1D433>007A +1D434>0061 +1D435>0062 +1D436>0063 +1D437>0064 +1D438>0065 +1D439>0066 +1D43A>0067 +1D43B>0068 +1D43C>0069 +1D43D>006A +1D43E>006B +1D43F>006C +1D440>006D +1D441>006E +1D442>006F +1D443>0070 +1D444>0071 +1D445>0072 +1D446>0073 +1D447>0074 +1D448>0075 +1D449>0076 +1D44A>0077 +1D44B>0078 +1D44C>0079 +1D44D>007A +1D44E>0061 +1D44F>0062 +1D450>0063 +1D451>0064 +1D452>0065 +1D453>0066 +1D454>0067 +1D456>0069 +1D457>006A +1D458>006B +1D459>006C +1D45A>006D +1D45B>006E +1D45C>006F +1D45D>0070 +1D45E>0071 +1D45F>0072 +1D460>0073 +1D461>0074 +1D462>0075 +1D463>0076 +1D464>0077 +1D465>0078 +1D466>0079 +1D467>007A +1D468>0061 +1D469>0062 +1D46A>0063 +1D46B>0064 +1D46C>0065 +1D46D>0066 +1D46E>0067 +1D46F>0068 +1D470>0069 +1D471>006A +1D472>006B +1D473>006C +1D474>006D +1D475>006E +1D476>006F +1D477>0070 +1D478>0071 +1D479>0072 +1D47A>0073 +1D47B>0074 +1D47C>0075 +1D47D>0076 +1D47E>0077 +1D47F>0078 +1D480>0079 +1D481>007A +1D482>0061 +1D483>0062 +1D484>0063 +1D485>0064 +1D486>0065 +1D487>0066 +1D488>0067 +1D489>0068 +1D48A>0069 +1D48B>006A +1D48C>006B +1D48D>006C +1D48E>006D +1D48F>006E +1D490>006F +1D491>0070 +1D492>0071 +1D493>0072 +1D494>0073 +1D495>0074 +1D496>0075 +1D497>0076 +1D498>0077 +1D499>0078 +1D49A>0079 +1D49B>007A +1D49C>0061 +1D49E>0063 +1D49F>0064 +1D4A2>0067 +1D4A5>006A +1D4A6>006B +1D4A9>006E +1D4AA>006F +1D4AB>0070 +1D4AC>0071 +1D4AE>0073 +1D4AF>0074 +1D4B0>0075 +1D4B1>0076 +1D4B2>0077 +1D4B3>0078 +1D4B4>0079 +1D4B5>007A +1D4B6>0061 +1D4B7>0062 +1D4B8>0063 +1D4B9>0064 +1D4BB>0066 +1D4BD>0068 +1D4BE>0069 +1D4BF>006A +1D4C0>006B +1D4C1>006C +1D4C2>006D +1D4C3>006E +1D4C5>0070 +1D4C6>0071 +1D4C7>0072 +1D4C8>0073 +1D4C9>0074 +1D4CA>0075 +1D4CB>0076 +1D4CC>0077 +1D4CD>0078 +1D4CE>0079 +1D4CF>007A +1D4D0>0061 +1D4D1>0062 +1D4D2>0063 +1D4D3>0064 +1D4D4>0065 +1D4D5>0066 +1D4D6>0067 +1D4D7>0068 +1D4D8>0069 +1D4D9>006A +1D4DA>006B +1D4DB>006C +1D4DC>006D +1D4DD>006E +1D4DE>006F +1D4DF>0070 +1D4E0>0071 +1D4E1>0072 +1D4E2>0073 +1D4E3>0074 +1D4E4>0075 +1D4E5>0076 +1D4E6>0077 +1D4E7>0078 +1D4E8>0079 +1D4E9>007A +1D4EA>0061 +1D4EB>0062 +1D4EC>0063 +1D4ED>0064 +1D4EE>0065 +1D4EF>0066 +1D4F0>0067 +1D4F1>0068 +1D4F2>0069 +1D4F3>006A +1D4F4>006B +1D4F5>006C +1D4F6>006D +1D4F7>006E +1D4F8>006F +1D4F9>0070 +1D4FA>0071 +1D4FB>0072 +1D4FC>0073 +1D4FD>0074 +1D4FE>0075 +1D4FF>0076 +1D500>0077 +1D501>0078 +1D502>0079 +1D503>007A +1D504>0061 +1D505>0062 +1D507>0064 +1D508>0065 +1D509>0066 +1D50A>0067 +1D50D>006A +1D50E>006B +1D50F>006C +1D510>006D +1D511>006E +1D512>006F +1D513>0070 +1D514>0071 +1D516>0073 +1D517>0074 +1D518>0075 +1D519>0076 +1D51A>0077 +1D51B>0078 +1D51C>0079 +1D51E>0061 +1D51F>0062 +1D520>0063 +1D521>0064 +1D522>0065 +1D523>0066 +1D524>0067 +1D525>0068 +1D526>0069 +1D527>006A +1D528>006B +1D529>006C +1D52A>006D +1D52B>006E +1D52C>006F +1D52D>0070 +1D52E>0071 +1D52F>0072 +1D530>0073 +1D531>0074 +1D532>0075 +1D533>0076 +1D534>0077 +1D535>0078 +1D536>0079 +1D537>007A +1D538>0061 +1D539>0062 +1D53B>0064 +1D53C>0065 +1D53D>0066 +1D53E>0067 +1D540>0069 +1D541>006A +1D542>006B +1D543>006C +1D544>006D +1D546>006F +1D54A>0073 +1D54B>0074 +1D54C>0075 +1D54D>0076 +1D54E>0077 +1D54F>0078 +1D550>0079 +1D552>0061 +1D553>0062 +1D554>0063 +1D555>0064 +1D556>0065 +1D557>0066 +1D558>0067 +1D559>0068 +1D55A>0069 +1D55B>006A +1D55C>006B +1D55D>006C +1D55E>006D +1D55F>006E +1D560>006F +1D561>0070 +1D562>0071 +1D563>0072 +1D564>0073 +1D565>0074 +1D566>0075 +1D567>0076 +1D568>0077 +1D569>0078 +1D56A>0079 +1D56B>007A +1D56C>0061 +1D56D>0062 +1D56E>0063 +1D56F>0064 +1D570>0065 +1D571>0066 +1D572>0067 +1D573>0068 +1D574>0069 +1D575>006A +1D576>006B +1D577>006C +1D578>006D +1D579>006E +1D57A>006F +1D57B>0070 +1D57C>0071 +1D57D>0072 +1D57E>0073 +1D57F>0074 +1D580>0075 +1D581>0076 +1D582>0077 +1D583>0078 +1D584>0079 +1D585>007A +1D586>0061 +1D587>0062 +1D588>0063 +1D589>0064 +1D58A>0065 +1D58B>0066 +1D58C>0067 +1D58D>0068 +1D58E>0069 +1D58F>006A +1D590>006B +1D591>006C +1D592>006D +1D593>006E +1D594>006F +1D595>0070 +1D596>0071 +1D597>0072 +1D598>0073 +1D599>0074 +1D59A>0075 +1D59B>0076 +1D59C>0077 +1D59D>0078 +1D59E>0079 +1D59F>007A +1D5A0>0061 +1D5A1>0062 +1D5A2>0063 +1D5A3>0064 +1D5A4>0065 +1D5A5>0066 +1D5A6>0067 +1D5A7>0068 +1D5A8>0069 +1D5A9>006A +1D5AA>006B +1D5AB>006C +1D5AC>006D +1D5AD>006E +1D5AE>006F +1D5AF>0070 +1D5B0>0071 +1D5B1>0072 +1D5B2>0073 +1D5B3>0074 +1D5B4>0075 +1D5B5>0076 +1D5B6>0077 +1D5B7>0078 +1D5B8>0079 +1D5B9>007A +1D5BA>0061 +1D5BB>0062 +1D5BC>0063 +1D5BD>0064 +1D5BE>0065 +1D5BF>0066 +1D5C0>0067 +1D5C1>0068 +1D5C2>0069 +1D5C3>006A +1D5C4>006B +1D5C5>006C +1D5C6>006D +1D5C7>006E +1D5C8>006F +1D5C9>0070 +1D5CA>0071 +1D5CB>0072 +1D5CC>0073 +1D5CD>0074 +1D5CE>0075 +1D5CF>0076 +1D5D0>0077 +1D5D1>0078 +1D5D2>0079 +1D5D3>007A +1D5D4>0061 +1D5D5>0062 +1D5D6>0063 +1D5D7>0064 +1D5D8>0065 +1D5D9>0066 +1D5DA>0067 +1D5DB>0068 +1D5DC>0069 +1D5DD>006A +1D5DE>006B +1D5DF>006C +1D5E0>006D +1D5E1>006E +1D5E2>006F +1D5E3>0070 +1D5E4>0071 +1D5E5>0072 +1D5E6>0073 +1D5E7>0074 +1D5E8>0075 +1D5E9>0076 +1D5EA>0077 +1D5EB>0078 +1D5EC>0079 +1D5ED>007A +1D5EE>0061 +1D5EF>0062 +1D5F0>0063 +1D5F1>0064 +1D5F2>0065 +1D5F3>0066 +1D5F4>0067 +1D5F5>0068 +1D5F6>0069 +1D5F7>006A +1D5F8>006B +1D5F9>006C +1D5FA>006D +1D5FB>006E +1D5FC>006F +1D5FD>0070 +1D5FE>0071 +1D5FF>0072 +1D600>0073 +1D601>0074 +1D602>0075 +1D603>0076 +1D604>0077 +1D605>0078 +1D606>0079 +1D607>007A +1D608>0061 +1D609>0062 +1D60A>0063 +1D60B>0064 +1D60C>0065 +1D60D>0066 +1D60E>0067 +1D60F>0068 +1D610>0069 +1D611>006A +1D612>006B +1D613>006C +1D614>006D +1D615>006E +1D616>006F +1D617>0070 +1D618>0071 +1D619>0072 +1D61A>0073 +1D61B>0074 +1D61C>0075 +1D61D>0076 +1D61E>0077 +1D61F>0078 +1D620>0079 +1D621>007A +1D622>0061 +1D623>0062 +1D624>0063 +1D625>0064 +1D626>0065 +1D627>0066 +1D628>0067 +1D629>0068 +1D62A>0069 +1D62B>006A +1D62C>006B +1D62D>006C +1D62E>006D +1D62F>006E +1D630>006F +1D631>0070 +1D632>0071 +1D633>0072 +1D634>0073 +1D635>0074 +1D636>0075 +1D637>0076 +1D638>0077 +1D639>0078 +1D63A>0079 +1D63B>007A +1D63C>0061 +1D63D>0062 +1D63E>0063 +1D63F>0064 +1D640>0065 +1D641>0066 +1D642>0067 +1D643>0068 +1D644>0069 +1D645>006A +1D646>006B +1D647>006C +1D648>006D +1D649>006E +1D64A>006F +1D64B>0070 +1D64C>0071 +1D64D>0072 +1D64E>0073 +1D64F>0074 +1D650>0075 +1D651>0076 +1D652>0077 +1D653>0078 +1D654>0079 +1D655>007A +1D656>0061 +1D657>0062 +1D658>0063 +1D659>0064 +1D65A>0065 +1D65B>0066 +1D65C>0067 +1D65D>0068 +1D65E>0069 +1D65F>006A +1D660>006B +1D661>006C +1D662>006D +1D663>006E +1D664>006F +1D665>0070 +1D666>0071 +1D667>0072 +1D668>0073 +1D669>0074 +1D66A>0075 +1D66B>0076 +1D66C>0077 +1D66D>0078 +1D66E>0079 +1D66F>007A +1D670>0061 +1D671>0062 +1D672>0063 +1D673>0064 +1D674>0065 +1D675>0066 +1D676>0067 +1D677>0068 +1D678>0069 +1D679>006A +1D67A>006B +1D67B>006C +1D67C>006D +1D67D>006E +1D67E>006F +1D67F>0070 +1D680>0071 +1D681>0072 +1D682>0073 +1D683>0074 +1D684>0075 +1D685>0076 +1D686>0077 +1D687>0078 +1D688>0079 +1D689>007A +1D68A>0061 +1D68B>0062 +1D68C>0063 +1D68D>0064 +1D68E>0065 +1D68F>0066 +1D690>0067 +1D691>0068 +1D692>0069 +1D693>006A +1D694>006B +1D695>006C +1D696>006D +1D697>006E +1D698>006F +1D699>0070 +1D69A>0071 +1D69B>0072 +1D69C>0073 +1D69D>0074 +1D69E>0075 +1D69F>0076 +1D6A0>0077 +1D6A1>0078 +1D6A2>0079 +1D6A3>007A +1D6A4>0131 +1D6A5>0237 +1D6A8>03B1 +1D6A9>03B2 +1D6AA>03B3 +1D6AB>03B4 +1D6AC>03B5 +1D6AD>03B6 +1D6AE>03B7 +1D6AF>03B8 +1D6B0>03B9 +1D6B1>03BA +1D6B2>03BB +1D6B3>03BC +1D6B4>03BD +1D6B5>03BE +1D6B6>03BF +1D6B7>03C0 +1D6B8>03C1 +1D6B9>03B8 +1D6BA>03C3 +1D6BB>03C4 +1D6BC>03C5 +1D6BD>03C6 +1D6BE>03C7 +1D6BF>03C8 +1D6C0>03C9 +1D6C1>2207 +1D6C2>03B1 +1D6C3>03B2 +1D6C4>03B3 +1D6C5>03B4 +1D6C6>03B5 +1D6C7>03B6 +1D6C8>03B7 +1D6C9>03B8 +1D6CA>03B9 +1D6CB>03BA +1D6CC>03BB +1D6CD>03BC +1D6CE>03BD +1D6CF>03BE +1D6D0>03BF +1D6D1>03C0 +1D6D2>03C1 +1D6D3..1D6D4>03C3 +1D6D5>03C4 +1D6D6>03C5 +1D6D7>03C6 +1D6D8>03C7 +1D6D9>03C8 +1D6DA>03C9 +1D6DB>2202 +1D6DC>03B5 +1D6DD>03B8 +1D6DE>03BA +1D6DF>03C6 +1D6E0>03C1 +1D6E1>03C0 +1D6E2>03B1 +1D6E3>03B2 +1D6E4>03B3 +1D6E5>03B4 +1D6E6>03B5 +1D6E7>03B6 +1D6E8>03B7 +1D6E9>03B8 +1D6EA>03B9 +1D6EB>03BA +1D6EC>03BB +1D6ED>03BC +1D6EE>03BD +1D6EF>03BE +1D6F0>03BF +1D6F1>03C0 +1D6F2>03C1 +1D6F3>03B8 +1D6F4>03C3 +1D6F5>03C4 +1D6F6>03C5 +1D6F7>03C6 +1D6F8>03C7 +1D6F9>03C8 +1D6FA>03C9 +1D6FB>2207 +1D6FC>03B1 +1D6FD>03B2 +1D6FE>03B3 +1D6FF>03B4 +1D700>03B5 +1D701>03B6 +1D702>03B7 +1D703>03B8 +1D704>03B9 +1D705>03BA +1D706>03BB +1D707>03BC +1D708>03BD +1D709>03BE +1D70A>03BF +1D70B>03C0 +1D70C>03C1 +1D70D..1D70E>03C3 +1D70F>03C4 +1D710>03C5 +1D711>03C6 +1D712>03C7 +1D713>03C8 +1D714>03C9 +1D715>2202 +1D716>03B5 +1D717>03B8 +1D718>03BA +1D719>03C6 +1D71A>03C1 +1D71B>03C0 +1D71C>03B1 +1D71D>03B2 +1D71E>03B3 +1D71F>03B4 +1D720>03B5 +1D721>03B6 +1D722>03B7 +1D723>03B8 +1D724>03B9 +1D725>03BA +1D726>03BB +1D727>03BC +1D728>03BD +1D729>03BE +1D72A>03BF +1D72B>03C0 +1D72C>03C1 +1D72D>03B8 +1D72E>03C3 +1D72F>03C4 +1D730>03C5 +1D731>03C6 +1D732>03C7 +1D733>03C8 +1D734>03C9 +1D735>2207 +1D736>03B1 +1D737>03B2 +1D738>03B3 +1D739>03B4 +1D73A>03B5 +1D73B>03B6 +1D73C>03B7 +1D73D>03B8 +1D73E>03B9 +1D73F>03BA +1D740>03BB +1D741>03BC +1D742>03BD +1D743>03BE +1D744>03BF +1D745>03C0 +1D746>03C1 +1D747..1D748>03C3 +1D749>03C4 +1D74A>03C5 +1D74B>03C6 +1D74C>03C7 +1D74D>03C8 +1D74E>03C9 +1D74F>2202 +1D750>03B5 +1D751>03B8 +1D752>03BA +1D753>03C6 +1D754>03C1 +1D755>03C0 +1D756>03B1 +1D757>03B2 +1D758>03B3 +1D759>03B4 +1D75A>03B5 +1D75B>03B6 +1D75C>03B7 +1D75D>03B8 +1D75E>03B9 +1D75F>03BA +1D760>03BB +1D761>03BC +1D762>03BD +1D763>03BE +1D764>03BF +1D765>03C0 +1D766>03C1 +1D767>03B8 +1D768>03C3 +1D769>03C4 +1D76A>03C5 +1D76B>03C6 +1D76C>03C7 +1D76D>03C8 +1D76E>03C9 +1D76F>2207 +1D770>03B1 +1D771>03B2 +1D772>03B3 +1D773>03B4 +1D774>03B5 +1D775>03B6 +1D776>03B7 +1D777>03B8 +1D778>03B9 +1D779>03BA +1D77A>03BB +1D77B>03BC +1D77C>03BD +1D77D>03BE +1D77E>03BF +1D77F>03C0 +1D780>03C1 +1D781..1D782>03C3 +1D783>03C4 +1D784>03C5 +1D785>03C6 +1D786>03C7 +1D787>03C8 +1D788>03C9 +1D789>2202 +1D78A>03B5 +1D78B>03B8 +1D78C>03BA +1D78D>03C6 +1D78E>03C1 +1D78F>03C0 +1D790>03B1 +1D791>03B2 +1D792>03B3 +1D793>03B4 +1D794>03B5 +1D795>03B6 +1D796>03B7 +1D797>03B8 +1D798>03B9 +1D799>03BA +1D79A>03BB +1D79B>03BC +1D79C>03BD +1D79D>03BE +1D79E>03BF +1D79F>03C0 +1D7A0>03C1 +1D7A1>03B8 +1D7A2>03C3 +1D7A3>03C4 +1D7A4>03C5 +1D7A5>03C6 +1D7A6>03C7 +1D7A7>03C8 +1D7A8>03C9 +1D7A9>2207 +1D7AA>03B1 +1D7AB>03B2 +1D7AC>03B3 +1D7AD>03B4 +1D7AE>03B5 +1D7AF>03B6 +1D7B0>03B7 +1D7B1>03B8 +1D7B2>03B9 +1D7B3>03BA +1D7B4>03BB +1D7B5>03BC +1D7B6>03BD +1D7B7>03BE +1D7B8>03BF +1D7B9>03C0 +1D7BA>03C1 +1D7BB..1D7BC>03C3 +1D7BD>03C4 +1D7BE>03C5 +1D7BF>03C6 +1D7C0>03C7 +1D7C1>03C8 +1D7C2>03C9 +1D7C3>2202 +1D7C4>03B5 +1D7C5>03B8 +1D7C6>03BA +1D7C7>03C6 +1D7C8>03C1 +1D7C9>03C0 +1D7CA..1D7CB>03DD +1D7CE>0030 +1D7CF>0031 +1D7D0>0032 +1D7D1>0033 +1D7D2>0034 +1D7D3>0035 +1D7D4>0036 +1D7D5>0037 +1D7D6>0038 +1D7D7>0039 +1D7D8>0030 +1D7D9>0031 +1D7DA>0032 +1D7DB>0033 +1D7DC>0034 +1D7DD>0035 +1D7DE>0036 +1D7DF>0037 +1D7E0>0038 +1D7E1>0039 +1D7E2>0030 +1D7E3>0031 +1D7E4>0032 +1D7E5>0033 +1D7E6>0034 +1D7E7>0035 +1D7E8>0036 +1D7E9>0037 +1D7EA>0038 +1D7EB>0039 +1D7EC>0030 +1D7ED>0031 +1D7EE>0032 +1D7EF>0033 +1D7F0>0034 +1D7F1>0035 +1D7F2>0036 +1D7F3>0037 +1D7F4>0038 +1D7F5>0039 +1D7F6>0030 +1D7F7>0031 +1D7F8>0032 +1D7F9>0033 +1D7FA>0034 +1D7FB>0035 +1D7FC>0036 +1D7FD>0037 +1D7FE>0038 +1D7FF>0039 +1F100>0030 002E +1F101>0030 002C +1F102>0031 002C +1F103>0032 002C +1F104>0033 002C +1F105>0034 002C +1F106>0035 002C +1F107>0036 002C +1F108>0037 002C +1F109>0038 002C +1F10A>0039 002C +1F110>0028 0061 0029 +1F111>0028 0062 0029 +1F112>0028 0063 0029 +1F113>0028 0064 0029 +1F114>0028 0065 0029 +1F115>0028 0066 0029 +1F116>0028 0067 0029 +1F117>0028 0068 0029 +1F118>0028 0069 0029 +1F119>0028 006A 0029 +1F11A>0028 006B 0029 +1F11B>0028 006C 0029 +1F11C>0028 006D 0029 +1F11D>0028 006E 0029 +1F11E>0028 006F 0029 +1F11F>0028 0070 0029 +1F120>0028 0071 0029 +1F121>0028 0072 0029 +1F122>0028 0073 0029 +1F123>0028 0074 0029 +1F124>0028 0075 0029 +1F125>0028 0076 0029 +1F126>0028 0077 0029 +1F127>0028 0078 0029 +1F128>0028 0079 0029 +1F129>0028 007A 0029 +1F12A>3014 0073 3015 +1F12B>0063 +1F12C>0072 +1F12D>0063 0064 +1F12E>0077 007A +1F131>0062 +1F13D>006E +1F13F>0070 +1F142>0073 +1F146>0077 +1F14A>0068 0076 +1F14B>006D 0076 +1F14C>0073 0064 +1F14D>0073 0073 +1F14E>0070 0070 0076 +1F190>0064 006A +1F200>307B 304B +1F210>624B +1F211>5B57 +1F212>53CC +1F213>30C7 +1F214>4E8C +1F215>591A +1F216>89E3 +1F217>5929 +1F218>4EA4 +1F219>6620 +1F21A>7121 +1F21B>6599 +1F21C>524D +1F21D>5F8C +1F21E>518D +1F21F>65B0 +1F220>521D +1F221>7D42 +1F222>751F +1F223>8CA9 +1F224>58F0 +1F225>5439 +1F226>6F14 +1F227>6295 +1F228>6355 +1F229>4E00 +1F22A>4E09 +1F22B>904A +1F22C>5DE6 +1F22D>4E2D +1F22E>53F3 +1F22F>6307 +1F230>8D70 +1F231>6253 +1F240>3014 672C 3015 +1F241>3014 4E09 3015 +1F242>3014 4E8C 3015 +1F243>3014 5B89 3015 +1F244>3014 70B9 3015 +1F245>3014 6253 3015 +1F246>3014 76D7 3015 +1F247>3014 52DD 3015 +1F248>3014 6557 3015 +2F800>4E3D +2F801>4E38 +2F802>4E41 +2F803>20122 +2F804>4F60 +2F805>4FAE +2F806>4FBB +2F807>5002 +2F808>507A +2F809>5099 +2F80A>50E7 +2F80B>50CF +2F80C>349E +2F80D>2063A +2F80E>514D +2F80F>5154 +2F810>5164 +2F811>5177 +2F812>2051C +2F813>34B9 +2F814>5167 +2F815>518D +2F816>2054B +2F817>5197 +2F818>51A4 +2F819>4ECC +2F81A>51AC +2F81B>51B5 +2F81C>291DF +2F81D>51F5 +2F81E>5203 +2F81F>34DF +2F820>523B +2F821>5246 +2F822>5272 +2F823>5277 +2F824>3515 +2F825>52C7 +2F826>52C9 +2F827>52E4 +2F828>52FA +2F829>5305 +2F82A>5306 +2F82B>5317 +2F82C>5349 +2F82D>5351 +2F82E>535A +2F82F>5373 +2F830>537D +2F831..2F833>537F +2F834>20A2C +2F835>7070 +2F836>53CA +2F837>53DF +2F838>20B63 +2F839>53EB +2F83A>53F1 +2F83B>5406 +2F83C>549E +2F83D>5438 +2F83E>5448 +2F83F>5468 +2F840>54A2 +2F841>54F6 +2F842>5510 +2F843>5553 +2F844>5563 +2F845..2F846>5584 +2F847>5599 +2F848>55AB +2F849>55B3 +2F84A>55C2 +2F84B>5716 +2F84C>5606 +2F84D>5717 +2F84E>5651 +2F84F>5674 +2F850>5207 +2F851>58EE +2F852>57CE +2F853>57F4 +2F854>580D +2F855>578B +2F856>5832 +2F857>5831 +2F858>58AC +2F859>214E4 +2F85A>58F2 +2F85B>58F7 +2F85C>5906 +2F85D>591A +2F85E>5922 +2F85F>5962 +2F860>216A8 +2F861>216EA +2F862>59EC +2F863>5A1B +2F864>5A27 +2F865>59D8 +2F866>5A66 +2F867>36EE +2F868>36FC +2F869>5B08 +2F86A..2F86B>5B3E +2F86C>219C8 +2F86D>5BC3 +2F86E>5BD8 +2F86F>5BE7 +2F870>5BF3 +2F871>21B18 +2F872>5BFF +2F873>5C06 +2F874>5F53 +2F875>5C22 +2F876>3781 +2F877>5C60 +2F878>5C6E +2F879>5CC0 +2F87A>5C8D +2F87B>21DE4 +2F87C>5D43 +2F87D>21DE6 +2F87E>5D6E +2F87F>5D6B +2F880>5D7C +2F881>5DE1 +2F882>5DE2 +2F883>382F +2F884>5DFD +2F885>5E28 +2F886>5E3D +2F887>5E69 +2F888>3862 +2F889>22183 +2F88A>387C +2F88B>5EB0 +2F88C>5EB3 +2F88D>5EB6 +2F88E>5ECA +2F88F>2A392 +2F890>5EFE +2F891..2F892>22331 +2F893>8201 +2F894..2F895>5F22 +2F896>38C7 +2F897>232B8 +2F898>261DA +2F899>5F62 +2F89A>5F6B +2F89B>38E3 +2F89C>5F9A +2F89D>5FCD +2F89E>5FD7 +2F89F>5FF9 +2F8A0>6081 +2F8A1>393A +2F8A2>391C +2F8A3>6094 +2F8A4>226D4 +2F8A5>60C7 +2F8A6>6148 +2F8A7>614C +2F8A8>614E +2F8A9>614C +2F8AA>617A +2F8AB>618E +2F8AC>61B2 +2F8AD>61A4 +2F8AE>61AF +2F8AF>61DE +2F8B0>61F2 +2F8B1>61F6 +2F8B2>6210 +2F8B3>621B +2F8B4>625D +2F8B5>62B1 +2F8B6>62D4 +2F8B7>6350 +2F8B8>22B0C +2F8B9>633D +2F8BA>62FC +2F8BB>6368 +2F8BC>6383 +2F8BD>63E4 +2F8BE>22BF1 +2F8BF>6422 +2F8C0>63C5 +2F8C1>63A9 +2F8C2>3A2E +2F8C3>6469 +2F8C4>647E +2F8C5>649D +2F8C6>6477 +2F8C7>3A6C +2F8C8>654F +2F8C9>656C +2F8CA>2300A +2F8CB>65E3 +2F8CC>66F8 +2F8CD>6649 +2F8CE>3B19 +2F8CF>6691 +2F8D0>3B08 +2F8D1>3AE4 +2F8D2>5192 +2F8D3>5195 +2F8D4>6700 +2F8D5>669C +2F8D6>80AD +2F8D7>43D9 +2F8D8>6717 +2F8D9>671B +2F8DA>6721 +2F8DB>675E +2F8DC>6753 +2F8DD>233C3 +2F8DE>3B49 +2F8DF>67FA +2F8E0>6785 +2F8E1>6852 +2F8E2>6885 +2F8E3>2346D +2F8E4>688E +2F8E5>681F +2F8E6>6914 +2F8E7>3B9D +2F8E8>6942 +2F8E9>69A3 +2F8EA>69EA +2F8EB>6AA8 +2F8EC>236A3 +2F8ED>6ADB +2F8EE>3C18 +2F8EF>6B21 +2F8F0>238A7 +2F8F1>6B54 +2F8F2>3C4E +2F8F3>6B72 +2F8F4>6B9F +2F8F5>6BBA +2F8F6>6BBB +2F8F7>23A8D +2F8F8>21D0B +2F8F9>23AFA +2F8FA>6C4E +2F8FB>23CBC +2F8FC>6CBF +2F8FD>6CCD +2F8FE>6C67 +2F8FF>6D16 +2F900>6D3E +2F901>6D77 +2F902>6D41 +2F903>6D69 +2F904>6D78 +2F905>6D85 +2F906>23D1E +2F907>6D34 +2F908>6E2F +2F909>6E6E +2F90A>3D33 +2F90B>6ECB +2F90C>6EC7 +2F90D>23ED1 +2F90E>6DF9 +2F90F>6F6E +2F910>23F5E +2F911>23F8E +2F912>6FC6 +2F913>7039 +2F914>701E +2F915>701B +2F916>3D96 +2F917>704A +2F918>707D +2F919>7077 +2F91A>70AD +2F91B>20525 +2F91C>7145 +2F91D>24263 +2F91E>719C +2F91F>243AB +2F920>7228 +2F921>7235 +2F922>7250 +2F923>24608 +2F924>7280 +2F925>7295 +2F926>24735 +2F927>24814 +2F928>737A +2F929>738B +2F92A>3EAC +2F92B>73A5 +2F92C..2F92D>3EB8 +2F92E>7447 +2F92F>745C +2F930>7471 +2F931>7485 +2F932>74CA +2F933>3F1B +2F934>7524 +2F935>24C36 +2F936>753E +2F937>24C92 +2F938>7570 +2F939>2219F +2F93A>7610 +2F93B>24FA1 +2F93C>24FB8 +2F93D>25044 +2F93E>3FFC +2F93F>4008 +2F940>76F4 +2F941>250F3 +2F942>250F2 +2F943>25119 +2F944>25133 +2F945>771E +2F946..2F947>771F +2F948>774A +2F949>4039 +2F94A>778B +2F94B>4046 +2F94C>4096 +2F94D>2541D +2F94E>784E +2F94F>788C +2F950>78CC +2F951>40E3 +2F952>25626 +2F953>7956 +2F954>2569A +2F955>256C5 +2F956>798F +2F957>79EB +2F958>412F +2F959>7A40 +2F95A>7A4A +2F95B>7A4F +2F95C>2597C +2F95D..2F95E>25AA7 +2F95F>7AEE +2F960>4202 +2F961>25BAB +2F962>7BC6 +2F963>7BC9 +2F964>4227 +2F965>25C80 +2F966>7CD2 +2F967>42A0 +2F968>7CE8 +2F969>7CE3 +2F96A>7D00 +2F96B>25F86 +2F96C>7D63 +2F96D>4301 +2F96E>7DC7 +2F96F>7E02 +2F970>7E45 +2F971>4334 +2F972>26228 +2F973>26247 +2F974>4359 +2F975>262D9 +2F976>7F7A +2F977>2633E +2F978>7F95 +2F979>7FFA +2F97A>8005 +2F97B>264DA +2F97C>26523 +2F97D>8060 +2F97E>265A8 +2F97F>8070 +2F980>2335F +2F981>43D5 +2F982>80B2 +2F983>8103 +2F984>440B +2F985>813E +2F986>5AB5 +2F987>267A7 +2F988>267B5 +2F989>23393 +2F98A>2339C +2F98B>8201 +2F98C>8204 +2F98D>8F9E +2F98E>446B +2F98F>8291 +2F990>828B +2F991>829D +2F992>52B3 +2F993>82B1 +2F994>82B3 +2F995>82BD +2F996>82E6 +2F997>26B3C +2F998>82E5 +2F999>831D +2F99A>8363 +2F99B>83AD +2F99C>8323 +2F99D>83BD +2F99E>83E7 +2F99F>8457 +2F9A0>8353 +2F9A1>83CA +2F9A2>83CC +2F9A3>83DC +2F9A4>26C36 +2F9A5>26D6B +2F9A6>26CD5 +2F9A7>452B +2F9A8>84F1 +2F9A9>84F3 +2F9AA>8516 +2F9AB>273CA +2F9AC>8564 +2F9AD>26F2C +2F9AE>455D +2F9AF>4561 +2F9B0>26FB1 +2F9B1>270D2 +2F9B2>456B +2F9B3>8650 +2F9B4>865C +2F9B5>8667 +2F9B6>8669 +2F9B7>86A9 +2F9B8>8688 +2F9B9>870E +2F9BA>86E2 +2F9BB>8779 +2F9BC>8728 +2F9BD>876B +2F9BE>8786 +2F9BF>45D7 +2F9C0>87E1 +2F9C1>8801 +2F9C2>45F9 +2F9C3>8860 +2F9C4>8863 +2F9C5>27667 +2F9C6>88D7 +2F9C7>88DE +2F9C8>4635 +2F9C9>88FA +2F9CA>34BB +2F9CB>278AE +2F9CC>27966 +2F9CD>46BE +2F9CE>46C7 +2F9CF>8AA0 +2F9D0>8AED +2F9D1>8B8A +2F9D2>8C55 +2F9D3>27CA8 +2F9D4>8CAB +2F9D5>8CC1 +2F9D6>8D1B +2F9D7>8D77 +2F9D8>27F2F +2F9D9>20804 +2F9DA>8DCB +2F9DB>8DBC +2F9DC>8DF0 +2F9DD>208DE +2F9DE>8ED4 +2F9DF>8F38 +2F9E0>285D2 +2F9E1>285ED +2F9E2>9094 +2F9E3>90F1 +2F9E4>9111 +2F9E5>2872E +2F9E6>911B +2F9E7>9238 +2F9E8>92D7 +2F9E9>92D8 +2F9EA>927C +2F9EB>93F9 +2F9EC>9415 +2F9ED>28BFA +2F9EE>958B +2F9EF>4995 +2F9F0>95B7 +2F9F1>28D77 +2F9F2>49E6 +2F9F3>96C3 +2F9F4>5DB2 +2F9F5>9723 +2F9F6>29145 +2F9F7>2921A +2F9F8>4A6E +2F9F9>4A76 +2F9FA>97E0 +2F9FB>2940A +2F9FC>4AB2 +2F9FD>29496 +2F9FE..2F9FF>980B +2FA00>9829 +2FA01>295B6 +2FA02>98E2 +2FA03>4B33 +2FA04>9929 +2FA05>99A7 +2FA06>99C2 +2FA07>99FE +2FA08>4BCE +2FA09>29B30 +2FA0A>9B12 +2FA0B>9C40 +2FA0C>9CFD +2FA0D>4CCE +2FA0E>4CED +2FA0F>9D67 +2FA10>2A0CE +2FA11>4CF8 +2FA12>2A105 +2FA13>2A20E +2FA14>2A291 +2FA15>9EBB +2FA16>4D56 +2FA17>9EF9 +2FA18>9EFE +2FA19>9F05 +2FA1A>9F0F +2FA1B>9F16 +2FA1C>9F3B +2FA1D>2A600 +E0000> +E0001> +E0002..E001F> +E0020..E007F> +E0080..E00FF> +E0100..E01EF> +E01F0..E0FFF> + +# Total code points: 9740 diff --git a/comm/mailnews/extensions/fts3/fts3_porter.c b/comm/mailnews/extensions/fts3/fts3_porter.c new file mode 100644 index 0000000000..e0e5d5ddaf --- /dev/null +++ b/comm/mailnews/extensions/fts3/fts3_porter.c @@ -0,0 +1,1140 @@ +/* +** 2006 September 30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Implementation of the full-text-search tokenizer that implements +** a Porter stemmer. +** +*/ + +/* + * This file is based on the SQLite FTS3 Porter Stemmer implementation. + * + * This is an attempt to provide some level of full-text search to users of + * Thunderbird who use languages that are not space/punctuation delimited. + * This is accomplished by performing bi-gram indexing of characters fall + * into the unicode space occupied by character sets used in such languages. + * + * Bi-gram indexing means that given the string "12345" we would index the + * pairs "12", "23", "34", and "45" (with position information). We do this + * because we are not sure where the word/semantic boundaries are in that + * string. Then, when a user searches for "234" the FTS3 engine tokenizes the + * search query into "23" and "34". Using special phrase-logic FTS3 requires + * the matches to have the tokens "23" and "34" adjacent to each other and in + * that order. In theory if the user searched for "2345" we we could just + * search for "23 NEAR/2 34". Unfortunately, NEAR does not imply ordering, + * so even though that would be more efficient, we would lose correctness + * and cannot do it. + * + * The efficiency and usability of bi-gram search assumes that the character + * space is large enough and actually observed bi-grams sufficiently + * distributed throughout the potential space so that the search bi-grams + * generated when the user issues a query find a 'reasonable' number of + * documents for each bi-gram match. + * + * Mozilla contributors: + * Makoto Kato <m_kato@ga2.so-net.ne.jp> + * Andrew Sutherland <asutherland@asutherland.org> + */ + +/* +** The code in this file is only compiled if: +** +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or +** +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +*/ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +# include <assert.h> +# include <stdlib.h> +# include <stdio.h> +# include <string.h> +# include <ctype.h> + +# include "fts3_tokenizer.h" + +/* need some defined to compile without sqlite3 code */ + +# define sqlite3_malloc malloc +# define sqlite3_free free +# define sqlite3_realloc realloc + +static const unsigned char sqlite3Utf8Trans1[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, + 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, + 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x00, + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, + 0x07, 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, +}; + +typedef unsigned char u8; + +/** + * SQLite helper macro from sqlite3.c (really utf.c) to encode a unicode + * character into utf8. + * + * @param zOut A pointer to the current write position that is updated by + * the routine. At entry it should point to one-past the last valid + * encoded byte. The same holds true at exit. + * @param c The character to encode; this should be an unsigned int. + */ +# define WRITE_UTF8(zOut, c) \ + { \ + if (c < 0x0080) { \ + *zOut++ = (u8)(c & 0xff); \ + } else if (c < 0x0800) { \ + *zOut++ = 0xC0 + (u8)((c >> 6) & 0x1F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + } else if (c < 0x10000) { \ + *zOut++ = 0xE0 + (u8)((c >> 12) & 0x0F); \ + *zOut++ = 0x80 + (u8)((c >> 6) & 0x3F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + } else { \ + *zOut++ = 0xf0 + (u8)((c >> 18) & 0x07); \ + *zOut++ = 0x80 + (u8)((c >> 12) & 0x3F); \ + *zOut++ = 0x80 + (u8)((c >> 6) & 0x3F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + } \ + } + +/** + * Fudge factor to avoid buffer overwrites when WRITE_UTF8 is involved. + * + * Our normalization table includes entries that may result in a larger + * utf-8 encoding. Namely, 023a maps to 2c65. This is a growth from 2 bytes + * as utf-8 encoded to 3 bytes. This is currently the only transition possible + * because 1-byte encodings are known to stay 1-byte and our normalization + * table is 16-bit and so can't generate a 4-byte encoded output. + * + * For simplicity, we just multiple by 2 which covers the current case and + * potential growth for 2-byte to 4-byte growth. We can afford to do this + * because we're not talking about a lot of memory here as a rule. + */ +# define MAX_UTF8_GROWTH_FACTOR 2 + +/** + * Helper from sqlite3.c to read a single UTF8 character. + * + * The clever bit with multi-byte reading is that you keep going until you find + * a byte whose top bits are not '10'. A single-byte UTF8 character will have + * '00' or '01', and a multi-byte UTF8 character must start with '11'. + * + * In the event of illegal UTF-8 this macro may read an arbitrary number of + * characters but will never read past zTerm. The resulting character value + * of illegal UTF-8 can be anything, although efforts are made to return the + * illegal character (0xfffd) for UTF-16 surrogates. + * + * @param zIn A pointer to the current position that is updated by the routine, + * pointing at the start of the next character when the routine returns. + * @param zTerm A pointer one past the end of the buffer. + * @param c The 'unsigned int' to hold the resulting character value. Do not + * use a short or a char. + */ +# define READ_UTF8(zIn, zTerm, c) \ + { \ + c = *(zIn++); \ + if (c >= 0xc0) { \ + c = sqlite3Utf8Trans1[c - 0xc0]; \ + while (zIn != zTerm && (*zIn & 0xc0) == 0x80) { \ + c = (c << 6) + (0x3f & *(zIn++)); \ + } \ + if (c < 0x80 || (c & 0xFFFFF800) == 0xD800 || \ + (c & 0xFFFFFFFE) == 0xFFFE) { \ + c = 0xFFFD; \ + } \ + } \ + } + +/* end of compatible block to complie codes */ + +/* +** Class derived from sqlite3_tokenizer +*/ +typedef struct porter_tokenizer { + sqlite3_tokenizer base; /* Base class */ +} porter_tokenizer; + +/* +** Class derived from sqlit3_tokenizer_cursor +*/ +typedef struct porter_tokenizer_cursor { + sqlite3_tokenizer_cursor base; + const char* zInput; /* input we are tokenizing */ + int nInput; /* size of the input */ + int iOffset; /* current position in zInput */ + int iToken; /* index of next token to be returned */ + unsigned char* zToken; /* storage for current token */ + int nAllocated; /* space allocated to zToken buffer */ + /** + * Store the offset of the second character in the bi-gram pair that we just + * emitted so that we can consider it being the first character in a bi-gram + * pair. + * The value 0 indicates that there is no previous such character. This is + * an acceptable sentinel value because the 0th offset can never be the + * offset of the second in a bi-gram pair. + * + * For example, let us say we are tokenizing a string of 4 CJK characters + * represented by the byte-string "11223344" where each repeated digit + * indicates 2-bytes of storage used to encode the character in UTF-8. + * (It actually takes 3, btw.) Then on the passes to emit each token, + * the iOffset and iPrevGigramOffset values at entry will be: + * + * 1122: iOffset = 0, iPrevBigramOffset = 0 + * 2233: iOffset = 4, iPrevBigramOffset = 2 + * 3344: iOffset = 6, iPrevBigramOffset = 4 + * (nothing will be emitted): iOffset = 8, iPrevBigramOffset = 6 + */ + int iPrevBigramOffset; /* previous result was bi-gram */ +} porter_tokenizer_cursor; + +/* Forward declaration */ +static const sqlite3_tokenizer_module porterTokenizerModule; + +/* from normalize.c */ +extern unsigned int normalize_character(const unsigned int c); + +/* +** Create a new tokenizer instance. +*/ +static int porterCreate(int argc, const char* const* argv, + sqlite3_tokenizer** ppTokenizer) { + porter_tokenizer* t; + t = (porter_tokenizer*)sqlite3_malloc(sizeof(*t)); + if (t == NULL) return SQLITE_NOMEM; + memset(t, 0, sizeof(*t)); + *ppTokenizer = &t->base; + return SQLITE_OK; +} + +/* +** Destroy a tokenizer +*/ +static int porterDestroy(sqlite3_tokenizer* pTokenizer) { + sqlite3_free(pTokenizer); + return SQLITE_OK; +} + +/* +** Prepare to begin tokenizing a particular string. The input +** string to be tokenized is zInput[0..nInput-1]. A cursor +** used to incrementally tokenize this string is returned in +** *ppCursor. +*/ +static int porterOpen( + sqlite3_tokenizer* pTokenizer, /* The tokenizer */ + const char* zInput, int nInput, /* String to be tokenized */ + sqlite3_tokenizer_cursor** ppCursor /* OUT: Tokenization cursor */ +) { + porter_tokenizer_cursor* c; + + c = (porter_tokenizer_cursor*)sqlite3_malloc(sizeof(*c)); + if (c == NULL) return SQLITE_NOMEM; + + c->zInput = zInput; + if (zInput == 0) { + c->nInput = 0; + } else if (nInput < 0) { + c->nInput = (int)strlen(zInput); + } else { + c->nInput = nInput; + } + c->iOffset = 0; /* start tokenizing at the beginning */ + c->iToken = 0; + c->zToken = NULL; /* no space allocated, yet. */ + c->nAllocated = 0; + c->iPrevBigramOffset = 0; + + *ppCursor = &c->base; + return SQLITE_OK; +} + +/* +** Close a tokenization cursor previously opened by a call to +** porterOpen() above. +*/ +static int porterClose(sqlite3_tokenizer_cursor* pCursor) { + porter_tokenizer_cursor* c = (porter_tokenizer_cursor*)pCursor; + sqlite3_free(c->zToken); + sqlite3_free(c); + return SQLITE_OK; +} +/* +** Vowel or consonant +*/ +static const char cType[] = {0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, + 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 2, 1}; + +/* +** isConsonant() and isVowel() determine if their first character in +** the string they point to is a consonant or a vowel, according +** to Porter ruls. +** +** A consonate is any letter other than 'a', 'e', 'i', 'o', or 'u'. +** 'Y' is a consonant unless it follows another consonant, +** in which case it is a vowel. +** +** In these routine, the letters are in reverse order. So the 'y' rule +** is that 'y' is a consonant unless it is followed by another +** consonant. +*/ +static int isVowel(const char*); +static int isConsonant(const char* z) { + int j; + char x = *z; + if (x == 0) return 0; + assert(x >= 'a' && x <= 'z'); + j = cType[x - 'a']; + if (j < 2) return j; + return z[1] == 0 || isVowel(z + 1); +} +static int isVowel(const char* z) { + int j; + char x = *z; + if (x == 0) return 0; + assert(x >= 'a' && x <= 'z'); + j = cType[x - 'a']; + if (j < 2) return 1 - j; + return isConsonant(z + 1); +} + +/* +** Let any sequence of one or more vowels be represented by V and let +** C be sequence of one or more consonants. Then every word can be +** represented as: +** +** [C] (VC){m} [V] +** +** In prose: A word is an optional consonant followed by zero or +** vowel-consonant pairs followed by an optional vowel. "m" is the +** number of vowel consonant pairs. This routine computes the value +** of m for the first i bytes of a word. +** +** Return true if the m-value for z is 1 or more. In other words, +** return true if z contains at least one vowel that is followed +** by a consonant. +** +** In this routine z[] is in reverse order. So we are really looking +** for an instance of of a consonant followed by a vowel. +*/ +static int m_gt_0(const char* z) { + while (isVowel(z)) { + z++; + } + if (*z == 0) return 0; + while (isConsonant(z)) { + z++; + } + return *z != 0; +} + +/* Like mgt0 above except we are looking for a value of m which is +** exactly 1 +*/ +static int m_eq_1(const char* z) { + while (isVowel(z)) { + z++; + } + if (*z == 0) return 0; + while (isConsonant(z)) { + z++; + } + if (*z == 0) return 0; + while (isVowel(z)) { + z++; + } + if (*z == 0) return 1; + while (isConsonant(z)) { + z++; + } + return *z == 0; +} + +/* Like mgt0 above except we are looking for a value of m>1 instead +** or m>0 +*/ +static int m_gt_1(const char* z) { + while (isVowel(z)) { + z++; + } + if (*z == 0) return 0; + while (isConsonant(z)) { + z++; + } + if (*z == 0) return 0; + while (isVowel(z)) { + z++; + } + if (*z == 0) return 0; + while (isConsonant(z)) { + z++; + } + return *z != 0; +} + +/* +** Return TRUE if there is a vowel anywhere within z[0..n-1] +*/ +static int hasVowel(const char* z) { + while (isConsonant(z)) { + z++; + } + return *z != 0; +} + +/* +** Return TRUE if the word ends in a double consonant. +** +** The text is reversed here. So we are really looking at +** the first two characters of z[]. +*/ +static int doubleConsonant(const char* z) { + return isConsonant(z) && z[0] == z[1] && isConsonant(z + 1); +} + +/* +** Return TRUE if the word ends with three letters which +** are consonant-vowel-consonent and where the final consonant +** is not 'w', 'x', or 'y'. +** +** The word is reversed here. So we are really checking the +** first three letters and the first one cannot be in [wxy]. +*/ +static int star_oh(const char* z) { + return z[0] != 0 && isConsonant(z) && z[0] != 'w' && z[0] != 'x' && + z[0] != 'y' && z[1] != 0 && isVowel(z + 1) && z[2] != 0 && + isConsonant(z + 2); +} + +/* +** If the word ends with zFrom and xCond() is true for the stem +** of the word that precedes the zFrom ending, then change the +** ending to zTo. +** +** The input word *pz and zFrom are both in reverse order. zTo +** is in normal order. +** +** Return TRUE if zFrom matches. Return FALSE if zFrom does not +** match. Not that TRUE is returned even if xCond() fails and +** no substitution occurs. +*/ +static int stem( + char** pz, /* The word being stemmed (Reversed) */ + const char* zFrom, /* If the ending matches this... (Reversed) */ + const char* zTo, /* ... change the ending to this (not reversed) */ + int (*xCond)(const char*) /* Condition that must be true */ +) { + char* z = *pz; + while (*zFrom && *zFrom == *z) { + z++; + zFrom++; + } + if (*zFrom != 0) return 0; + if (xCond && !xCond(z)) return 1; + while (*zTo) { + *(--z) = *(zTo++); + } + *pz = z; + return 1; +} + +/** + * Voiced sound mark is only on Japanese. It is like accent. It combines with + * previous character. Example, "サ" (Katakana) with "゛" (voiced sound mark) + * is "ザ". Although full-width character mapping has combined character like + * "ザ", there is no combined character on half-width Katanaka character + * mapping. + */ +static int isVoicedSoundMark(const unsigned int c) { + if (c == 0xff9e || c == 0xff9f || c == 0x3099 || c == 0x309a) return 1; + return 0; +} + +/** + * How many unicode characters to take from the front and back of a term in + * |copy_stemmer|. + */ +# define COPY_STEMMER_COPY_HALF_LEN 10 + +/** + * Normalizing but non-stemming term copying. + * + * The original function would take 10 bytes from the front and 10 bytes from + * the back if there were no digits in the string and it was more than 20 + * bytes long. If there were digits involved that would decrease to 3 bytes + * from the front and 3 from the back. This would potentially corrupt utf-8 + * encoded characters, which is fine from the perspective of the FTS3 logic. + * + * In our revised form we now operate on a unicode character basis rather than + * a byte basis. Additionally we use the same length limit even if there are + * digits involved because it's not clear digit token-space reduction is saving + * us from anything and could be hurting. Specifically, if no one is ever + * going to search on things with digits, then we should just remove them. + * Right now, the space reduction is going to increase false positives when + * people do search on them and increase the number of collisions sufficiently + * to make it really expensive. The caveat is there will be some increase in + * index size which could be meaningful if people are receiving lots of emails + * full of distinct numbers. + * + * In order to do the copy-from-the-front and copy-from-the-back trick, once + * we reach N characters in, we set zFrontEnd to the current value of zOut + * (which represents the termination of the first part of the result string) + * and set zBackStart to the value of zOutStart. We then advanced zBackStart + * along a character at a time as we write more characters. Once we have + * traversed the entire string, if zBackStart > zFrontEnd, then we know + * the string should be shrunk using the characters in the two ranges. + * + * (It would be faster to scan from the back with specialized logic but that + * particular logic seems easy to screw up and we don't have unit tests in here + * to the extent required.) + * + * @param zIn Input string to normalize and potentially shrink. + * @param nBytesIn The number of bytes in zIn, distinct from the number of + * unicode characters encoded in zIn. + * @param zOut The string to write our output into. This must have at least + * nBytesIn * MAX_UTF8_GROWTH_FACTOR in order to compensate for + * normalization that results in a larger utf-8 encoding. + * @param pnBytesOut Integer to write the number of bytes in zOut into. + */ +static void copy_stemmer(const unsigned char* zIn, const int nBytesIn, + unsigned char* zOut, int* pnBytesOut) { + const unsigned char* zInTerm = zIn + nBytesIn; + unsigned char* zOutStart = zOut; + unsigned int c; + unsigned int charCount = 0; + unsigned char *zFrontEnd = NULL, *zBackStart = NULL; + unsigned int trashC; + + /* copy normalized character */ + while (zIn < zInTerm) { + READ_UTF8(zIn, zInTerm, c); + c = normalize_character(c); + + /* ignore voiced/semi-voiced sound mark */ + if (!isVoicedSoundMark(c)) { + /* advance one non-voiced sound mark character. */ + if (zBackStart) READ_UTF8(zBackStart, zOut, trashC); + + WRITE_UTF8(zOut, c); + charCount++; + if (charCount == COPY_STEMMER_COPY_HALF_LEN) { + zFrontEnd = zOut; + zBackStart = zOutStart; + } + } + } + + /* if we need to shrink the string, transplant the back bytes */ + if (zBackStart > zFrontEnd) { /* this handles when both are null too */ + size_t backBytes = zOut - zBackStart; + memmove(zFrontEnd, zBackStart, backBytes); + zOut = zFrontEnd + backBytes; + } + *zOut = 0; + *pnBytesOut = zOut - zOutStart; +} + +/* +** Stem the input word zIn[0..nIn-1]. Store the output in zOut. +** zOut is at least big enough to hold nIn bytes. Write the actual +** size of the output word (exclusive of the '\0' terminator) into *pnOut. +** +** Any upper-case characters in the US-ASCII character set ([A-Z]) +** are converted to lower case. Upper-case UTF characters are +** unchanged. +** +** Words that are longer than about 20 bytes are stemmed by retaining +** a few bytes from the beginning and the end of the word. If the +** word contains digits, 3 bytes are taken from the beginning and +** 3 bytes from the end. For long words without digits, 10 bytes +** are taken from each end. US-ASCII case folding still applies. +** +** If the input word contains not digits but does characters not +** in [a-zA-Z] then no stemming is attempted and this routine just +** copies the input into the input into the output with US-ASCII +** case folding. +** +** Stemming never increases the length of the word. So there is +** no chance of overflowing the zOut buffer. +*/ +static void porter_stemmer(const unsigned char* zIn, unsigned int nIn, + unsigned char* zOut, int* pnOut) { + unsigned int i, j, c; + char zReverse[28]; + char *z, *z2; + const unsigned char* zTerm = zIn + nIn; + const unsigned char* zTmp = zIn; + + if (nIn < 3 || nIn >= sizeof(zReverse) - 7) { + /* The word is too big or too small for the porter stemmer. + ** Fallback to the copy stemmer */ + copy_stemmer(zIn, nIn, zOut, pnOut); + return; + } + for (j = sizeof(zReverse) - 6; zTmp < zTerm; j--) { + READ_UTF8(zTmp, zTerm, c); + c = normalize_character(c); + if (c >= 'a' && c <= 'z') { + zReverse[j] = c; + } else { + /* The use of a character not in [a-zA-Z] means that we fallback + ** to the copy stemmer */ + copy_stemmer(zIn, nIn, zOut, pnOut); + return; + } + } + memset(&zReverse[sizeof(zReverse) - 5], 0, 5); + z = &zReverse[j + 1]; + + /* Step 1a */ + if (z[0] == 's') { + if (!stem(&z, "sess", "ss", 0) && !stem(&z, "sei", "i", 0) && + !stem(&z, "ss", "ss", 0)) { + z++; + } + } + + /* Step 1b */ + z2 = z; + if (stem(&z, "dee", "ee", m_gt_0)) { + /* Do nothing. The work was all in the test */ + } else if ((stem(&z, "gni", "", hasVowel) || stem(&z, "de", "", hasVowel)) && + z != z2) { + if (stem(&z, "ta", "ate", 0) || stem(&z, "lb", "ble", 0) || + stem(&z, "zi", "ize", 0)) { + /* Do nothing. The work was all in the test */ + } else if (doubleConsonant(z) && (*z != 'l' && *z != 's' && *z != 'z')) { + z++; + } else if (m_eq_1(z) && star_oh(z)) { + *(--z) = 'e'; + } + } + + /* Step 1c */ + if (z[0] == 'y' && hasVowel(z + 1)) { + z[0] = 'i'; + } + + /* Step 2 */ + switch (z[1]) { + case 'a': + (void)(stem(&z, "lanoita", "ate", m_gt_0) || + stem(&z, "lanoit", "tion", m_gt_0)); + break; + case 'c': + (void)(stem(&z, "icne", "ence", m_gt_0) || + stem(&z, "icna", "ance", m_gt_0)); + break; + case 'e': + (void)(stem(&z, "rezi", "ize", m_gt_0)); + break; + case 'g': + (void)(stem(&z, "igol", "log", m_gt_0)); + break; + case 'l': + (void)(stem(&z, "ilb", "ble", m_gt_0) || stem(&z, "illa", "al", m_gt_0) || + stem(&z, "iltne", "ent", m_gt_0) || stem(&z, "ile", "e", m_gt_0) || + stem(&z, "ilsuo", "ous", m_gt_0)); + break; + case 'o': + (void)(stem(&z, "noitazi", "ize", m_gt_0) || + stem(&z, "noita", "ate", m_gt_0) || + stem(&z, "rota", "ate", m_gt_0)); + break; + case 's': + (void)(stem(&z, "msila", "al", m_gt_0) || + stem(&z, "ssenevi", "ive", m_gt_0) || + stem(&z, "ssenluf", "ful", m_gt_0) || + stem(&z, "ssensuo", "ous", m_gt_0)); + break; + case 't': + (void)(stem(&z, "itila", "al", m_gt_0) || + stem(&z, "itivi", "ive", m_gt_0) || + stem(&z, "itilib", "ble", m_gt_0)); + break; + } + + /* Step 3 */ + switch (z[0]) { + case 'e': + (void)(stem(&z, "etaci", "ic", m_gt_0) || stem(&z, "evita", "", m_gt_0) || + stem(&z, "ezila", "al", m_gt_0)); + break; + case 'i': + (void)(stem(&z, "itici", "ic", m_gt_0)); + break; + case 'l': + (void)(stem(&z, "laci", "ic", m_gt_0) || stem(&z, "luf", "", m_gt_0)); + break; + case 's': + (void)(stem(&z, "ssen", "", m_gt_0)); + break; + } + + /* Step 4 */ + switch (z[1]) { + case 'a': + if (z[0] == 'l' && m_gt_1(z + 2)) { + z += 2; + } + break; + case 'c': + if (z[0] == 'e' && z[2] == 'n' && (z[3] == 'a' || z[3] == 'e') && + m_gt_1(z + 4)) { + z += 4; + } + break; + case 'e': + if (z[0] == 'r' && m_gt_1(z + 2)) { + z += 2; + } + break; + case 'i': + if (z[0] == 'c' && m_gt_1(z + 2)) { + z += 2; + } + break; + case 'l': + if (z[0] == 'e' && z[2] == 'b' && (z[3] == 'a' || z[3] == 'i') && + m_gt_1(z + 4)) { + z += 4; + } + break; + case 'n': + if (z[0] == 't') { + if (z[2] == 'a') { + if (m_gt_1(z + 3)) { + z += 3; + } + } else if (z[2] == 'e') { + (void)(stem(&z, "tneme", "", m_gt_1) || + stem(&z, "tnem", "", m_gt_1) || stem(&z, "tne", "", m_gt_1)); + } + } + break; + case 'o': + if (z[0] == 'u') { + if (m_gt_1(z + 2)) { + z += 2; + } + } else if (z[3] == 's' || z[3] == 't') { + (void)(stem(&z, "noi", "", m_gt_1)); + } + break; + case 's': + if (z[0] == 'm' && z[2] == 'i' && m_gt_1(z + 3)) { + z += 3; + } + break; + case 't': + (void)(stem(&z, "eta", "", m_gt_1) || stem(&z, "iti", "", m_gt_1)); + break; + case 'u': + if (z[0] == 's' && z[2] == 'o' && m_gt_1(z + 3)) { + z += 3; + } + break; + case 'v': + case 'z': + if (z[0] == 'e' && z[2] == 'i' && m_gt_1(z + 3)) { + z += 3; + } + break; + } + + /* Step 5a */ + if (z[0] == 'e') { + if (m_gt_1(z + 1)) { + z++; + } else if (m_eq_1(z + 1) && !star_oh(z + 1)) { + z++; + } + } + + /* Step 5b */ + if (m_gt_1(z) && z[0] == 'l' && z[1] == 'l') { + z++; + } + + /* z[] is now the stemmed word in reverse order. Flip it back + ** around into forward order and return. + */ + *pnOut = i = strlen(z); + zOut[i] = 0; + while (*z) { + zOut[--i] = *(z++); + } +} + +/** + * Indicate whether characters in the 0x30 - 0x7f region can be part of a token. + * Letters and numbers can; punctuation (and 'del') can't. + */ +static const char porterIdChar[] = { + /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ +}; + +/** + * Test whether a character is a (non-ascii) space character or not. isDelim + * uses the existing porter stemmer logic for anything in the ASCII (< 0x80) + * space which covers 0x20. + * + * 0x2000-0x206F is the general punctuation table. 0x2000 - 0x200b are spaces. + * The spaces 0x2000 - 0x200a are all defined as roughly equivalent to a + * standard 0x20 space. 0x200b is a "zero width space" (ZWSP) and not like an + * 0x20 space. 0x202f is a narrow no-break space and roughly equivalent to an + * 0x20 space. 0x205f is a "medium mathematical space" and defined as roughly + * equivalent to an 0x20 space. + */ +# define IS_UNI_SPACE(x) \ + (((x) >= 0x2000 && (x) <= 0x200a) || (x) == 0x202f || (x) == 0x205f) +/** + * What we are checking for: + * - 0x3001: Ideographic comma (-> 0x2c ',') + * - 0x3002: Ideographic full stop (-> 0x2e '.') + * - 0xff0c: fullwidth comma (~ wide 0x2c ',') + * - 0xff0e: fullwidth full stop (~ wide 0x2e '.') + * - 0xff61: halfwidth ideographic full stop (~ narrow 0x3002) + * - 0xff64: halfwidth ideographic comma (~ narrow 0x3001) + * + * It is possible we should be treating other things as delimiters! + */ +# define IS_JA_DELIM(x) \ + (((x) == 0x3001) || ((x) == 0xFF64) || ((x) == 0xFF0E) || \ + ((x) == 0x3002) || ((x) == 0xFF61) || ((x) == 0xFF0C)) + +/** + * The previous character was a delimiter (which includes the start of the + * string). + */ +# define BIGRAM_RESET 0 +/** + * The previous character was a CJK character and we have only seen one of them. + * If we had seen more than one in a row it would be the BIGRAM_USE state. + */ +# define BIGRAM_UNKNOWN 1 +/** + * We have seen two or more CJK characters in a row. + */ +# define BIGRAM_USE 2 +/** + * The previous character was ASCII or something in the unicode general scripts + * area that we do not believe is a delimiter. We call it 'alpha' as in + * alphabetic/alphanumeric and something that should be tokenized based on + * delimiters rather than on a bi-gram basis. + */ +# define BIGRAM_ALPHA 3 + +static int isDelim( + const unsigned char* zCur, /* IN: current pointer of token */ + const unsigned char* zTerm, /* IN: one character beyond end of token */ + int* len, /* OUT: analyzed bytes in this token */ + int* state /* IN/OUT: analyze state */ +) { + const unsigned char* zIn = zCur; + unsigned int c; + int delim; + + /* get the unicode character to analyze */ + READ_UTF8(zIn, zTerm, c); + c = normalize_character(c); + *len = zIn - zCur; + + /* ASCII character range has rule */ + if (c < 0x80) { + // This is original porter stemmer isDelim logic. + // 0x0 - 0x1f are all control characters, 0x20 is space, 0x21-0x2f are + // punctuation. + delim = (c < 0x30 || !porterIdChar[c - 0x30]); + // cases: "&a", "&." + if (*state == BIGRAM_USE || *state == BIGRAM_UNKNOWN) { + /* previous maybe CJK and current is ascii */ + *state = BIGRAM_ALPHA; /*ascii*/ + delim = 1; /* must break */ + } else if (delim == 1) { + // cases: "a.", ".." + /* this is delimiter character */ + *state = BIGRAM_RESET; /*reset*/ + } else { + // cases: "aa", ".a" + *state = BIGRAM_ALPHA; /*ascii*/ + } + return delim; + } + + // (at this point we must be a non-ASCII character) + + /* voiced/semi-voiced sound mark is ignore */ + if (isVoicedSoundMark(c) && *state != BIGRAM_ALPHA) { + /* ignore this because it is combined with previous char */ + return 0; + } + + /* this isn't CJK range, so return as no delim */ + // Anything less than 0x2000 (except to U+0E00-U+0EFF and U+1780-U+17FF) + // is the general scripts area and should not be bi-gram indexed. + // 0xa000 - 0a4cf is the Yi area. It is apparently a phonetic language whose + // usage does not appear to have simple delimiter rules, so we're leaving it + // as bigram processed. This is a guess, if you know better, let us know. + // (We previously bailed on this range too.) + // Addition, U+0E00-U+0E7F is Thai, U+0E80-U+0EFF is Laos, + // and U+1780-U+17FF is Khmer. It is no easy way to break each word. + // So these should use bi-gram too. + // cases: "aa", ".a", "&a" + if (c < 0xe00 || (c >= 0xf00 && c < 0x1780) || (c >= 0x1800 && c < 0x2000)) { + *state = BIGRAM_ALPHA; /* not really ASCII but same idea; tokenize it */ + return 0; + } + + // (at this point we must be a bi-grammable char or delimiter) + + /* this is space character or delim character */ + // cases: "a.", "..", "&." + if (IS_UNI_SPACE(c) || IS_JA_DELIM(c)) { + *state = BIGRAM_RESET; /* reset */ + return 1; /* it actually is a delimiter; report as such */ + } + + // (at this point we must be a bi-grammable char) + + // cases: "a&" + if (*state == BIGRAM_ALPHA) { + /* Previous is ascii and current maybe CJK */ + *state = BIGRAM_UNKNOWN; /* mark as unknown */ + return 1; /* break to emit the ASCII token*/ + } + + /* We have no rule for CJK!. use bi-gram */ + // cases: "&&" + if (*state == BIGRAM_UNKNOWN || *state == BIGRAM_USE) { + /* previous state is unknown. mark as bi-gram */ + *state = BIGRAM_USE; + return 1; /* break to emit the digram */ + } + + // cases: ".&" (*state == BIGRAM_RESET) + *state = BIGRAM_UNKNOWN; /* mark as unknown */ + return 0; /* no need to break; nothing to emit */ +} + +/** + * Generate a new token. There are basically three types of token we can + * generate: + * - A porter stemmed token. This is a word entirely comprised of ASCII + * characters. We run the porter stemmer algorithm against the word. + * Because we have no way to know what is and is not an English word + * (the only language for which the porter stemmer was designed), this + * could theoretically map multiple words that are not variations of the + * same word down to the same root, resulting in potentially unexpected + * result inclusions in the search results. We accept this result because + * there's not a lot we can do about it and false positives are much + * better than false negatives. + * - A copied token; case/accent-folded but not stemmed. We call the porter + * stemmer for all non-CJK cases and it diverts to the copy stemmer if it + * sees any non-ASCII characters (after folding) or if the string is too + * long. The copy stemmer will shrink the string if it is deemed too long. + * - A bi-gram token; two CJK-ish characters. For query reasons we generate a + * series of overlapping bi-grams. (We can't require the user to start their + * search based on the arbitrary context of the indexed documents.) + * + * It may be useful to think of this function as operating at the points between + * characters. While we are considering the 'current' character (the one after + * the 'point'), we are also interested in the 'previous' character (the one + * preceding the point). + * At any 'point', there are a number of possible situations which I will + * illustrate with pairs of characters. 'a' means alphanumeric ASCII or a + * non-ASCII character that is not bi-grammable or a delimiter, '.' + * means a delimiter (space or punctuation), '&' means a bi-grammable + * character. + * - aa: We are in the midst of a token. State remains BIGRAM_ALPHA. + * - a.: We will generate a porter stemmed or copied token. State was + * BIGRAM_ALPHA, gets set to BIGRAM_RESET. + * - a&: We will generate a porter stemmed or copied token; we will set our + * state to BIGRAM_UNKNOWN to indicate we have seen one bigram character + * but that it is not yet time to emit a bigram. + * - .a: We are starting a token. State was BIGRAM_RESET, gets set to + * BIGRAM_ALPHA. + * - ..: We skip/eat the delimiters. State stays BIGRAM_RESET. + * - .&: State set to BIGRAM_UNKNOWN to indicate we have seen one bigram char. + * - &a: If the state was BIGRAM_USE, we generate a bi-gram token. If the state + * was BIGRAM_UNKNOWN we had only seen one CJK character and so don't do + * anything. State is set to BIGRAM_ALPHA. + * - &.: Same as the "&a" case, but state is set to BIGRAM_RESET. + * - &&: We will generate a bi-gram token. State was either BIGRAM_UNKNOWN or + * BIGRAM_USE, gets set to BIGRAM_USE. + */ +static int porterNext( + sqlite3_tokenizer_cursor* pCursor, /* Cursor returned by porterOpen */ + const char** pzToken, /* OUT: *pzToken is the token text */ + int* pnBytes, /* OUT: Number of bytes in token */ + int* piStartOffset, /* OUT: Starting offset of token */ + int* piEndOffset, /* OUT: Ending offset of token */ + int* piPosition /* OUT: Position integer of token */ +) { + porter_tokenizer_cursor* c = (porter_tokenizer_cursor*)pCursor; + const unsigned char* z = (unsigned char*)c->zInput; + int len = 0; + int state; + + while (c->iOffset < c->nInput) { + int iStartOffset, numChars; + + /* + * This loop basically has two modes of operation: + * - general processing (iPrevBigramOffset == 0 here) + * - CJK processing (iPrevBigramOffset != 0 here) + * + * In an general processing pass we skip over all the delimiters, leaving us + * at a character that promises to produce a token. This could be a CJK + * token (state == BIGRAM_USE) or an ALPHA token (state == BIGRAM_ALPHA). + * If it was a CJK token, we transition into CJK state for the next loop. + * If it was an alpha token, our current offset is pointing at a delimiter + * (which could be a CJK character), so it is good that our next pass + * through the function and loop will skip over any delimiters. If the + * delimiter we hit was a CJK character, the next time through we will + * not treat it as a delimiter though; the entry state for that scan is + * BIGRAM_RESET so the transition is not treated as a delimiter! + * + * The CJK pass always starts with the second character in a bi-gram emitted + * as a token in the previous step. No delimiter skipping is required + * because we know that first character might produce a token for us. It + * only 'might' produce a token because the previous pass performed no + * lookahead and cannot be sure it is followed by another CJK character. + * This is why + */ + + // If we have a previous bigram offset + if (c->iPrevBigramOffset == 0) { + /* Scan past delimiter characters */ + state = BIGRAM_RESET; /* reset */ + while (c->iOffset < c->nInput && + isDelim(z + c->iOffset, z + c->nInput, &len, &state)) { + c->iOffset += len; + } + + } else { + /* for bigram indexing, use previous offset */ + c->iOffset = c->iPrevBigramOffset; + } + + /* Count non-delimiter characters. */ + iStartOffset = c->iOffset; + numChars = 0; + + // Start from a reset state. This means the first character we see + // (which will not be a delimiter) determines which of ALPHA or CJK modes + // we are operating in. (It won't be a delimiter because in a 'general' + // pass as defined above, we will have eaten all the delimiters, and in + // a CJK pass we are guaranteed that the first character is CJK.) + state = BIGRAM_RESET; /* state is reset */ + // Advance until it is time to emit a token. + // For ALPHA characters, this means advancing until we encounter a delimiter + // or a CJK character. iOffset will be pointing at the delimiter or CJK + // character, aka one beyond the last ALPHA character. + // For CJK characters this means advancing until we encounter an ALPHA + // character, a delimiter, or we have seen two consecutive CJK + // characters. iOffset points at the ALPHA/delimiter in the first 2 cases + // and the second of two CJK characters in the last case. + // Because of the way this loop is structured, iOffset is only updated + // when we don't terminate. However, if we terminate, len still contains + // the number of bytes in the character found at iOffset. (This is useful + // in the CJK case.) + while (c->iOffset < c->nInput && + !isDelim(z + c->iOffset, z + c->nInput, &len, &state)) { + c->iOffset += len; + numChars++; + } + + if (state == BIGRAM_USE) { + /* Split word by bigram */ + // Right now iOffset is pointing at the second character in a pair. + // Save this offset so next-time through we start with that as the + // first character. + c->iPrevBigramOffset = c->iOffset; + // And now advance so that iOffset is pointing at the character after + // the second character in the bi-gram pair. Also count the char. + c->iOffset += len; + numChars++; + } else { + /* Reset bigram offset */ + c->iPrevBigramOffset = 0; + } + + /* We emit a token if: + * - there are two ideograms together, + * - there are three chars or more, + * - we think this is a query and wildcard magic is desired. + * We think is a wildcard query when we have a single character, it starts + * at the start of the buffer, it's CJK, our current offset is one shy of + * nInput and the character at iOffset is '*'. Because the state gets + * clobbered by the incidence of '*' our requirement for CJK is that the + * implied character length is at least 3 given that it takes at least 3 + * bytes to encode to 0x2000. + */ + // It is possible we have no token to emit here if iPrevBigramOffset was not + // 0 on entry and there was no second CJK character. iPrevBigramOffset + // will now be 0 if that is the case (and c->iOffset == iStartOffset). + if ( // allow two-character words only if in bigram + (numChars == 2 && state == BIGRAM_USE) || + // otherwise, drop two-letter words (considered stop-words) + (numChars >= 3) || + // wildcard case: + (numChars == 1 && iStartOffset == 0 && (c->iOffset >= 3) && + (c->iOffset == c->nInput - 1) && (z[c->iOffset] == '*'))) { + /* figure out the number of bytes to copy/stem */ + int n = c->iOffset - iStartOffset; + /* make sure there is enough buffer space */ + if (n * MAX_UTF8_GROWTH_FACTOR > c->nAllocated) { + c->nAllocated = n * MAX_UTF8_GROWTH_FACTOR + 20; + c->zToken = sqlite3_realloc(c->zToken, c->nAllocated); + if (c->zToken == NULL) return SQLITE_NOMEM; + } + + if (state == BIGRAM_USE) { + /* This is by bigram. So it is unnecessary to convert word */ + copy_stemmer(&z[iStartOffset], n, c->zToken, pnBytes); + } else { + porter_stemmer(&z[iStartOffset], n, c->zToken, pnBytes); + } + *pzToken = (const char*)c->zToken; + *piStartOffset = iStartOffset; + *piEndOffset = c->iOffset; + *piPosition = c->iToken++; + return SQLITE_OK; + } + } + return SQLITE_DONE; +} + +/* +** The set of routines that implement the porter-stemmer tokenizer +*/ +static const sqlite3_tokenizer_module porterTokenizerModule = { + 0, porterCreate, porterDestroy, porterOpen, porterClose, porterNext, +}; + +/* +** Allocate a new porter tokenizer. Return a pointer to the new +** tokenizer in *ppModule +*/ +void sqlite3Fts3PorterTokenizerModule( + sqlite3_tokenizer_module const** ppModule) { + *ppModule = &porterTokenizerModule; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ diff --git a/comm/mailnews/extensions/fts3/fts3_tokenizer.h b/comm/mailnews/extensions/fts3/fts3_tokenizer.h new file mode 100644 index 0000000000..41973483a3 --- /dev/null +++ b/comm/mailnews/extensions/fts3/fts3_tokenizer.h @@ -0,0 +1,146 @@ +/* +** 2006 July 10 +** +** The author disclaims copyright to this source code. +** +************************************************************************* +** Defines the interface to tokenizers used by fulltext-search. There +** are three basic components: +** +** sqlite3_tokenizer_module is a singleton defining the tokenizer +** interface functions. This is essentially the class structure for +** tokenizers. +** +** sqlite3_tokenizer is used to define a particular tokenizer, perhaps +** including customization information defined at creation time. +** +** sqlite3_tokenizer_cursor is generated by a tokenizer to generate +** tokens from a particular input. +*/ +#ifndef _FTS3_TOKENIZER_H_ +#define _FTS3_TOKENIZER_H_ + +/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time. +** If tokenizers are to be allowed to call sqlite3_*() functions, then +** we will need a way to register the API consistently. +*/ +#include "sqlite3.h" + +/* +** Structures used by the tokenizer interface. When a new tokenizer +** implementation is registered, the caller provides a pointer to +** an sqlite3_tokenizer_module containing pointers to the callback +** functions that make up an implementation. +** +** When an fts3 table is created, it passes any arguments passed to +** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the +** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer +** implementation. The xCreate() function in turn returns an +** sqlite3_tokenizer structure representing the specific tokenizer to +** be used for the fts3 table (customized by the tokenizer clause arguments). +** +** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen() +** method is called. It returns an sqlite3_tokenizer_cursor object +** that may be used to tokenize a specific input buffer based on +** the tokenization rules supplied by a specific sqlite3_tokenizer +** object. +*/ +typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module; +typedef struct sqlite3_tokenizer sqlite3_tokenizer; +typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor; + +struct sqlite3_tokenizer_module { + /* + ** Structure version. Should always be set to 0. + */ + int iVersion; + + /* + ** Create a new tokenizer. The values in the argv[] array are the + ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL + ** TABLE statement that created the fts3 table. For example, if + ** the following SQL is executed: + ** + ** CREATE .. USING fts3( ... , tokenizer <tokenizer-name> arg1 arg2) + ** + ** then argc is set to 2, and the argv[] array contains pointers + ** to the strings "arg1" and "arg2". + ** + ** This method should return either SQLITE_OK (0), or an SQLite error + ** code. If SQLITE_OK is returned, then *ppTokenizer should be set + ** to point at the newly created tokenizer structure. The generic + ** sqlite3_tokenizer.pModule variable should not be initialised by + ** this callback. The caller will do so. + */ + int (*xCreate)(int argc, /* Size of argv array */ + const char* const* argv, /* Tokenizer argument strings */ + sqlite3_tokenizer** ppTokenizer /* OUT: Created tokenizer */ + ); + + /* + ** Destroy an existing tokenizer. The fts3 module calls this method + ** exactly once for each successful call to xCreate(). + */ + int (*xDestroy)(sqlite3_tokenizer* pTokenizer); + + /* + ** Create a tokenizer cursor to tokenize an input buffer. The caller + ** is responsible for ensuring that the input buffer remains valid + ** until the cursor is closed (using the xClose() method). + */ + int (*xOpen)( + sqlite3_tokenizer* pTokenizer, /* Tokenizer object */ + const char* pInput, int nBytes, /* Input buffer */ + sqlite3_tokenizer_cursor** ppCursor /* OUT: Created tokenizer cursor */ + ); + + /* + ** Destroy an existing tokenizer cursor. The fts3 module calls this + ** method exactly once for each successful call to xOpen(). + */ + int (*xClose)(sqlite3_tokenizer_cursor* pCursor); + + /* + ** Retrieve the next token from the tokenizer cursor pCursor. This + ** method should either return SQLITE_OK and set the values of the + ** "OUT" variables identified below, or SQLITE_DONE to indicate that + ** the end of the buffer has been reached, or an SQLite error code. + ** + ** *ppToken should be set to point at a buffer containing the + ** normalized version of the token (i.e. after any case-folding and/or + ** stemming has been performed). *pnBytes should be set to the length + ** of this buffer in bytes. The input text that generated the token is + ** identified by the byte offsets returned in *piStartOffset and + ** *piEndOffset. *piStartOffset should be set to the index of the first + ** byte of the token in the input buffer. *piEndOffset should be set + ** to the index of the first byte just past the end of the token in + ** the input buffer. + ** + ** The buffer *ppToken is set to point at is managed by the tokenizer + ** implementation. It is only required to be valid until the next call + ** to xNext() or xClose(). + */ + /* TODO(shess) current implementation requires pInput to be + ** nul-terminated. This should either be fixed, or pInput/nBytes + ** should be converted to zInput. + */ + int (*xNext)( + sqlite3_tokenizer_cursor* pCursor, /* Tokenizer cursor */ + const char** ppToken, int* pnBytes, /* OUT: Normalized text for token */ + int* piStartOffset, /* OUT: Byte offset of token in input buffer */ + int* piEndOffset, /* OUT: Byte offset of end of token in input buffer */ + int* piPosition /* OUT: Number of tokens returned before this one */ + ); +}; + +struct sqlite3_tokenizer { + const sqlite3_tokenizer_module* pModule; /* The module for this tokenizer */ + /* Tokenizer implementations will typically add additional fields */ +}; + +struct sqlite3_tokenizer_cursor { + sqlite3_tokenizer* pTokenizer; /* Tokenizer for this cursor. */ + /* Tokenizer implementations will typically add additional fields */ +}; + +#endif /* _FTS3_TOKENIZER_H_ */ diff --git a/comm/mailnews/extensions/fts3/moz.build b/comm/mailnews/extensions/fts3/moz.build new file mode 100644 index 0000000000..a2d7a8de83 --- /dev/null +++ b/comm/mailnews/extensions/fts3/moz.build @@ -0,0 +1,25 @@ +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +XPIDL_SOURCES += [ + "nsIFts3Tokenizer.idl", +] + +XPIDL_MODULE = "fts3tok" + +SOURCES += [ + "fts3_porter.c", + "Normalize.c", + "nsFts3Tokenizer.cpp", + "nsGlodaRankerFunction.cpp", +] + +FINAL_LIBRARY = "mail" + +CXXFLAGS += CONFIG["SQLITE_CFLAGS"] + +XPCOM_MANIFESTS += [ + "components.conf", +] diff --git a/comm/mailnews/extensions/fts3/nsFts3Tokenizer.cpp b/comm/mailnews/extensions/fts3/nsFts3Tokenizer.cpp new file mode 100644 index 0000000000..9c85543147 --- /dev/null +++ b/comm/mailnews/extensions/fts3/nsFts3Tokenizer.cpp @@ -0,0 +1,59 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsFts3Tokenizer.h" + +#include "nsGlodaRankerFunction.h" + +#include "nsIFts3Tokenizer.h" +#include "mozIStorageConnection.h" +#include "mozIStorageStatement.h" +#include "nsString.h" + +extern "C" void sqlite3Fts3PorterTokenizerModule( + sqlite3_tokenizer_module const** ppModule); + +extern "C" void glodaRankFunc(sqlite3_context* pCtx, int nVal, + sqlite3_value** apVal); + +NS_IMPL_ISUPPORTS(nsFts3Tokenizer, nsIFts3Tokenizer) + +nsFts3Tokenizer::nsFts3Tokenizer() {} + +nsFts3Tokenizer::~nsFts3Tokenizer() {} + +NS_IMETHODIMP +nsFts3Tokenizer::RegisterTokenizer(mozIStorageConnection* connection) { + nsresult rv; + nsCOMPtr<mozIStorageStatement> selectStatement; + + // -- register the tokenizer + rv = connection->CreateStatement("SELECT fts3_tokenizer(?1, ?2)"_ns, + getter_AddRefs(selectStatement)); + NS_ENSURE_SUCCESS(rv, rv); + + const sqlite3_tokenizer_module* module = nullptr; + sqlite3Fts3PorterTokenizerModule(&module); + if (!module) return NS_ERROR_FAILURE; + + rv = selectStatement->BindUTF8StringByIndex(0, "mozporter"_ns); + NS_ENSURE_SUCCESS(rv, rv); + rv = selectStatement->BindBlobByIndex(1, (uint8_t*)&module, sizeof(module)); + NS_ENSURE_SUCCESS(rv, rv); + + bool hasMore; + rv = selectStatement->ExecuteStep(&hasMore); + NS_ENSURE_SUCCESS(rv, rv); + + // -- register the ranking function + nsCOMPtr<mozIStorageFunction> func = new nsGlodaRankerFunction(); + NS_ENSURE_TRUE(func, NS_ERROR_OUT_OF_MEMORY); + rv = connection->CreateFunction("glodaRank"_ns, + -1, // variable argument support + func); + NS_ENSURE_SUCCESS(rv, rv); + + return rv; +} diff --git a/comm/mailnews/extensions/fts3/nsFts3Tokenizer.h b/comm/mailnews/extensions/fts3/nsFts3Tokenizer.h new file mode 100644 index 0000000000..8e6bb7dab4 --- /dev/null +++ b/comm/mailnews/extensions/fts3/nsFts3Tokenizer.h @@ -0,0 +1,26 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef nsFts3Tokenizer_h__ +#define nsFts3Tokenizer_h__ + +#include "nsCOMPtr.h" +#include "nsIFts3Tokenizer.h" +#include "fts3_tokenizer.h" + +extern const sqlite3_tokenizer_module* getWindowsTokenizer(); + +class nsFts3Tokenizer final : public nsIFts3Tokenizer { + public: + NS_DECL_ISUPPORTS + NS_DECL_NSIFTS3TOKENIZER + + nsFts3Tokenizer(); + + private: + ~nsFts3Tokenizer(); +}; + +#endif diff --git a/comm/mailnews/extensions/fts3/nsGlodaRankerFunction.cpp b/comm/mailnews/extensions/fts3/nsGlodaRankerFunction.cpp new file mode 100644 index 0000000000..9ba125466f --- /dev/null +++ b/comm/mailnews/extensions/fts3/nsGlodaRankerFunction.cpp @@ -0,0 +1,106 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsGlodaRankerFunction.h" +#include "mozIStorageValueArray.h" + +#include "sqlite3.h" + +#include "nsCOMPtr.h" +#include "nsVariant.h" +#include "nsComponentManagerUtils.h" + +#ifndef SQLITE_VERSION_NUMBER +# error "We need SQLITE_VERSION_NUMBER defined!" +#endif + +NS_IMPL_ISUPPORTS(nsGlodaRankerFunction, mozIStorageFunction) + +nsGlodaRankerFunction::nsGlodaRankerFunction() {} + +nsGlodaRankerFunction::~nsGlodaRankerFunction() {} + +static uint32_t COLUMN_SATURATION[] = {10, 1, 1, 1, 1}; + +/** + * Our ranking function basically just multiplies the weight of the column + * against the number of (saturating) matches. + * + * The original code is a SQLite example ranking function, although somewhat + * rather modified at this point. All SQLite code is public domain, so we are + * subsuming it to MPL1.1/LGPL2/GPL2. + */ +NS_IMETHODIMP +nsGlodaRankerFunction::OnFunctionCall(mozIStorageValueArray* aArguments, + nsIVariant** _result) { + // all argument names are maintained from the original SQLite code. + uint32_t nVal; + nsresult rv = aArguments->GetNumEntries(&nVal); + NS_ENSURE_SUCCESS(rv, rv); + + /* Check that the number of arguments passed to this function is correct. + * If not, return an error. Set aArgsData to point to the array + * of unsigned integer values returned by FTS3 function. Set nPhrase + * to contain the number of reportable phrases in the users full-text + * query, and nCol to the number of columns in the table. + */ + if (nVal < 1) return NS_ERROR_INVALID_ARG; + + uint32_t lenArgsData; + uint32_t* aArgsData = (uint32_t*)aArguments->AsSharedBlob(0, &lenArgsData); + + uint32_t nPhrase = aArgsData[0]; + uint32_t nCol = aArgsData[1]; + if (nVal != (1 + nCol)) return NS_ERROR_INVALID_ARG; + + double score = 0.0; + + // SQLite 3.6.22 has a different matchinfo layout than SQLite 3.6.23+ +#if SQLITE_VERSION_NUMBER <= 3006022 + /* Iterate through each phrase in the users query. */ + for (uint32_t iPhrase = 0; iPhrase < nPhrase; iPhrase++) { + // in SQ + for (uint32_t iCol = 0; iCol < nCol; iCol++) { + uint32_t nHitCount = aArgsData[2 + (iPhrase + 1) * nCol + iCol]; + double weight = aArguments->AsDouble(iCol + 1); + if (nHitCount > 0) { + score += (nHitCount > COLUMN_SATURATION[iCol]) + ? (COLUMN_SATURATION[iCol] * weight) + : (nHitCount * weight); + } + } + } +#else + /* Iterate through each phrase in the users query. */ + for (uint32_t iPhrase = 0; iPhrase < nPhrase; iPhrase++) { + /* Now iterate through each column in the users query. For each column, + ** increment the relevancy score by: + ** + ** (<hit count> / <global hit count>) * <column weight> + ** + ** aPhraseinfo[] points to the start of the data for phrase iPhrase. So + ** the hit count and global hit counts for each column are found in + ** aPhraseinfo[iCol*3] and aPhraseinfo[iCol*3+1], respectively. + */ + uint32_t* aPhraseinfo = &aArgsData[2 + iPhrase * nCol * 3]; + for (uint32_t iCol = 0; iCol < nCol; iCol++) { + uint32_t nHitCount = aPhraseinfo[3 * iCol]; + double weight = aArguments->AsDouble(iCol + 1); + if (nHitCount > 0) { + score += (nHitCount > COLUMN_SATURATION[iCol]) + ? (COLUMN_SATURATION[iCol] * weight) + : (nHitCount * weight); + } + } + } +#endif + + nsCOMPtr<nsIWritableVariant> result = new nsVariant(); + + rv = result->SetAsDouble(score); + NS_ENSURE_SUCCESS(rv, rv); + + result.forget(_result); + return NS_OK; +} diff --git a/comm/mailnews/extensions/fts3/nsGlodaRankerFunction.h b/comm/mailnews/extensions/fts3/nsGlodaRankerFunction.h new file mode 100644 index 0000000000..0a19073a90 --- /dev/null +++ b/comm/mailnews/extensions/fts3/nsGlodaRankerFunction.h @@ -0,0 +1,25 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef _nsGlodaRankerFunction_h_ +#define _nsGlodaRankerFunction_h_ + +#include "mozIStorageFunction.h" + +/** + * Basically a port of the example FTS3 ranking function to mozStorage's + * view of the universe. This might get fancier at some point. + */ +class nsGlodaRankerFunction final : public mozIStorageFunction { + public: + NS_DECL_ISUPPORTS + NS_DECL_MOZISTORAGEFUNCTION + + nsGlodaRankerFunction(); + + private: + ~nsGlodaRankerFunction(); +}; + +#endif // _nsGlodaRankerFunction_h_ diff --git a/comm/mailnews/extensions/fts3/nsIFts3Tokenizer.idl b/comm/mailnews/extensions/fts3/nsIFts3Tokenizer.idl new file mode 100644 index 0000000000..c2bb7d435a --- /dev/null +++ b/comm/mailnews/extensions/fts3/nsIFts3Tokenizer.idl @@ -0,0 +1,15 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" + +interface mozIStorageConnection; + +[scriptable, uuid(136c88ea-7003-4fe8-8835-333fd18e598c)] +interface nsIFts3Tokenizer : nsISupports { + // register FTS3 tokenizer module for "mozporter" tokenizer + // mozporter is based by porter tokenizer with bi-gram tokenizer for CJK + void registerTokenizer(in mozIStorageConnection connection); +}; diff --git a/comm/mailnews/extensions/mailviews/components.conf b/comm/mailnews/extensions/mailviews/components.conf new file mode 100644 index 0000000000..757d1eca93 --- /dev/null +++ b/comm/mailnews/extensions/mailviews/components.conf @@ -0,0 +1,12 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, you can obtain one at http://mozilla.org/MPL/2.0/. + +Classes = [ + { + "cid": "{a0258267-44fd-4886-a858-8192615178ec}", + "contract_ids": ["@mozilla.org/messenger/mailviewlist;1"], + "type": "nsMsgMailViewList", + "headers": ["/comm/mailnews/extensions/mailviews/nsMsgMailViewList.h"], + }, +] diff --git a/comm/mailnews/extensions/mailviews/mailViews.dat b/comm/mailnews/extensions/mailviews/mailViews.dat new file mode 100644 index 0000000000..3c5af3831c --- /dev/null +++ b/comm/mailnews/extensions/mailviews/mailViews.dat @@ -0,0 +1,22 @@ +version="9" +logging="no" +name="People I Know" +enabled="yes" +type="1" +condition="AND (from,is in ab,jsaddrbook://abook.sqlite)" +name="Recent Mail" +enabled="yes" +type="1" +condition="AND (age in days,is less than,1)" +name="Last 5 Days" +enabled="yes" +type="1" +condition="AND (age in days,is less than,5)" +name="Not Junk" +enabled="yes" +type="1" +condition="AND (junk status,isn't,2)" +name="Has Attachments" +enabled="yes" +type="1" +condition="AND (has attachment status,is,true)" diff --git a/comm/mailnews/extensions/mailviews/moz.build b/comm/mailnews/extensions/mailviews/moz.build new file mode 100644 index 0000000000..de2df9ec3a --- /dev/null +++ b/comm/mailnews/extensions/mailviews/moz.build @@ -0,0 +1,25 @@ +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +SOURCES += [ + "nsMsgMailViewList.cpp", +] + +FINAL_LIBRARY = "mail" + +XPIDL_SOURCES += [ + "nsIMsgMailView.idl", + "nsIMsgMailViewList.idl", +] + +XPIDL_MODULE = "mailview" + +FINAL_TARGET_FILES.defaults.messenger += [ + "mailViews.dat", +] + +XPCOM_MANIFESTS += [ + "components.conf", +] diff --git a/comm/mailnews/extensions/mailviews/nsIMsgMailView.idl b/comm/mailnews/extensions/mailviews/nsIMsgMailView.idl new file mode 100644 index 0000000000..730cdcbc39 --- /dev/null +++ b/comm/mailnews/extensions/mailviews/nsIMsgMailView.idl @@ -0,0 +1,24 @@ +/* -*- Mode: IDL; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + + +#include "nsISupports.idl" + +interface nsIMsgSearchTerm; + +[scriptable, uuid(28AC84DF-CBE5-430d-A5C0-4FA63B5424DF)] +interface nsIMsgMailView : nsISupports { + attribute wstring mailViewName; + readonly attribute wstring prettyName; // localized pretty name + + // the array of search terms + attribute Array<nsIMsgSearchTerm> searchTerms; + + // these two helper methods are required to allow searchTermsOverlay.js to + // manipulate a mail view without knowing it is dealing with a mail view. nsIMsgFilter + // and nsIMsgSearchSession have the same two methods....we should probably make an interface around them. + void appendTerm(in nsIMsgSearchTerm term); + nsIMsgSearchTerm createTerm(); +}; diff --git a/comm/mailnews/extensions/mailviews/nsIMsgMailViewList.idl b/comm/mailnews/extensions/mailviews/nsIMsgMailViewList.idl new file mode 100644 index 0000000000..cc8b2f6d4b --- /dev/null +++ b/comm/mailnews/extensions/mailviews/nsIMsgMailViewList.idl @@ -0,0 +1,28 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" +#include "nsIMsgMailView.idl" + +/////////////////////////////////////////////////////////////////////////////// +// A mail view list is a list of mail views a particular implementor provides +/////////////////////////////////////////////////////////////////////////////// + +typedef long nsMsgMailViewListFileAttribValue; + +[scriptable, uuid(6DD798D7-9528-49e6-9447-3AAF14D2D36F)] +interface nsIMsgMailViewList : nsISupports { + + readonly attribute unsigned long mailViewCount; + + nsIMsgMailView getMailViewAt(in unsigned long mailViewIndex); + + void addMailView(in nsIMsgMailView mailView); + void removeMailView(in nsIMsgMailView mailView); + + nsIMsgMailView createMailView(); + + void save(); +}; diff --git a/comm/mailnews/extensions/mailviews/nsMsgMailViewList.cpp b/comm/mailnews/extensions/mailviews/nsMsgMailViewList.cpp new file mode 100644 index 0000000000..753061a540 --- /dev/null +++ b/comm/mailnews/extensions/mailviews/nsMsgMailViewList.cpp @@ -0,0 +1,281 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsMsgMailViewList.h" +#include "nsArray.h" +#include "nsIMsgFilterService.h" +#include "nsIMsgMailSession.h" +#include "nsIMsgSearchTerm.h" +#include "nsAppDirectoryServiceDefs.h" +#include "nsDirectoryServiceUtils.h" +#include "nsIFile.h" +#include "nsComponentManagerUtils.h" +#include "mozilla/Components.h" +#include "nsIMsgFilter.h" + +#define kDefaultViewPeopleIKnow "People I Know" +#define kDefaultViewRecent "Recent Mail" +#define kDefaultViewFiveDays "Last 5 Days" +#define kDefaultViewNotJunk "Not Junk" +#define kDefaultViewHasAttachments "Has Attachments" + +nsMsgMailView::nsMsgMailView() {} + +NS_IMPL_ADDREF(nsMsgMailView) +NS_IMPL_RELEASE(nsMsgMailView) +NS_IMPL_QUERY_INTERFACE(nsMsgMailView, nsIMsgMailView) + +nsMsgMailView::~nsMsgMailView() {} + +NS_IMETHODIMP nsMsgMailView::GetMailViewName(char16_t** aMailViewName) { + NS_ENSURE_ARG_POINTER(aMailViewName); + + *aMailViewName = ToNewUnicode(mName); + return NS_OK; +} + +NS_IMETHODIMP nsMsgMailView::SetMailViewName(const char16_t* aMailViewName) { + mName = aMailViewName; + return NS_OK; +} + +NS_IMETHODIMP nsMsgMailView::GetPrettyName(char16_t** aMailViewName) { + NS_ENSURE_ARG_POINTER(aMailViewName); + + nsresult rv = NS_OK; + if (!mBundle) { + nsCOMPtr<nsIStringBundleService> bundleService = + mozilla::components::StringBundle::Service(); + NS_ENSURE_TRUE(bundleService, NS_ERROR_UNEXPECTED); + bundleService->CreateBundle( + "chrome://messenger/locale/mailviews.properties", + getter_AddRefs(mBundle)); + } + + NS_ENSURE_TRUE(mBundle, NS_ERROR_FAILURE); + + // see if mName has an associated pretty name inside our string bundle and if + // so, use that as the pretty name otherwise just return mName + nsAutoString mailViewName; + if (mName.EqualsLiteral(kDefaultViewPeopleIKnow)) { + rv = mBundle->GetStringFromName("mailViewPeopleIKnow", mailViewName); + *aMailViewName = ToNewUnicode(mailViewName); + } else if (mName.EqualsLiteral(kDefaultViewRecent)) { + rv = mBundle->GetStringFromName("mailViewRecentMail", mailViewName); + *aMailViewName = ToNewUnicode(mailViewName); + } else if (mName.EqualsLiteral(kDefaultViewFiveDays)) { + rv = mBundle->GetStringFromName("mailViewLastFiveDays", mailViewName); + *aMailViewName = ToNewUnicode(mailViewName); + } else if (mName.EqualsLiteral(kDefaultViewNotJunk)) { + rv = mBundle->GetStringFromName("mailViewNotJunk", mailViewName); + *aMailViewName = ToNewUnicode(mailViewName); + } else if (mName.EqualsLiteral(kDefaultViewHasAttachments)) { + rv = mBundle->GetStringFromName("mailViewHasAttachments", mailViewName); + *aMailViewName = ToNewUnicode(mailViewName); + } else { + *aMailViewName = ToNewUnicode(mName); + } + + return rv; +} + +NS_IMETHODIMP nsMsgMailView::GetSearchTerms( + nsTArray<RefPtr<nsIMsgSearchTerm>>& terms) { + terms = mViewSearchTerms.Clone(); + return NS_OK; +} + +NS_IMETHODIMP nsMsgMailView::SetSearchTerms( + nsTArray<RefPtr<nsIMsgSearchTerm>> const& terms) { + mViewSearchTerms = terms.Clone(); + return NS_OK; +} + +NS_IMETHODIMP nsMsgMailView::AppendTerm(nsIMsgSearchTerm* aTerm) { + NS_ENSURE_TRUE(aTerm, NS_ERROR_NULL_POINTER); + + mViewSearchTerms.AppendElement(aTerm); + return NS_OK; +} + +NS_IMETHODIMP nsMsgMailView::CreateTerm(nsIMsgSearchTerm** aResult) { + NS_ENSURE_ARG_POINTER(aResult); + nsCOMPtr<nsIMsgSearchTerm> searchTerm = + do_CreateInstance("@mozilla.org/messenger/searchTerm;1"); + searchTerm.forget(aResult); + return NS_OK; +} + +///////////////////////////////////////////////////////////////////////////// +// nsMsgMailViewList implementation +///////////////////////////////////////////////////////////////////////////// +nsMsgMailViewList::nsMsgMailViewList() { LoadMailViews(); } + +NS_IMPL_ADDREF(nsMsgMailViewList) +NS_IMPL_RELEASE(nsMsgMailViewList) +NS_IMPL_QUERY_INTERFACE(nsMsgMailViewList, nsIMsgMailViewList) + +nsMsgMailViewList::~nsMsgMailViewList() {} + +NS_IMETHODIMP nsMsgMailViewList::GetMailViewCount(uint32_t* aCount) { + NS_ENSURE_ARG_POINTER(aCount); + + *aCount = m_mailViews.Length(); + return NS_OK; +} + +NS_IMETHODIMP nsMsgMailViewList::GetMailViewAt(uint32_t aMailViewIndex, + nsIMsgMailView** aMailView) { + NS_ENSURE_ARG_POINTER(aMailView); + + uint32_t mailViewCount = m_mailViews.Length(); + + NS_ENSURE_ARG(mailViewCount > aMailViewIndex); + + NS_IF_ADDREF(*aMailView = m_mailViews[aMailViewIndex]); + return NS_OK; +} + +NS_IMETHODIMP nsMsgMailViewList::AddMailView(nsIMsgMailView* aMailView) { + NS_ENSURE_ARG_POINTER(aMailView); + + m_mailViews.AppendElement(aMailView); + return NS_OK; +} + +NS_IMETHODIMP nsMsgMailViewList::RemoveMailView(nsIMsgMailView* aMailView) { + NS_ENSURE_ARG_POINTER(aMailView); + + m_mailViews.RemoveElement(aMailView); + return NS_OK; +} + +NS_IMETHODIMP nsMsgMailViewList::CreateMailView(nsIMsgMailView** aMailView) { + NS_ENSURE_ARG_POINTER(aMailView); + NS_ADDREF(*aMailView = new nsMsgMailView); + return NS_OK; +} + +NS_IMETHODIMP nsMsgMailViewList::Save() { + // brute force...remove all the old filters in our filter list, then we'll + // re-add our current list + nsCOMPtr<nsIMsgFilter> msgFilter; + uint32_t numFilters = 0; + if (mFilterList) mFilterList->GetFilterCount(&numFilters); + while (numFilters) { + mFilterList->RemoveFilterAt(numFilters - 1); + numFilters--; + } + + // now convert our mail view list into a filter list and save it + ConvertMailViewListToFilterList(); + + // now save the filters to our file + return mFilterList ? mFilterList->SaveToDefaultFile() : NS_ERROR_FAILURE; +} + +nsresult nsMsgMailViewList::ConvertMailViewListToFilterList() { + uint32_t mailViewCount = m_mailViews.Length(); + nsCOMPtr<nsIMsgMailView> mailView; + nsCOMPtr<nsIMsgFilter> newMailFilter; + nsString mailViewName; + for (uint32_t index = 0; index < mailViewCount; index++) { + GetMailViewAt(index, getter_AddRefs(mailView)); + if (!mailView) continue; + mailView->GetMailViewName(getter_Copies(mailViewName)); + mFilterList->CreateFilter(mailViewName, getter_AddRefs(newMailFilter)); + if (!newMailFilter) continue; + + nsTArray<RefPtr<nsIMsgSearchTerm>> searchTerms; + mailView->GetSearchTerms(searchTerms); + newMailFilter->SetSearchTerms(searchTerms); + mFilterList->InsertFilterAt(index, newMailFilter); + } + + return NS_OK; +} + +nsresult nsMsgMailViewList::LoadMailViews() { + nsCOMPtr<nsIFile> file; + nsresult rv = + NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = file->AppendNative("mailViews.dat"_ns); + + // if the file doesn't exist, we should try to get it from the defaults + // directory and copy it over + bool exists = false; + file->Exists(&exists); + if (!exists) { + nsCOMPtr<nsIMsgMailSession> mailSession = + do_GetService("@mozilla.org/messenger/services/session;1", &rv); + NS_ENSURE_SUCCESS(rv, rv); + nsCOMPtr<nsIFile> defaultMessagesFile; + nsCOMPtr<nsIFile> profileDir; + rv = mailSession->GetDataFilesDir("messenger", + getter_AddRefs(defaultMessagesFile)); + rv = defaultMessagesFile->AppendNative("mailViews.dat"_ns); + + // get the profile directory + rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, + getter_AddRefs(profileDir)); + + // now copy the file over to the profile directory + defaultMessagesFile->CopyToNative(profileDir, EmptyCString()); + } + // this is kind of a hack but I think it will be an effective hack. The filter + // service already knows how to take a nsIFile and parse the contents into + // filters which are very similar to mail views. Instead of re-writing all of + // that dirty parsing code, let's just re-use it then convert the results into + // a data structure we wish to give to our consumers. + + nsCOMPtr<nsIMsgFilterService> filterService = + do_GetService("@mozilla.org/messenger/services/filters;1", &rv); + nsCOMPtr<nsIMsgFilterList> mfilterList; + + rv = filterService->OpenFilterList(file, nullptr, nullptr, + getter_AddRefs(mFilterList)); + NS_ENSURE_SUCCESS(rv, rv); + + return ConvertFilterListToMailViews(); +} +/** + * Converts the filter list into our mail view objects, + * stripping out just the info we need. + */ +nsresult nsMsgMailViewList::ConvertFilterListToMailViews() { + nsresult rv = NS_OK; + m_mailViews.Clear(); + + // iterate over each filter in the list + uint32_t numFilters = 0; + mFilterList->GetFilterCount(&numFilters); + for (uint32_t index = 0; index < numFilters; index++) { + nsCOMPtr<nsIMsgFilter> msgFilter; + rv = mFilterList->GetFilterAt(index, getter_AddRefs(msgFilter)); + if (NS_FAILED(rv) || !msgFilter) continue; + + // create a new nsIMsgMailView for this item + nsCOMPtr<nsIMsgMailView> newMailView; + rv = CreateMailView(getter_AddRefs(newMailView)); + NS_ENSURE_SUCCESS(rv, rv); + + nsString filterName; + msgFilter->GetFilterName(filterName); + newMailView->SetMailViewName(filterName.get()); + + nsTArray<RefPtr<nsIMsgSearchTerm>> filterSearchTerms; + rv = msgFilter->GetSearchTerms(filterSearchTerms); + NS_ENSURE_SUCCESS(rv, rv); + rv = newMailView->SetSearchTerms(filterSearchTerms); + NS_ENSURE_SUCCESS(rv, rv); + + // now append this new mail view to our global list view + m_mailViews.AppendElement(newMailView); + } + + return rv; +} diff --git a/comm/mailnews/extensions/mailviews/nsMsgMailViewList.h b/comm/mailnews/extensions/mailviews/nsMsgMailViewList.h new file mode 100644 index 0000000000..19e283681c --- /dev/null +++ b/comm/mailnews/extensions/mailviews/nsMsgMailViewList.h @@ -0,0 +1,51 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef _nsMsgMailViewList_H_ +#define _nsMsgMailViewList_H_ + +#include "nscore.h" +#include "nsIMsgMailViewList.h" +#include "nsCOMPtr.h" +#include "nsCOMArray.h" +#include "nsIStringBundle.h" +#include "nsString.h" +#include "nsIMsgFilterList.h" + +// a mail View is just a name and an array of search terms +class nsMsgMailView : public nsIMsgMailView { + public: + NS_DECL_ISUPPORTS + NS_DECL_NSIMSGMAILVIEW + + nsMsgMailView(); + + protected: + virtual ~nsMsgMailView(); + nsString mName; + nsCOMPtr<nsIStringBundle> mBundle; + nsTArray<RefPtr<nsIMsgSearchTerm>> mViewSearchTerms; +}; + +class nsMsgMailViewList : public nsIMsgMailViewList { + public: + NS_DECL_ISUPPORTS + NS_DECL_NSIMSGMAILVIEWLIST + + nsMsgMailViewList(); + + protected: + virtual ~nsMsgMailViewList(); + nsresult + LoadMailViews(); // reads in user defined mail views from our default file + nsresult ConvertFilterListToMailViews(); + nsresult ConvertMailViewListToFilterList(); + + nsCOMArray<nsIMsgMailView> m_mailViews; + nsCOMPtr<nsIMsgFilterList> + mFilterList; // our internal filter list representation +}; + +#endif diff --git a/comm/mailnews/extensions/mdn/MDNService.jsm b/comm/mailnews/extensions/mdn/MDNService.jsm new file mode 100644 index 0000000000..47f5257b6a --- /dev/null +++ b/comm/mailnews/extensions/mdn/MDNService.jsm @@ -0,0 +1,24 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +var EXPORTED_SYMBOLS = ["MDNService"]; + +function MDNService() {} + +MDNService.prototype = { + name: "mdn", + chromePackageName: "messenger", + showPanel(server) { + // don't show the panel for news, rss, im or local accounts + return ( + server.type != "nntp" && + server.type != "rss" && + server.type != "im" && + server.type != "none" + ); + }, + + QueryInterface: ChromeUtils.generateQI(["nsIMsgAccountManagerExtension"]), +}; diff --git a/comm/mailnews/extensions/mdn/am-mdn.js b/comm/mailnews/extensions/mdn/am-mdn.js new file mode 100644 index 0000000000..7725339fbd --- /dev/null +++ b/comm/mailnews/extensions/mdn/am-mdn.js @@ -0,0 +1,161 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* import-globals-from ../../base/prefs/content/amUtils.js */ + +var useCustomPrefs; +var requestReceipt; +var leaveInInbox; +var moveToSent; +var receiptSend; +var neverReturn; +var returnSome; +var notInToCcPref; +var notInToCcLabel; +var outsideDomainPref; +var outsideDomainLabel; +var otherCasesPref; +var otherCasesLabel; +var receiptArriveLabel; +var receiptRequestLabel; +var gIdentity; +var gIncomingServer; +var gMdnPrefBranch; + +function onInit() { + useCustomPrefs = document.getElementById("identity.use_custom_prefs"); + requestReceipt = document.getElementById( + "identity.request_return_receipt_on" + ); + leaveInInbox = document.getElementById("leave_in_inbox"); + moveToSent = document.getElementById("move_to_sent"); + receiptSend = document.getElementById("server.mdn_report_enabled"); + neverReturn = document.getElementById("never_return"); + returnSome = document.getElementById("return_some"); + notInToCcPref = document.getElementById("server.mdn_not_in_to_cc"); + notInToCcLabel = document.getElementById("notInToCcLabel"); + outsideDomainPref = document.getElementById("server.mdn_outside_domain"); + outsideDomainLabel = document.getElementById("outsideDomainLabel"); + otherCasesPref = document.getElementById("server.mdn_other"); + otherCasesLabel = document.getElementById("otherCasesLabel"); + receiptArriveLabel = document.getElementById("receiptArriveLabel"); + receiptRequestLabel = document.getElementById("receiptRequestLabel"); + + EnableDisableCustomSettings(); + + return true; +} + +function onSave() {} + +function EnableDisableCustomSettings() { + if (useCustomPrefs && useCustomPrefs.getAttribute("value") == "false") { + requestReceipt.setAttribute("disabled", "true"); + leaveInInbox.setAttribute("disabled", "true"); + moveToSent.setAttribute("disabled", "true"); + neverReturn.setAttribute("disabled", "true"); + returnSome.setAttribute("disabled", "true"); + receiptArriveLabel.setAttribute("disabled", "true"); + receiptRequestLabel.setAttribute("disabled", "true"); + } else { + requestReceipt.removeAttribute("disabled"); + leaveInInbox.removeAttribute("disabled"); + moveToSent.removeAttribute("disabled"); + neverReturn.removeAttribute("disabled"); + returnSome.removeAttribute("disabled"); + receiptArriveLabel.removeAttribute("disabled"); + receiptRequestLabel.removeAttribute("disabled"); + } + EnableDisableAllowedReceipts(); + // Lock id based prefs + onLockPreference("mail.identity", gIdentity.key); + // Lock server based prefs + onLockPreference("mail.server", gIncomingServer.key); + return true; +} + +function EnableDisableAllowedReceipts() { + if (receiptSend) { + if ( + !neverReturn.getAttribute("disabled") && + receiptSend.getAttribute("value") != "false" + ) { + notInToCcPref.removeAttribute("disabled"); + notInToCcLabel.removeAttribute("disabled"); + outsideDomainPref.removeAttribute("disabled"); + outsideDomainLabel.removeAttribute("disabled"); + otherCasesPref.removeAttribute("disabled"); + otherCasesLabel.removeAttribute("disabled"); + } else { + notInToCcPref.setAttribute("disabled", "true"); + notInToCcLabel.setAttribute("disabled", "true"); + outsideDomainPref.setAttribute("disabled", "true"); + outsideDomainLabel.setAttribute("disabled", "true"); + otherCasesPref.setAttribute("disabled", "true"); + otherCasesLabel.setAttribute("disabled", "true"); + } + } + return true; +} + +function onPreInit(account, accountValues) { + gIdentity = account.defaultIdentity; + gIncomingServer = account.incomingServer; +} + +// Disables xul elements that have associated preferences locked. +function onLockPreference(initPrefString, keyString) { + var allPrefElements = [ + { + prefstring: "request_return_receipt_on", + id: "identity.request_return_receipt_on", + }, + { prefstring: "select_custom_prefs", id: "identity.select_custom_prefs" }, + { prefstring: "select_global_prefs", id: "identity.select_global_prefs" }, + { + prefstring: "incorporate_return_receipt", + id: "server.incorporate_return_receipt", + }, + { prefstring: "never_return", id: "never_return" }, + { prefstring: "return_some", id: "return_some" }, + { prefstring: "mdn_not_in_to_cc", id: "server.mdn_not_in_to_cc" }, + { prefstring: "mdn_outside_domain", id: "server.mdn_outside_domain" }, + { prefstring: "mdn_other", id: "server.mdn_other" }, + ]; + + var finalPrefString = initPrefString + "." + keyString + "."; + gMdnPrefBranch = Services.prefs.getBranch(finalPrefString); + + disableIfLocked(allPrefElements); +} + +function disableIfLocked(prefstrArray) { + for (let i = 0; i < prefstrArray.length; i++) { + var id = prefstrArray[i].id; + var element = document.getElementById(id); + if (gMdnPrefBranch.prefIsLocked(prefstrArray[i].prefstring)) { + if (id == "server.incorporate_return_receipt") { + document + .getElementById("leave_in_inbox") + .setAttribute("disabled", "true"); + document + .getElementById("move_to_sent") + .setAttribute("disabled", "true"); + } else { + element.setAttribute("disabled", "true"); + } + } + } +} + +/** + * Opens Preferences (Options) dialog on the pane and tab where + * the global receipts settings can be found. + */ +function showGlobalReceipts() { + parent.gSubDialog.open( + "chrome://messenger/content/preferences/receipts.xhtml", + { features: "resizable=no" } + ); +} diff --git a/comm/mailnews/extensions/mdn/am-mdn.xhtml b/comm/mailnews/extensions/mdn/am-mdn.xhtml new file mode 100644 index 0000000000..3cdf85d9e5 --- /dev/null +++ b/comm/mailnews/extensions/mdn/am-mdn.xhtml @@ -0,0 +1,231 @@ +<?xml version="1.0"?> +<!-- + This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> + +<?xml-stylesheet href="chrome://messenger/skin/accountManage.css" type="text/css"?> + +<!DOCTYPE html SYSTEM "chrome://messenger/locale/am-mdn.dtd"> + +<html + xmlns="http://www.w3.org/1999/xhtml" + xmlns:xul="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul" + xmlns:html="http://www.w3.org/1999/xhtml" +> + <head> + <title>&pane.title;</title> + <script + defer="defer" + src="chrome://messenger/content/AccountManager.js" + ></script> + <script defer="defer" src="chrome://messenger/content/amUtils.js"></script> + <script defer="defer" src="chrome://messenger/content/am-mdn.js"></script> + <script> + // FIXME: move to script file. + window.addEventListener("load", event => { + parent.onPanelLoaded("am-mdn.xhtml"); + }); + </script> + </head> + <html:body + xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul" + > + <vbox id="containerBox" flex="1"> + <stringbundle + id="bundle_smime" + src="chrome://messenger/locale/am-mdn.properties" + /> + + <hbox class="dialogheader"> + <label class="dialogheader-title" value="&pane.title;" /> + </hbox> + + <separator class="thin" /> + + <html:div> + <html:fieldset> + <hbox id="prefChoices" align="center" flex="1"> + <radiogroup + id="identity.use_custom_prefs" + wsm_persist="true" + genericattr="true" + preftype="bool" + prefstring="mail.identity.%identitykey%.use_custom_prefs" + oncommand="EnableDisableCustomSettings();" + flex="1" + > + <radio + id="identity.select_global_prefs" + value="false" + label="&useGlobalPrefs.label;" + accesskey="&useGlobalPrefs.accesskey;" + /> + <hbox flex="1"> + <spacer flex="1" /> + <button + id="globalReceiptsLink" + label="&globalReceipts.label;" + accesskey="&globalReceipts.accesskey;" + oncommand="showGlobalReceipts();" + /> + </hbox> + <radio + id="identity.select_custom_prefs" + value="true" + label="&useCustomPrefs.label;" + accesskey="&useCustomPrefs.accesskey;" + /> + </radiogroup> + </hbox> + + <vbox id="returnReceiptSettings" class="indent" align="start"> + <checkbox + id="identity.request_return_receipt_on" + label="&requestReceipt.label;" + accesskey="&requestReceipt.accesskey;" + wsm_persist="true" + genericattr="true" + iscontrolcontainer="true" + preftype="bool" + prefstring="mail.identity.%identitykey%.request_return_receipt_on" + /> + + <separator /> + + <vbox id="receiptArrive"> + <label + id="receiptArriveLabel" + control="server.incorporate_return_receipt" + >&receiptArrive.label;</label + > + <radiogroup + id="server.incorporate_return_receipt" + wsm_persist="true" + genericattr="true" + preftype="int" + prefstring="mail.server.%serverkey%.incorporate_return_receipt" + class="indent" + > + <radio + id="leave_in_inbox" + value="0" + label="&leaveIt.label;" + accesskey="&leaveIt.accesskey;" + /> + <radio + id="move_to_sent" + value="1" + label="&moveToSent.label;" + accesskey="&moveToSent.accesskey;" + /> + </radiogroup> + </vbox> + + <separator /> + + <vbox id="receiptRequest"> + <label + id="receiptRequestLabel" + control="server.mdn_report_enabled" + >&requestMDN.label;</label + > + <radiogroup + id="server.mdn_report_enabled" + wsm_persist="true" + genericattr="true" + preftype="bool" + prefstring="mail.server.%serverkey%.mdn_report_enabled" + oncommand="EnableDisableAllowedReceipts();" + class="indent" + > + <radio + id="never_return" + value="false" + label="&never.label;" + accesskey="&never.accesskey;" + /> + <radio + id="return_some" + value="true" + label="&returnSome.label;" + accesskey="&returnSome.accesskey;" + /> + + <hbox id="receiptSendIf" class="indent"> + <vbox> + <hbox flex="1" align="center"> + <label + id="notInToCcLabel" + value="¬InToCc.label;" + accesskey="¬InToCc.accesskey;" + control="server.mdn_not_in_to_cc" + /> + </hbox> + <hbox flex="1" align="center"> + <label + id="outsideDomainLabel" + value="&outsideDomain.label;" + accesskey="&outsideDomain.accesskey;" + control="server.mdn_outside_domain" + /> + </hbox> + <hbox flex="1" align="center"> + <label + id="otherCasesLabel" + value="&otherCases.label;" + accesskey="&otherCases.accesskey;" + control="server.mdn_other" + /> + </hbox> + </vbox> + <vbox> + <menulist + id="server.mdn_not_in_to_cc" + wsm_persist="true" + genericattr="true" + preftype="int" + prefstring="mail.server.%serverkey%.mdn_not_in_to_cc" + > + <menupopup> + <menuitem value="0" label="&neverSend.label;" /> + <menuitem value="1" label="&alwaysSend.label;" /> + <menuitem value="2" label="&askMe.label;" /> + </menupopup> + </menulist> + <menulist + id="server.mdn_outside_domain" + wsm_persist="true" + genericattr="true" + preftype="int" + prefstring="mail.server.%serverkey%.mdn_outside_domain" + > + <menupopup> + <menuitem value="0" label="&neverSend.label;" /> + <menuitem value="1" label="&alwaysSend.label;" /> + <menuitem value="2" label="&askMe.label;" /> + </menupopup> + </menulist> + <menulist + id="server.mdn_other" + wsm_persist="true" + genericattr="true" + preftype="int" + prefstring="mail.server.%serverkey%.mdn_other" + > + <menupopup> + <menuitem value="0" label="&neverSend.label;" /> + <menuitem value="1" label="&alwaysSend.label;" /> + <menuitem value="2" label="&askMe.label;" /> + </menupopup> + </menulist> + </vbox> + </hbox> + </radiogroup> + </vbox> + </vbox> + </html:fieldset> + </html:div> + </vbox> + </html:body> +</html> diff --git a/comm/mailnews/extensions/mdn/components.conf b/comm/mailnews/extensions/mdn/components.conf new file mode 100644 index 0000000000..84f53febbf --- /dev/null +++ b/comm/mailnews/extensions/mdn/components.conf @@ -0,0 +1,23 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +Classes = [ + { + "cid": "{e007d92e-1dd1-11b2-a61e-dc962c9b8571}", + "contract_ids": ["@mozilla.org/accountmanager/extension;1?name=mdn"], + "jsm": "resource:///modules/MDNService.jsm", + "constructor": "MDNService", + "categories": { + "mailnews-accountmanager-extensions": "mdn-account-manager-extension" + }, + }, + { + "cid": "{ec917b13-8f73-4d4d-9146-d7f7aafe9076}", + "contract_ids": ["@mozilla.org/messenger-mdn/generator;1"], + "type": "nsMsgMdnGenerator", + "headers": ["/comm/mailnews/extensions/mdn/nsMsgMdnGenerator.h"], + }, +] diff --git a/comm/mailnews/extensions/mdn/jar.mn b/comm/mailnews/extensions/mdn/jar.mn new file mode 100644 index 0000000000..cb487cd969 --- /dev/null +++ b/comm/mailnews/extensions/mdn/jar.mn @@ -0,0 +1,7 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +messenger.jar: + content/messenger/am-mdn.xhtml (am-mdn.xhtml) + content/messenger/am-mdn.js (am-mdn.js) diff --git a/comm/mailnews/extensions/mdn/mdn.js b/comm/mailnews/extensions/mdn/mdn.js new file mode 100644 index 0000000000..ae82382f39 --- /dev/null +++ b/comm/mailnews/extensions/mdn/mdn.js @@ -0,0 +1,25 @@ +#filter dumbComments emptyLines substitution + +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// +// default prefs for mdn +// + +// false: Use global true: Use custom +pref("mail.identity.default.use_custom_prefs", false); +pref("mail.identity.default.request_return_receipt_on", false); +// 0: Inbox/filter 1: Sent folder +pref("mail.server.default.incorporate_return_receipt", 0); +// false: Never return receipts true: Return some receipts +pref("mail.server.default.mdn_report_enabled", true); +// 0: Never 1: Always 2: Ask me 3: Denial +pref("mail.server.default.mdn_not_in_to_cc", 2); +pref("mail.server.default.mdn_outside_domain", 2); +pref("mail.server.default.mdn_other", 2); +// return receipt header type - 0: MDN-DNT 1: RRT 2: Both +pref("mail.identity.default.request_receipt_header_type", 0); + +pref("mail.server.default.mdn_report_enabled", true); diff --git a/comm/mailnews/extensions/mdn/moz.build b/comm/mailnews/extensions/mdn/moz.build new file mode 100644 index 0000000000..2d7b8bce5d --- /dev/null +++ b/comm/mailnews/extensions/mdn/moz.build @@ -0,0 +1,26 @@ +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +JAR_MANIFESTS += ["jar.mn"] + +JS_PREFERENCE_PP_FILES += [ + "mdn.js", +] + +XPCSHELL_TESTS_MANIFESTS += ["test/unit/xpcshell.ini"] + +SOURCES += [ + "nsMsgMdnGenerator.cpp", +] + +EXTRA_JS_MODULES += [ + "MDNService.jsm", +] + +XPCOM_MANIFESTS += [ + "components.conf", +] + +FINAL_LIBRARY = "mail" diff --git a/comm/mailnews/extensions/mdn/nsMsgMdnGenerator.cpp b/comm/mailnews/extensions/mdn/nsMsgMdnGenerator.cpp new file mode 100644 index 0000000000..76fc6d5214 --- /dev/null +++ b/comm/mailnews/extensions/mdn/nsMsgMdnGenerator.cpp @@ -0,0 +1,1040 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsMsgMdnGenerator.h" +#include "nsImapCore.h" +#include "nsIMsgImapMailFolder.h" +#include "nsIMsgAccountManager.h" +#include "nsMimeTypes.h" +#include "prprf.h" +#include "prmem.h" +#include "prsystem.h" +#include "nsMsgI18N.h" +#include "nsMailHeaders.h" +#include "nsMsgLocalFolderHdrs.h" +#include "nsIHttpProtocolHandler.h" +#include "nsISmtpService.h" // for actually sending the message... +#include "nsComposeStrings.h" +#include "nsISmtpServer.h" +#include "nsIPrompt.h" +#include "nsIMsgCompUtils.h" +#include "nsIPrefService.h" +#include "nsIPrefBranch.h" +#include "nsIStringBundle.h" +#include "nsDirectoryServiceDefs.h" +#include "nsMsgUtils.h" +#include "nsNetUtil.h" +#include "nsIMsgDatabase.h" +#include "mozilla/Components.h" +#include "mozilla/mailnews/MimeHeaderParser.h" +#include "mozilla/Unused.h" +#include "nsIPromptService.h" +#include "nsEmbedCID.h" + +using namespace mozilla::mailnews; + +#define MDN_NOT_IN_TO_CC ((int)0x0001) +#define MDN_OUTSIDE_DOMAIN ((int)0x0002) + +#define HEADER_RETURN_PATH "Return-Path" +#define HEADER_DISPOSITION_NOTIFICATION_TO "Disposition-Notification-To" +#define HEADER_APPARENTLY_TO "Apparently-To" +#define HEADER_ORIGINAL_RECIPIENT "Original-Recipient" +#define HEADER_REPORTING_UA "Reporting-UA" +#define HEADER_MDN_GATEWAY "MDN-Gateway" +#define HEADER_FINAL_RECIPIENT "Final-Recipient" +#define HEADER_DISPOSITION "Disposition" +#define HEADER_ORIGINAL_MESSAGE_ID "Original-Message-ID" +#define HEADER_FAILURE "Failure" +#define HEADER_ERROR "Error" +#define HEADER_WARNING "Warning" +#define HEADER_RETURN_RECEIPT_TO "Return-Receipt-To" +#define HEADER_X_ACCEPT_LANGUAGE "X-Accept-Language" + +#define PUSH_N_FREE_STRING(p) \ + do { \ + if (p) { \ + rv = WriteString(p); \ + PR_smprintf_free(p); \ + p = 0; \ + if (NS_FAILED(rv)) return rv; \ + } else { \ + return NS_ERROR_OUT_OF_MEMORY; \ + } \ + } while (0) + +// String bundle for mdn. Class static. +#define MDN_STRINGBUNDLE_URL "chrome://messenger/locale/msgmdn.properties" + +#if defined(DEBUG_jefft) +# define DEBUG_MDN(s) printf("%s\n", s) +#else +# define DEBUG_MDN(s) +#endif + +// machine parsible string; should not be localized +char DispositionTypes[7][16] = { + "displayed", "dispatched", "processed", "deleted", "denied", "failed", ""}; + +NS_IMPL_ISUPPORTS(nsMsgMdnGenerator, nsIMsgMdnGenerator, nsIUrlListener) + +nsMsgMdnGenerator::nsMsgMdnGenerator() + : m_disposeType(eDisplayed), + m_key(nsMsgKey_None), + m_notInToCcOp(eNeverSendOp), + m_outsideDomainOp(eNeverSendOp), + m_otherOp(eNeverSendOp), + m_reallySendMdn(false), + m_autoSend(false), + m_autoAction(false), + m_mdnEnabled(false) {} + +nsMsgMdnGenerator::~nsMsgMdnGenerator() {} + +nsresult nsMsgMdnGenerator::FormatStringFromName(const char* aName, + const nsString& aString, + nsAString& aResultString) { + DEBUG_MDN("nsMsgMdnGenerator::FormatStringFromName"); + + nsCOMPtr<nsIStringBundleService> bundleService = + mozilla::components::StringBundle::Service(); + NS_ENSURE_TRUE(bundleService, NS_ERROR_UNEXPECTED); + + nsCOMPtr<nsIStringBundle> bundle; + nsresult rv = + bundleService->CreateBundle(MDN_STRINGBUNDLE_URL, getter_AddRefs(bundle)); + NS_ENSURE_SUCCESS(rv, rv); + + AutoTArray<nsString, 1> formatStrings = {aString}; + rv = bundle->FormatStringFromName(aName, formatStrings, aResultString); + NS_ENSURE_SUCCESS(rv, rv); + return rv; +} + +nsresult nsMsgMdnGenerator::GetStringFromName(const char* aName, + nsAString& aResultString) { + DEBUG_MDN("nsMsgMdnGenerator::GetStringFromName"); + + nsCOMPtr<nsIStringBundleService> bundleService = + mozilla::components::StringBundle::Service(); + NS_ENSURE_TRUE(bundleService, NS_ERROR_UNEXPECTED); + + nsCOMPtr<nsIStringBundle> bundle; + nsresult rv = + bundleService->CreateBundle(MDN_STRINGBUNDLE_URL, getter_AddRefs(bundle)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = bundle->GetStringFromName(aName, aResultString); + NS_ENSURE_SUCCESS(rv, rv); + return rv; +} + +nsresult nsMsgMdnGenerator::StoreMDNSentFlag(nsIMsgFolder* folder, + nsMsgKey key) { + DEBUG_MDN("nsMsgMdnGenerator::StoreMDNSentFlag"); + + nsCOMPtr<nsIMsgDatabase> msgDB; + nsresult rv = folder->GetMsgDatabase(getter_AddRefs(msgDB)); + NS_ENSURE_SUCCESS(rv, rv); + rv = msgDB->MarkMDNSent(key, true, nullptr); + + nsCOMPtr<nsIMsgImapMailFolder> imapFolder = do_QueryInterface(folder); + // Store the $MDNSent flag if the folder is an Imap Mail Folder + if (imapFolder) + return imapFolder->StoreImapFlags(kImapMsgMDNSentFlag, true, {key}, + nullptr); + return rv; +} + +nsresult nsMsgMdnGenerator::ClearMDNNeededFlag(nsIMsgFolder* folder, + nsMsgKey key) { + DEBUG_MDN("nsMsgMdnGenerator::ClearMDNNeededFlag"); + + nsCOMPtr<nsIMsgDatabase> msgDB; + nsresult rv = folder->GetMsgDatabase(getter_AddRefs(msgDB)); + NS_ENSURE_SUCCESS(rv, rv); + return msgDB->MarkMDNNeeded(key, false, nullptr); +} + +bool nsMsgMdnGenerator::ProcessSendMode() { + DEBUG_MDN("nsMsgMdnGenerator::ProcessSendMode"); + int32_t miscState = 0; + + if (m_identity) { + m_identity->GetEmail(m_email); + if (m_email.IsEmpty()) return m_reallySendMdn; + + const char* accountDomain = strchr(m_email.get(), '@'); + if (!accountDomain) return m_reallySendMdn; + + if (MailAddrMatch(m_email.get(), + m_dntRrt.get())) // return address is self, don't send + return false; + + // *** fix me see Bug 132504 for more information + // *** what if the message has been filtered to different account + if (!PL_strcasestr(m_dntRrt.get(), accountDomain)) + miscState |= MDN_OUTSIDE_DOMAIN; + if (NotInToOrCc()) miscState |= MDN_NOT_IN_TO_CC; + m_reallySendMdn = true; + // ********* + // How are we gona deal with the auto forwarding issues? Some server + // didn't bother to add addition header or modify existing header to + // the message when forwarding. They simply copy the exact same + // message to another user's mailbox. Some change To: to + // Apparently-To: + // Unfortunately, there is nothing we can do. It's out of our control. + // ********* + // starting from lowest denominator to highest + if (!miscState) { // under normal situation: recipent is in to and cc list, + // and the sender is from the same domain + switch (m_otherOp) { + default: + case eNeverSendOp: + m_reallySendMdn = false; + break; + case eAutoSendOp: + m_autoSend = true; + break; + case eAskMeOp: + m_autoSend = false; + break; + case eDeniedOp: + m_autoSend = true; + m_disposeType = eDenied; + break; + } + } else if (miscState == (MDN_OUTSIDE_DOMAIN | MDN_NOT_IN_TO_CC)) { + if (m_outsideDomainOp != m_notInToCcOp) { + m_autoSend = false; // ambiguous; always ask user + } else { + switch (m_outsideDomainOp) { + default: + case eNeverSendOp: + m_reallySendMdn = false; + break; + case eAutoSendOp: + m_autoSend = true; + break; + case eAskMeOp: + m_autoSend = false; + break; + } + } + } else if (miscState & MDN_OUTSIDE_DOMAIN) { + switch (m_outsideDomainOp) { + default: + case eNeverSendOp: + m_reallySendMdn = false; + break; + case eAutoSendOp: + m_autoSend = true; + break; + case eAskMeOp: + m_autoSend = false; + break; + } + } else if (miscState & MDN_NOT_IN_TO_CC) { + switch (m_notInToCcOp) { + default: + case eNeverSendOp: + m_reallySendMdn = false; + break; + case eAutoSendOp: + m_autoSend = true; + break; + case eAskMeOp: + m_autoSend = false; + break; + } + } + } + return m_reallySendMdn; +} + +bool nsMsgMdnGenerator::MailAddrMatch(const char* addr1, const char* addr2) { + // Comparing two email addresses returns true if matched; local/account + // part comparison is case sensitive; domain part comparison is case + // insensitive + DEBUG_MDN("nsMsgMdnGenerator::MailAddrMatch"); + bool isMatched = true; + const char *atSign1 = nullptr, *atSign2 = nullptr; + const char *lt = nullptr, *local1 = nullptr, *local2 = nullptr; + const char *end1 = nullptr, *end2 = nullptr; + + if (!addr1 || !addr2) return false; + + lt = strchr(addr1, '<'); + local1 = !lt ? addr1 : lt + 1; + lt = strchr(addr2, '<'); + local2 = !lt ? addr2 : lt + 1; + end1 = strchr(local1, '>'); + if (!end1) end1 = addr1 + strlen(addr1); + end2 = strchr(local2, '>'); + if (!end2) end2 = addr2 + strlen(addr2); + atSign1 = strchr(local1, '@'); + atSign2 = strchr(local2, '@'); + if (!atSign1 || !atSign2 // ill formed addr spec + || (atSign1 - local1) != (atSign2 - local2)) + isMatched = false; + else if (strncmp(local1, local2, (atSign1 - local1))) // case sensitive + // compare for local part + isMatched = false; + else if ((end1 - atSign1) != (end2 - atSign2) || + PL_strncasecmp(atSign1, atSign2, (end1 - atSign1))) // case + // insensitive compare for domain part + isMatched = false; + return isMatched; +} + +bool nsMsgMdnGenerator::NotInToOrCc() { + DEBUG_MDN("nsMsgMdnGenerator::NotInToOrCc"); + nsCString reply_to; + nsCString to; + nsCString cc; + + m_identity->GetReplyTo(reply_to); + m_headers->ExtractHeader(HEADER_TO, true, to); + m_headers->ExtractHeader(HEADER_CC, true, cc); + + // start with a simple check + if ((!to.IsEmpty() && PL_strcasestr(to.get(), m_email.get())) || + (!cc.IsEmpty() && PL_strcasestr(cc.get(), m_email.get()))) { + return false; + } + + if ((!reply_to.IsEmpty() && !to.IsEmpty() && + PL_strcasestr(to.get(), reply_to.get())) || + (!reply_to.IsEmpty() && !cc.IsEmpty() && + PL_strcasestr(cc.get(), reply_to.get()))) { + return false; + } + return true; +} + +bool nsMsgMdnGenerator::ValidateReturnPath() { + DEBUG_MDN("nsMsgMdnGenerator::ValidateReturnPath"); + // ValidateReturnPath applies to Automatic Send Mode only. If we were not + // in auto send mode we simply by passing the check + if (!m_autoSend) return m_reallySendMdn; + + nsCString returnPath; + m_headers->ExtractHeader(HEADER_RETURN_PATH, false, returnPath); + if (returnPath.IsEmpty()) { + m_autoSend = false; + return m_reallySendMdn; + } + m_autoSend = MailAddrMatch(returnPath.get(), m_dntRrt.get()); + return m_reallySendMdn; +} + +nsresult nsMsgMdnGenerator::CreateMdnMsg() { + DEBUG_MDN("nsMsgMdnGenerator::CreateMdnMsg"); + nsresult rv; + + nsCOMPtr<nsIFile> tmpFile; + rv = GetSpecialDirectoryWithFileName(NS_OS_TEMP_DIR, "mdnmsg", + getter_AddRefs(m_file)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = m_file->CreateUnique(nsIFile::NORMAL_FILE_TYPE, 00600); + NS_ENSURE_SUCCESS(rv, rv); + rv = MsgNewBufferedFileOutputStream(getter_AddRefs(m_outputStream), m_file, + PR_CREATE_FILE | PR_WRONLY | PR_TRUNCATE, + 0664); + NS_ASSERTION(NS_SUCCEEDED(rv), "creating mdn: failed to output stream"); + if (NS_FAILED(rv)) return NS_OK; + + rv = CreateFirstPart(); + if (NS_SUCCEEDED(rv)) { + rv = CreateSecondPart(); + if (NS_SUCCEEDED(rv)) rv = CreateThirdPart(); + } + + if (m_outputStream) { + m_outputStream->Flush(); + m_outputStream->Close(); + } + if (NS_FAILED(rv)) + m_file->Remove(false); + else + rv = SendMdnMsg(); + + return NS_OK; +} + +nsresult nsMsgMdnGenerator::CreateFirstPart() { + DEBUG_MDN("nsMsgMdnGenerator::CreateFirstPart"); + char *convbuf = nullptr, *tmpBuffer = nullptr; + char* parm = nullptr; + nsString firstPart1; + nsString firstPart2; + nsresult rv = NS_OK; + nsCOMPtr<nsIMsgCompUtils> compUtils; + + if (m_mimeSeparator.IsEmpty()) { + compUtils = do_GetService("@mozilla.org/messengercompose/computils;1", &rv); + NS_ENSURE_SUCCESS(rv, rv); + rv = compUtils->MimeMakeSeparator("mdn", getter_Copies(m_mimeSeparator)); + NS_ENSURE_SUCCESS(rv, rv); + } + if (m_mimeSeparator.IsEmpty()) return NS_ERROR_OUT_OF_MEMORY; + + tmpBuffer = (char*)PR_CALLOC(256); + + if (!tmpBuffer) return NS_ERROR_OUT_OF_MEMORY; + + PRExplodedTime now; + PR_ExplodeTime(PR_Now(), PR_LocalTimeParameters, &now); + + int gmtoffset = + (now.tm_params.tp_gmt_offset + now.tm_params.tp_dst_offset) / 60; + /* Use PR_FormatTimeUSEnglish() to format the date in US English format, + then figure out what our local GMT offset is, and append it (since + PR_FormatTimeUSEnglish() can't do that.) Generate four digit years as + per RFC 1123 (superseding RFC 822.) + */ + PR_FormatTimeUSEnglish(tmpBuffer, 100, "Date: %a, %d %b %Y %H:%M:%S ", &now); + + PR_snprintf(tmpBuffer + strlen(tmpBuffer), 100, "%c%02d%02d" CRLF, + (gmtoffset >= 0 ? '+' : '-'), + ((gmtoffset >= 0 ? gmtoffset : -gmtoffset) / 60), + ((gmtoffset >= 0 ? gmtoffset : -gmtoffset) % 60)); + + rv = WriteString(tmpBuffer); + PR_Free(tmpBuffer); + if (NS_FAILED(rv)) return rv; + + bool conformToStandard = false; + if (compUtils) compUtils->GetMsgMimeConformToStandard(&conformToStandard); + + nsString fullName; + m_identity->GetFullName(fullName); + + nsCString fullAddress; + // convert fullName to UTF8 before passing it to MakeMimeAddress + MakeMimeAddress(NS_ConvertUTF16toUTF8(fullName), m_email, fullAddress); + + convbuf = nsMsgI18NEncodeMimePartIIStr(fullAddress.get(), true, "UTF-8", 0, + conformToStandard); + + parm = PR_smprintf("From: %s" CRLF, convbuf ? convbuf : m_email.get()); + + rv = FormatStringFromName("MsgMdnMsgSentTo", NS_ConvertASCIItoUTF16(m_email), + firstPart1); + if (NS_FAILED(rv)) return rv; + + PUSH_N_FREE_STRING(parm); + + PR_Free(convbuf); + + if (compUtils) { + nsCString msgId; + rv = compUtils->MsgGenerateMessageId(m_identity, ""_ns, msgId); + NS_ENSURE_SUCCESS(rv, rv); + + tmpBuffer = PR_smprintf("Message-ID: %s" CRLF, msgId.get()); + PUSH_N_FREE_STRING(tmpBuffer); + } + + nsString receipt_string; + switch (m_disposeType) { + case nsIMsgMdnGenerator::eDisplayed: + rv = GetStringFromName("MdnDisplayedReceipt", receipt_string); + break; + case nsIMsgMdnGenerator::eDispatched: + rv = GetStringFromName("MdnDispatchedReceipt", receipt_string); + break; + case nsIMsgMdnGenerator::eProcessed: + rv = GetStringFromName("MdnProcessedReceipt", receipt_string); + break; + case nsIMsgMdnGenerator::eDeleted: + rv = GetStringFromName("MdnDeletedReceipt", receipt_string); + break; + case nsIMsgMdnGenerator::eDenied: + rv = GetStringFromName("MdnDeniedReceipt", receipt_string); + break; + case nsIMsgMdnGenerator::eFailed: + rv = GetStringFromName("MdnFailedReceipt", receipt_string); + break; + default: + rv = NS_ERROR_INVALID_ARG; + break; + } + + if (NS_FAILED(rv)) return rv; + + receipt_string.AppendLiteral(" - "); + + char* encodedReceiptString = + nsMsgI18NEncodeMimePartIIStr(NS_ConvertUTF16toUTF8(receipt_string).get(), + false, "UTF-8", 0, conformToStandard); + + nsCString subject; + m_headers->ExtractHeader(HEADER_SUBJECT, false, subject); + convbuf = nsMsgI18NEncodeMimePartIIStr( + subject.Length() ? subject.get() : "[no subject]", false, "UTF-8", 0, + conformToStandard); + tmpBuffer = PR_smprintf( + "Subject: %s%s" CRLF, encodedReceiptString, + (convbuf ? convbuf + : (subject.Length() ? subject.get() : "[no subject]"))); + + PUSH_N_FREE_STRING(tmpBuffer); + PR_Free(convbuf); + PR_Free(encodedReceiptString); + + convbuf = nsMsgI18NEncodeMimePartIIStr(m_dntRrt.get(), true, "UTF-8", 0, + conformToStandard); + tmpBuffer = PR_smprintf("To: %s" CRLF, convbuf ? convbuf : m_dntRrt.get()); + PUSH_N_FREE_STRING(tmpBuffer); + + PR_Free(convbuf); + + // *** This is not in the spec. I am adding this so we could do + // threading + m_headers->ExtractHeader(HEADER_MESSAGE_ID, false, m_messageId); + + if (!m_messageId.IsEmpty()) { + if (*m_messageId.get() == '<') + tmpBuffer = PR_smprintf("References: %s" CRLF, m_messageId.get()); + else + tmpBuffer = PR_smprintf("References: <%s>" CRLF, m_messageId.get()); + PUSH_N_FREE_STRING(tmpBuffer); + } + tmpBuffer = PR_smprintf("%s" CRLF, "MIME-Version: 1.0"); + PUSH_N_FREE_STRING(tmpBuffer); + + tmpBuffer = PR_smprintf( + "Content-Type: multipart/report; \ +report-type=disposition-notification;\r\n\tboundary=\"%s\"" CRLF CRLF, + m_mimeSeparator.get()); + PUSH_N_FREE_STRING(tmpBuffer); + + tmpBuffer = PR_smprintf("--%s" CRLF, m_mimeSeparator.get()); + PUSH_N_FREE_STRING(tmpBuffer); + + tmpBuffer = PR_smprintf("Content-Type: text/plain; charset=UTF-8" CRLF); + PUSH_N_FREE_STRING(tmpBuffer); + + tmpBuffer = + PR_smprintf("Content-Transfer-Encoding: %s" CRLF CRLF, ENCODING_8BIT); + PUSH_N_FREE_STRING(tmpBuffer); + + if (!firstPart1.IsEmpty()) { + tmpBuffer = + PR_smprintf("%s" CRLF CRLF, NS_ConvertUTF16toUTF8(firstPart1).get()); + PUSH_N_FREE_STRING(tmpBuffer); + } + + switch (m_disposeType) { + case nsIMsgMdnGenerator::eDisplayed: + rv = GetStringFromName("MsgMdnDisplayed", firstPart2); + break; + case nsIMsgMdnGenerator::eDispatched: + rv = GetStringFromName("MsgMdnDispatched", firstPart2); + break; + case nsIMsgMdnGenerator::eProcessed: + rv = GetStringFromName("MsgMdnProcessed", firstPart2); + break; + case nsIMsgMdnGenerator::eDeleted: + rv = GetStringFromName("MsgMdnDeleted", firstPart2); + break; + case nsIMsgMdnGenerator::eDenied: + rv = GetStringFromName("MsgMdnDenied", firstPart2); + break; + case nsIMsgMdnGenerator::eFailed: + rv = GetStringFromName("MsgMdnFailed", firstPart2); + break; + default: + rv = NS_ERROR_INVALID_ARG; + break; + } + + if (NS_FAILED(rv)) return rv; + + if (!firstPart2.IsEmpty()) { + tmpBuffer = + PR_smprintf("%s" CRLF CRLF, NS_ConvertUTF16toUTF8(firstPart2).get()); + PUSH_N_FREE_STRING(tmpBuffer); + } + + return rv; +} + +nsresult nsMsgMdnGenerator::CreateSecondPart() { + DEBUG_MDN("nsMsgMdnGenerator::CreateSecondPart"); + char* tmpBuffer = nullptr; + char* convbuf = nullptr; + nsresult rv = NS_OK; + nsCOMPtr<nsIMsgCompUtils> compUtils; + bool conformToStandard = false; + + tmpBuffer = PR_smprintf("--%s" CRLF, m_mimeSeparator.get()); + PUSH_N_FREE_STRING(tmpBuffer); + + tmpBuffer = PR_smprintf("%s" CRLF, + "Content-Type: message/disposition-notification; " + "name=\042MDNPart2.txt\042"); + PUSH_N_FREE_STRING(tmpBuffer); + + tmpBuffer = PR_smprintf("%s" CRLF, "Content-Disposition: inline"); + PUSH_N_FREE_STRING(tmpBuffer); + + tmpBuffer = + PR_smprintf("Content-Transfer-Encoding: %s" CRLF CRLF, ENCODING_7BIT); + PUSH_N_FREE_STRING(tmpBuffer); + + nsCOMPtr<nsIHttpProtocolHandler> pHTTPHandler = + do_GetService(NS_NETWORK_PROTOCOL_CONTRACTID_PREFIX "http", &rv); + if (NS_SUCCEEDED(rv) && pHTTPHandler) { + bool sendUserAgent = false; + nsCOMPtr<nsIPrefBranch> prefBranch( + do_GetService(NS_PREFSERVICE_CONTRACTID, &rv)); + if (NS_SUCCEEDED(rv) && prefBranch) { + prefBranch->GetBoolPref("mailnews.headers.sendUserAgent", &sendUserAgent); + } + + if (sendUserAgent) { + bool useMinimalUserAgent = false; + if (prefBranch) { + prefBranch->GetBoolPref("mailnews.headers.useMinimalUserAgent", + &useMinimalUserAgent); + } + if (useMinimalUserAgent) { + nsCOMPtr<nsIStringBundleService> bundleService = + mozilla::components::StringBundle::Service(); + if (bundleService) { + nsCOMPtr<nsIStringBundle> brandBundle; + rv = bundleService->CreateBundle( + "chrome://branding/locale/brand.properties", + getter_AddRefs(brandBundle)); + if (NS_SUCCEEDED(rv)) { + nsString brandName; + brandBundle->GetStringFromName("brandFullName", brandName); + if (!brandName.IsEmpty()) { + NS_ConvertUTF16toUTF8 ua8(brandName); + tmpBuffer = PR_smprintf("Reporting-UA: %s" CRLF, ua8.get()); + PUSH_N_FREE_STRING(tmpBuffer); + } + } + } + } else { + nsAutoCString userAgentString; + // Ignore error since we're testing the return value. + mozilla::Unused << pHTTPHandler->GetUserAgent(userAgentString); + + if (!userAgentString.IsEmpty()) { + // Prepend the product name with the dns name according to RFC 3798. + char hostName[256]; + PR_GetSystemInfo(PR_SI_HOSTNAME_UNTRUNCATED, hostName, + sizeof hostName); + if ((hostName[0] != '\0') && (strchr(hostName, '.') != NULL)) { + userAgentString.InsertLiteral("; ", 0); + userAgentString.Insert(nsDependentCString(hostName), 0); + } + + tmpBuffer = + PR_smprintf("Reporting-UA: %s" CRLF, userAgentString.get()); + PUSH_N_FREE_STRING(tmpBuffer); + } + } + } + } + + nsCString originalRecipient; + m_headers->ExtractHeader(HEADER_ORIGINAL_RECIPIENT, false, originalRecipient); + + if (!originalRecipient.IsEmpty()) { + tmpBuffer = + PR_smprintf("Original-Recipient: %s" CRLF, originalRecipient.get()); + PUSH_N_FREE_STRING(tmpBuffer); + } + + compUtils = do_GetService("@mozilla.org/messengercompose/computils;1", &rv); + if (compUtils) compUtils->GetMsgMimeConformToStandard(&conformToStandard); + + convbuf = nsMsgI18NEncodeMimePartIIStr(m_email.get(), true, "UTF-8", 0, + conformToStandard); + tmpBuffer = PR_smprintf("Final-Recipient: rfc822;%s" CRLF, + convbuf ? convbuf : m_email.get()); + PUSH_N_FREE_STRING(tmpBuffer); + + PR_Free(convbuf); + + if (*m_messageId.get() == '<') + tmpBuffer = PR_smprintf("Original-Message-ID: %s" CRLF, m_messageId.get()); + else + tmpBuffer = + PR_smprintf("Original-Message-ID: <%s>" CRLF, m_messageId.get()); + PUSH_N_FREE_STRING(tmpBuffer); + + tmpBuffer = + PR_smprintf("Disposition: %s/%s; %s" CRLF CRLF, + (m_autoAction ? "automatic-action" : "manual-action"), + (m_autoSend ? "MDN-sent-automatically" : "MDN-sent-manually"), + DispositionTypes[(int)m_disposeType]); + PUSH_N_FREE_STRING(tmpBuffer); + + return rv; +} + +nsresult nsMsgMdnGenerator::CreateThirdPart() { + DEBUG_MDN("nsMsgMdnGenerator::CreateThirdPart"); + char* tmpBuffer = nullptr; + nsresult rv = NS_OK; + + tmpBuffer = PR_smprintf("--%s" CRLF, m_mimeSeparator.get()); + PUSH_N_FREE_STRING(tmpBuffer); + + tmpBuffer = PR_smprintf( + "%s" CRLF, + "Content-Type: text/rfc822-headers; name=\042MDNPart3.txt\042"); + PUSH_N_FREE_STRING(tmpBuffer); + + tmpBuffer = PR_smprintf("%s" CRLF, "Content-Transfer-Encoding: 7bit"); + PUSH_N_FREE_STRING(tmpBuffer); + + tmpBuffer = PR_smprintf("%s" CRLF CRLF, "Content-Disposition: inline"); + PUSH_N_FREE_STRING(tmpBuffer); + + rv = OutputAllHeaders(); + + if (NS_FAILED(rv)) return rv; + + rv = WriteString(CRLF); + if (NS_FAILED(rv)) return rv; + + tmpBuffer = PR_smprintf("--%s--" CRLF, m_mimeSeparator.get()); + PUSH_N_FREE_STRING(tmpBuffer); + + return rv; +} + +nsresult nsMsgMdnGenerator::OutputAllHeaders() { + DEBUG_MDN("nsMsgMdnGenerator::OutputAllHeaders"); + nsCString all_headers; + int32_t all_headers_size = 0; + nsresult rv = NS_OK; + + rv = m_headers->GetAllHeaders(all_headers); + if (NS_FAILED(rv)) return rv; + all_headers_size = all_headers.Length(); + char *buf = (char*)all_headers.get(), + *buf_end = (char*)all_headers.get() + all_headers_size; + char *start = buf, *end = buf; + + while (buf < buf_end) { + switch (*buf) { + case 0: + if (*(buf + 1) == '\n') { + // *buf = '\r'; + end = buf; + } else if (*(buf + 1) == 0) { + // the case of message id + *buf = '>'; + } + break; + case '\r': + end = buf; + *buf = 0; + break; + case '\n': + if (buf > start && *(buf - 1) == 0) { + start = buf + 1; + end = start; + } else { + end = buf; + } + *buf = 0; + break; + default: + break; + } + buf++; + + if (end > start && *end == 0) { + // strip out private X-Mozilla-Status header & X-Mozilla-Draft-Info && + // envelope header + if (!PL_strncasecmp(start, X_MOZILLA_STATUS, X_MOZILLA_STATUS_LEN) || + !PL_strncasecmp(start, X_MOZILLA_DRAFT_INFO, + X_MOZILLA_DRAFT_INFO_LEN) || + !PL_strncasecmp(start, "From ", 5)) { + while (end < buf_end && (*end == '\n' || *end == '\r' || *end == 0)) + end++; + start = end; + } else { + NS_ASSERTION(*end == 0, "content of end should be null"); + rv = WriteString(start); + NS_ENSURE_SUCCESS(rv, rv); + rv = WriteString(CRLF); + NS_ENSURE_SUCCESS(rv, rv); + while (end < buf_end && (*end == '\n' || *end == '\r' || *end == 0)) + end++; + start = end; + } + buf = start; + } + } + return NS_OK; +} + +nsresult nsMsgMdnGenerator::SendMdnMsg() { + DEBUG_MDN("nsMsgMdnGenerator::SendMdnMsg"); + nsresult rv; + nsCOMPtr<nsISmtpService> smtpService = + do_GetService("@mozilla.org/messengercompose/smtp;1", &rv); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr<nsIURI> aUri; + nsCOMPtr<nsIRequest> aRequest; + nsCString identEmail; + m_identity->GetEmail(identEmail); + smtpService->SendMailMessage(m_file, m_dntRrt.get(), m_identity, + identEmail.get(), EmptyString(), this, nullptr, + nullptr, false, ""_ns, getter_AddRefs(aUri), + getter_AddRefs(aRequest)); + + return NS_OK; +} + +nsresult nsMsgMdnGenerator::WriteString(const char* str) { + NS_ENSURE_ARG(str); + uint32_t len = strlen(str); + uint32_t wLen = 0; + + return m_outputStream->Write(str, len, &wLen); +} + +nsresult nsMsgMdnGenerator::InitAndProcess(bool* needToAskUser) { + DEBUG_MDN("nsMsgMdnGenerator::InitAndProcess"); + nsresult rv = m_folder->GetServer(getter_AddRefs(m_server)); + nsCOMPtr<nsIMsgAccountManager> accountManager = + do_GetService("@mozilla.org/messenger/account-manager;1", &rv); + if (accountManager && m_server) { + if (!m_identity) { + // check if this is a message delivered to the global inbox, + // in which case we find the originating account's identity. + nsCString accountKey; + m_headers->ExtractHeader(HEADER_X_MOZILLA_ACCOUNT_KEY, false, accountKey); + nsCOMPtr<nsIMsgAccount> account; + if (!accountKey.IsEmpty()) + accountManager->GetAccount(accountKey, getter_AddRefs(account)); + if (account) account->GetIncomingServer(getter_AddRefs(m_server)); + + if (m_server) { + // Find the correct identity based on the "To:" and "Cc:" header + nsCString mailTo; + nsCString mailCC; + m_headers->ExtractHeader(HEADER_TO, true, mailTo); + m_headers->ExtractHeader(HEADER_CC, true, mailCC); + nsTArray<RefPtr<nsIMsgIdentity>> servIdentities; + accountManager->GetIdentitiesForServer(m_server, servIdentities); + + // First check in the "To:" header + for (auto ident : servIdentities) { + nsCString identEmail; + ident->GetEmail(identEmail); + if (!mailTo.IsEmpty() && !identEmail.IsEmpty() && + FindInReadable(identEmail, mailTo, + nsCaseInsensitiveCStringComparator)) { + m_identity = ident; + break; + } + } + // If no match, check the "Cc:" header + if (!m_identity) { + for (auto ident : servIdentities) { + nsCString identEmail; + ident->GetEmail(identEmail); + if (!mailCC.IsEmpty() && !identEmail.IsEmpty() && + FindInReadable(identEmail, mailCC, + nsCaseInsensitiveCStringComparator)) { + m_identity = ident; + break; + } + } + } + + // If still no match, use the first identity + if (!m_identity) { + rv = accountManager->GetFirstIdentityForServer( + m_server, getter_AddRefs(m_identity)); + } + } + } + NS_ENSURE_SUCCESS(rv, rv); + + if (m_identity) { + bool useCustomPrefs = false; + m_identity->GetBoolAttribute("use_custom_prefs", &useCustomPrefs); + if (useCustomPrefs) { + bool bVal = false; + m_server->GetBoolValue("mdn_report_enabled", &bVal); + m_mdnEnabled = bVal; + m_server->GetIntValue("mdn_not_in_to_cc", &m_notInToCcOp); + m_server->GetIntValue("mdn_outside_domain", &m_outsideDomainOp); + m_server->GetIntValue("mdn_other", &m_otherOp); + } else { + bool bVal = false; + + nsCOMPtr<nsIPrefBranch> prefBranch( + do_GetService(NS_PREFSERVICE_CONTRACTID, &rv)); + if (NS_FAILED(rv)) return rv; + + if (prefBranch) { + prefBranch->GetBoolPref("mail.mdn.report.enabled", &bVal); + m_mdnEnabled = bVal; + prefBranch->GetIntPref("mail.mdn.report.not_in_to_cc", + &m_notInToCcOp); + prefBranch->GetIntPref("mail.mdn.report.outside_domain", + &m_outsideDomainOp); + prefBranch->GetIntPref("mail.mdn.report.other", &m_otherOp); + } + } + } + } + + if (m_mdnEnabled) { + m_headers->ExtractHeader(HEADER_DISPOSITION_NOTIFICATION_TO, false, + m_dntRrt); + if (m_dntRrt.IsEmpty()) + m_headers->ExtractHeader(HEADER_RETURN_RECEIPT_TO, false, m_dntRrt); + if (!m_dntRrt.IsEmpty() && ProcessSendMode() && ValidateReturnPath()) { + if (!m_autoSend) { + *needToAskUser = true; + rv = NS_OK; + } else { + *needToAskUser = false; + rv = UserAgreed(); + } + } + } + return rv; +} + +NS_IMETHODIMP nsMsgMdnGenerator::Process(EDisposeType type, + nsIMsgWindow* aWindow, + nsIMsgFolder* folder, nsMsgKey key, + nsIMimeHeaders* headers, + bool autoAction, bool* _retval) { + DEBUG_MDN("nsMsgMdnGenerator::Process"); + NS_ENSURE_ARG_POINTER(folder); + NS_ENSURE_ARG_POINTER(headers); + NS_ENSURE_ARG_POINTER(aWindow); + NS_ENSURE_TRUE(key != nsMsgKey_None, NS_ERROR_INVALID_ARG); + m_disposeType = type; + m_autoAction = autoAction; + m_window = aWindow; + m_folder = folder; + m_headers = headers; + m_key = key; + + mozilla::DebugOnly<nsresult> rv = InitAndProcess(_retval); + NS_ASSERTION(NS_SUCCEEDED(rv), "InitAndProcess failed"); + return NS_OK; +} + +NS_IMETHODIMP nsMsgMdnGenerator::UserAgreed() { + DEBUG_MDN("nsMsgMdnGenerator::UserAgreed"); + (void)NoteMDNRequestHandled(); + return CreateMdnMsg(); +} + +NS_IMETHODIMP nsMsgMdnGenerator::UserDeclined() { + DEBUG_MDN("nsMsgMdnGenerator::UserDeclined"); + return NoteMDNRequestHandled(); +} + +/** + * Set/clear flags appropriately so we won't ask user again about MDN + * request for this message. + */ +nsresult nsMsgMdnGenerator::NoteMDNRequestHandled() { + nsresult rv = StoreMDNSentFlag(m_folder, m_key); + NS_ASSERTION(NS_SUCCEEDED(rv), "StoreMDNSentFlag failed"); + rv = ClearMDNNeededFlag(m_folder, m_key); + NS_ASSERTION(NS_SUCCEEDED(rv), "ClearMDNNeededFlag failed"); + return rv; +} + +NS_IMETHODIMP nsMsgMdnGenerator::OnStartRunningUrl(nsIURI* url) { + DEBUG_MDN("nsMsgMdnGenerator::OnStartRunningUrl"); + return NS_OK; +} + +NS_IMETHODIMP nsMsgMdnGenerator::OnStopRunningUrl(nsIURI* url, + nsresult aExitCode) { + nsresult rv; + + DEBUG_MDN("nsMsgMdnGenerator::OnStopRunningUrl"); + if (m_file) m_file->Remove(false); + + if (NS_SUCCEEDED(aExitCode)) return NS_OK; + + const char* exitString; + + switch (aExitCode) { + case NS_ERROR_UNKNOWN_HOST: + case NS_ERROR_UNKNOWN_PROXY_HOST: + exitString = "smtpSendFailedUnknownServer"; + break; + case NS_ERROR_CONNECTION_REFUSED: + case NS_ERROR_PROXY_CONNECTION_REFUSED: + exitString = "smtpSendRequestRefused"; + break; + case NS_ERROR_NET_INTERRUPT: + case NS_ERROR_ABORT: // we have no proper string for error code + // NS_ERROR_ABORT in compose bundle + exitString = "smtpSendInterrupted"; + break; + case NS_ERROR_NET_TIMEOUT: + case NS_ERROR_NET_RESET: + exitString = "smtpSendTimeout"; + break; + default: + exitString = errorStringNameForErrorCode(aExitCode); + break; + } + + nsCOMPtr<nsISmtpService> smtpService( + do_GetService("@mozilla.org/messengercompose/smtp;1", &rv)); + NS_ENSURE_SUCCESS(rv, rv); + + // Get the smtp hostname and format the string. + nsCString smtpHostName; + nsCOMPtr<nsISmtpServer> smtpServer; + rv = smtpService->GetServerByIdentity(m_identity, getter_AddRefs(smtpServer)); + if (NS_SUCCEEDED(rv)) smtpServer->GetHostname(smtpHostName); + + AutoTArray<nsString, 1> params; + CopyASCIItoUTF16(smtpHostName, *params.AppendElement()); + + nsCOMPtr<nsIStringBundle> bundle; + nsCOMPtr<nsIStringBundleService> bundleService = + mozilla::components::StringBundle::Service(); + NS_ENSURE_TRUE(bundleService, NS_ERROR_UNEXPECTED); + + rv = bundleService->CreateBundle( + "chrome://messenger/locale/messengercompose/composeMsgs.properties", + getter_AddRefs(bundle)); + NS_ENSURE_SUCCESS(rv, rv); + + nsString failed_msg, dialogTitle; + + bundle->FormatStringFromName(exitString, params, failed_msg); + bundle->GetStringFromName("sendMessageErrorTitle", dialogTitle); + + nsCOMPtr<mozIDOMWindowProxy> domWindow; + m_window->GetDomWindow(getter_AddRefs(domWindow)); + + nsCOMPtr<nsIPromptService> dlgService( + do_GetService(NS_PROMPTSERVICE_CONTRACTID, &rv)); + NS_ENSURE_SUCCESS(rv, rv); + + dlgService->Alert(domWindow, dialogTitle.get(), failed_msg.get()); + + return NS_OK; +} diff --git a/comm/mailnews/extensions/mdn/nsMsgMdnGenerator.h b/comm/mailnews/extensions/mdn/nsMsgMdnGenerator.h new file mode 100644 index 0000000000..74eea8bcad --- /dev/null +++ b/comm/mailnews/extensions/mdn/nsMsgMdnGenerator.h @@ -0,0 +1,86 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef _nsMsgMdnGenerator_H_ +#define _nsMsgMdnGenerator_H_ + +#include "nsIMsgMdnGenerator.h" +#include "nsCOMPtr.h" +#include "nsIUrlListener.h" +#include "nsIMsgIncomingServer.h" +#include "nsIOutputStream.h" +#include "nsIFile.h" +#include "nsIMsgIdentity.h" +#include "nsIMsgWindow.h" +#include "nsIMimeHeaders.h" +#include "nsString.h" +#include "MailNewsTypes2.h" + +#define eNeverSendOp ((int32_t)0) +#define eAutoSendOp ((int32_t)1) +#define eAskMeOp ((int32_t)2) +#define eDeniedOp ((int32_t)3) + +class nsMsgMdnGenerator : public nsIMsgMdnGenerator, public nsIUrlListener { + public: + NS_DECL_ISUPPORTS + NS_DECL_NSIMSGMDNGENERATOR + NS_DECL_NSIURLLISTENER + + nsMsgMdnGenerator(); + + private: + virtual ~nsMsgMdnGenerator(); + + // Sanity Check methods + bool ProcessSendMode(); // must called prior ValidateReturnPath + bool ValidateReturnPath(); + bool NotInToOrCc(); + bool MailAddrMatch(const char* addr1, const char* addr2); + + nsresult StoreMDNSentFlag(nsIMsgFolder* folder, nsMsgKey key); + nsresult ClearMDNNeededFlag(nsIMsgFolder* folder, nsMsgKey key); + nsresult NoteMDNRequestHandled(); + + nsresult CreateMdnMsg(); + nsresult CreateFirstPart(); + nsresult CreateSecondPart(); + nsresult CreateThirdPart(); + nsresult SendMdnMsg(); + + // string bundle helper methods + nsresult GetStringFromName(const char* aName, nsAString& aResultString); + nsresult FormatStringFromName(const char* aName, const nsString& aString, + nsAString& aResultString); + + // other helper methods + nsresult InitAndProcess(bool* needToAskUser); + nsresult OutputAllHeaders(); + nsresult WriteString(const char* str); + + private: + EDisposeType m_disposeType; + nsCOMPtr<nsIMsgWindow> m_window; + nsCOMPtr<nsIOutputStream> m_outputStream; + nsCOMPtr<nsIFile> m_file; + nsCOMPtr<nsIMsgIdentity> m_identity; + nsMsgKey m_key; + nsCString m_email; + nsCString m_mimeSeparator; + nsCString m_messageId; + nsCOMPtr<nsIMsgFolder> m_folder; + nsCOMPtr<nsIMsgIncomingServer> m_server; + nsCOMPtr<nsIMimeHeaders> m_headers; + nsCString m_dntRrt; + int32_t m_notInToCcOp; + int32_t m_outsideDomainOp; + int32_t m_otherOp; + bool m_reallySendMdn; + bool m_autoSend; + bool m_autoAction; + bool m_mdnEnabled; +}; + +#endif // _nsMsgMdnGenerator_H_ diff --git a/comm/mailnews/extensions/mdn/test/unit/head_mdn.js b/comm/mailnews/extensions/mdn/test/unit/head_mdn.js new file mode 100644 index 0000000000..0735341e30 --- /dev/null +++ b/comm/mailnews/extensions/mdn/test/unit/head_mdn.js @@ -0,0 +1,18 @@ +var { MailServices } = ChromeUtils.import( + "resource:///modules/MailServices.jsm" +); +var { mailTestUtils } = ChromeUtils.import( + "resource://testing-common/mailnews/MailTestUtils.jsm" +); +var { localAccountUtils } = ChromeUtils.import( + "resource://testing-common/mailnews/LocalAccountUtils.jsm" +); + +var CC = Components.Constructor; + +// Ensure the profile directory is set up +do_get_profile(); + +registerCleanupFunction(function () { + load("../../../../../mailnews/resources/mailShutdown.js"); +}); diff --git a/comm/mailnews/extensions/mdn/test/unit/test_askuser.js b/comm/mailnews/extensions/mdn/test/unit/test_askuser.js new file mode 100644 index 0000000000..b30ab49b0f --- /dev/null +++ b/comm/mailnews/extensions/mdn/test/unit/test_askuser.js @@ -0,0 +1,67 @@ +localAccountUtils.loadLocalMailAccount(); + +var localAccount = MailServices.accounts.FindAccountForServer( + localAccountUtils.incomingServer +); +var identity = MailServices.accounts.createIdentity(); +identity.email = "bob@t2.example.net"; +localAccount.addIdentity(identity); +localAccount.defaultIdentity = identity; + +function run_test() { + var headers = + "from: alice@t1.example.com\r\n" + + "to: bob@t2.example.net\r\n" + + "return-path: alice@t1.example.com\r\n" + + "Disposition-Notification-To: alice@t1.example.com\r\n"; + + let mimeHdr = Cc["@mozilla.org/messenger/mimeheaders;1"].createInstance( + Ci.nsIMimeHeaders + ); + mimeHdr.initialize(headers); + let receivedHeader = mimeHdr.extractHeader("To", false); + dump(receivedHeader + "\n"); + + localAccountUtils.inboxFolder.QueryInterface(Ci.nsIMsgLocalMailFolder); + localAccountUtils.inboxFolder.addMessage( + "From \r\n" + headers + "\r\nhello\r\n" + ); + // Need to setup some prefs + Services.prefs.setBoolPref("mail.mdn.report.enabled", true); + Services.prefs.setIntPref("mail.mdn.report.not_in_to_cc", 2); + Services.prefs.setIntPref("mail.mdn.report.other", 2); + Services.prefs.setIntPref("mail.mdn.report.outside_domain", 2); + + var msgFolder = localAccountUtils.inboxFolder; + + var msgWindow = {}; + + var msgHdr = mailTestUtils.firstMsgHdr(localAccountUtils.inboxFolder); + + // Everything looks good so far, let's generate the MDN response. + var mdnGenerator = Cc[ + "@mozilla.org/messenger-mdn/generator;1" + ].createInstance(Ci.nsIMsgMdnGenerator); + + Services.prefs.setIntPref("mail.mdn.report.outside_domain", 1); + var askUser = mdnGenerator.process( + Ci.nsIMsgMdnGenerator.eDisplayed, + msgWindow, + msgFolder, + msgHdr.messageKey, + mimeHdr, + false + ); + Assert.ok(!askUser); + + Services.prefs.setIntPref("mail.mdn.report.outside_domain", 2); + askUser = mdnGenerator.process( + Ci.nsIMsgMdnGenerator.eDisplayed, + msgWindow, + msgFolder, + msgHdr.messageKey, + mimeHdr, + false + ); + Assert.ok(askUser); +} diff --git a/comm/mailnews/extensions/mdn/test/unit/test_mdnFlags.js b/comm/mailnews/extensions/mdn/test/unit/test_mdnFlags.js new file mode 100644 index 0000000000..4d6094e89d --- /dev/null +++ b/comm/mailnews/extensions/mdn/test/unit/test_mdnFlags.js @@ -0,0 +1,60 @@ +/** + * This tests that setting mdn flags works correctly, so that we don't + * reprompt when the user re-selects a message. + */ + +localAccountUtils.loadLocalMailAccount(); + +var localAccount = MailServices.accounts.FindAccountForServer( + localAccountUtils.incomingServer +); +var identity = MailServices.accounts.createIdentity(); +identity.email = "bob@t2.example.net"; +localAccount.addIdentity(identity); +localAccount.defaultIdentity = identity; + +function run_test() { + var headers = + "from: alice@t1.example.com\r\n" + + "to: bob@t2.example.net\r\n" + + "return-path: alice@t1.example.com\r\n" + + "Disposition-Notification-To: alice@t1.example.com\r\n"; + + let mimeHdr = Cc["@mozilla.org/messenger/mimeheaders;1"].createInstance( + Ci.nsIMimeHeaders + ); + mimeHdr.initialize(headers); + mimeHdr.extractHeader("To", false); + + localAccountUtils.inboxFolder.QueryInterface(Ci.nsIMsgLocalMailFolder); + localAccountUtils.inboxFolder.addMessage( + "From \r\n" + headers + "\r\nhello\r\n" + ); + // Need to setup some prefs + Services.prefs.setBoolPref("mail.mdn.report.enabled", true); + Services.prefs.setIntPref("mail.mdn.report.not_in_to_cc", 2); + Services.prefs.setIntPref("mail.mdn.report.other", 2); + Services.prefs.setIntPref("mail.mdn.report.outside_domain", 2); + + var msgFolder = localAccountUtils.inboxFolder; + + var msgWindow = {}; + + var msgHdr = mailTestUtils.firstMsgHdr(localAccountUtils.inboxFolder); + + // Everything looks good so far, let's generate the MDN response. + var mdnGenerator = Cc[ + "@mozilla.org/messenger-mdn/generator;1" + ].createInstance(Ci.nsIMsgMdnGenerator); + mdnGenerator.process( + Ci.nsIMsgMdnGenerator.eDisplayed, + msgWindow, + msgFolder, + msgHdr.messageKey, + mimeHdr, + false + ); + mdnGenerator.userDeclined(); + Assert.notEqual(msgHdr.flags & Ci.nsMsgMessageFlags.MDNReportSent, 0); + Assert.equal(msgHdr.flags & Ci.nsMsgMessageFlags.MDNReportNeeded, 0); +} diff --git a/comm/mailnews/extensions/mdn/test/unit/xpcshell.ini b/comm/mailnews/extensions/mdn/test/unit/xpcshell.ini new file mode 100644 index 0000000000..243145b676 --- /dev/null +++ b/comm/mailnews/extensions/mdn/test/unit/xpcshell.ini @@ -0,0 +1,6 @@ +[DEFAULT] +head = head_mdn.js +tail = + +[test_askuser.js] +[test_mdnFlags.js] diff --git a/comm/mailnews/extensions/moz.build b/comm/mailnews/extensions/moz.build new file mode 100644 index 0000000000..2f56ec9d2e --- /dev/null +++ b/comm/mailnews/extensions/moz.build @@ -0,0 +1,15 @@ +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# These extensions are not optional. +DIRS += [ + "mdn", + "mailviews", + "bayesian-spam-filter", + "offline-startup", + "newsblog", + "fts3", + "smime", +] diff --git a/comm/mailnews/extensions/newsblog/.eslintrc.js b/comm/mailnews/extensions/newsblog/.eslintrc.js new file mode 100644 index 0000000000..8254a84aaa --- /dev/null +++ b/comm/mailnews/extensions/newsblog/.eslintrc.js @@ -0,0 +1,18 @@ +"use strict"; + +module.exports = { + rules: { + // Enforce valid JSDoc comments. + "valid-jsdoc": [ + "error", + { + prefer: { return: "returns" }, + preferType: { + map: "Map", + set: "Set", + date: "Date", + }, + }, + ], + }, +}; diff --git a/comm/mailnews/extensions/newsblog/Feed.jsm b/comm/mailnews/extensions/newsblog/Feed.jsm new file mode 100644 index 0000000000..8b899308d5 --- /dev/null +++ b/comm/mailnews/extensions/newsblog/Feed.jsm @@ -0,0 +1,700 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const EXPORTED_SYMBOLS = ["Feed"]; + +const lazy = {}; + +ChromeUtils.defineModuleGetter( + lazy, + "FeedParser", + "resource:///modules/FeedParser.jsm" +); +ChromeUtils.defineModuleGetter( + lazy, + "FeedUtils", + "resource:///modules/FeedUtils.jsm" +); + +// Cache for all of the feeds currently being downloaded, indexed by URL, +// so the load event listener can access the Feed objects after it finishes +// downloading the feed. +var FeedCache = { + mFeeds: {}, + + putFeed(aFeed) { + this.mFeeds[this.normalizeHost(aFeed.url)] = aFeed; + }, + + getFeed(aUrl) { + let index = this.normalizeHost(aUrl); + if (index in this.mFeeds) { + return this.mFeeds[index]; + } + + return null; + }, + + removeFeed(aUrl) { + let index = this.normalizeHost(aUrl); + if (index in this.mFeeds) { + delete this.mFeeds[index]; + } + }, + + normalizeHost(aUrl) { + try { + let normalizedUrl = Services.io.newURI(aUrl); + let newHost = normalizedUrl.host.toLowerCase(); + normalizedUrl = normalizedUrl.mutate().setHost(newHost).finalize(); + return normalizedUrl.spec; + } catch (ex) { + return aUrl; + } + }, +}; + +/** + * A Feed object. If aFolder is the account root folder, a new subfolder + * for the feed url is created otherwise the url will be subscribed to the + * existing aFolder, upon successful download() completion. + * + * @class + * @param {string} aFeedUrl - feed url. + * @param {nsIMsgFolder} aFolder - folder containing or to contain the feed + * subscription. + */ +function Feed(aFeedUrl, aFolder) { + this.url = aFeedUrl; + this.server = aFolder.server; + if (!aFolder.isServer) { + this.mFolder = aFolder; + } +} + +Feed.prototype = { + url: null, + description: null, + author: null, + request: null, + server: null, + downloadCallback: null, + resource: null, + itemsToStore: [], + itemsStored: 0, + fileSize: 0, + mFolder: null, + mInvalidFeed: false, + mFeedType: null, + mLastModified: null, + + get folder() { + return this.mFolder; + }, + + set folder(aFolder) { + this.mFolder = aFolder; + }, + + get name() { + // Used for the feed's title in Subscribe dialog and opml export. + let name = this.title || this.description || this.url; + /* eslint-disable-next-line no-control-regex */ + return name.replace(/[\n\r\t]+/g, " ").replace(/[\x00-\x1F]+/g, ""); + }, + + get folderName() { + if (this.mFolderName) { + return this.mFolderName; + } + + // Get a unique sanitized name. Use title or description as a base; + // these are mandatory by spec. Length of 80 is plenty. + let folderName = (this.title || this.description || "").substr(0, 80); + let defaultName = + lazy.FeedUtils.strings.GetStringFromName("ImportFeedsNew"); + return (this.mFolderName = lazy.FeedUtils.getSanitizedFolderName( + this.server.rootMsgFolder, + folderName, + defaultName, + true + )); + }, + + download(aParseItems, aCallback) { + // May be null. + this.downloadCallback = aCallback; + + // Whether or not to parse items when downloading and parsing the feed. + // Defaults to true, but setting to false is useful for obtaining + // just the title of the feed when the user subscribes to it. + this.parseItems = aParseItems == null || aParseItems; + + // Before we do anything, make sure the url is an http url. This is just + // a sanity check so we don't try opening mailto urls, imap urls, etc. that + // the user may have tried to subscribe to as an rss feed. + if (!lazy.FeedUtils.isValidScheme(this.url)) { + // Simulate an invalid feed error. + lazy.FeedUtils.log.info( + "Feed.download: invalid protocol for - " + this.url + ); + this.onParseError(this); + return; + } + + // Before we try to download the feed, make sure we aren't already + // processing the feed by looking up the url in our feed cache. + if (FeedCache.getFeed(this.url)) { + if (this.downloadCallback) { + this.downloadCallback.downloaded( + this, + lazy.FeedUtils.kNewsBlogFeedIsBusy + ); + } + + // Return, the feed is already in use. + return; + } + + if (Services.io.offline) { + // If offline and don't want to go online, just add the feed subscription; + // it can be verified later (the folder name will be the url if not adding + // to an existing folder). Only for subscribe actions; passive biff and + // active get new messages are handled prior to getting here. + let win = Services.wm.getMostRecentWindow("mail:3pane"); + if (!win.MailOfflineMgr.getNewMail()) { + this.storeNextItem(); + return; + } + } + + this.request = new XMLHttpRequest(); + // Must set onProgress before calling open. + this.request.onprogress = this.onProgress; + this.request.open("GET", this.url, true); + this.request.channel.loadFlags |= + Ci.nsIRequest.LOAD_BYPASS_CACHE | Ci.nsIRequest.INHIBIT_CACHING; + + // Some servers, if sent If-Modified-Since, will send 304 if subsequently + // not sent If-Modified-Since, as in the case of an unsubscribe and new + // subscribe. Send start of century date to force a download; some servers + // will 304 on older dates (such as epoch 1970). + let lastModified = this.lastModified || "Sat, 01 Jan 2000 00:00:00 GMT"; + this.request.setRequestHeader("If-Modified-Since", lastModified); + + // Only order what you're going to eat... + this.request.responseType = "document"; + this.request.overrideMimeType("text/xml"); + this.request.setRequestHeader("Accept", lazy.FeedUtils.REQUEST_ACCEPT); + this.request.timeout = lazy.FeedUtils.REQUEST_TIMEOUT; + this.request.onload = this.onDownloaded; + this.request.onreadystatechange = this.onReadyStateChange; + this.request.onerror = this.onDownloadError; + this.request.ontimeout = this.onDownloadError; + FeedCache.putFeed(this); + this.request.send(null); + }, + + onReadyStateChange(aEvent) { + // Once a server responds with data, reset the timeout to allow potentially + // large files to complete the download. + let request = aEvent.target; + if (request.timeout && request.readyState == request.LOADING) { + request.timeout = 0; + } + }, + + onDownloaded(aEvent) { + let request = aEvent.target; + let isHttp = request.channel.originalURI.scheme.startsWith("http"); + let url = request.channel.originalURI.spec; + if (isHttp && (request.status < 200 || request.status >= 300)) { + Feed.prototype.onDownloadError(aEvent); + return; + } + + lazy.FeedUtils.log.debug( + "Feed.onDownloaded: got a download, fileSize:url - " + + aEvent.loaded + + " : " + + url + ); + let feed = FeedCache.getFeed(url); + if (!feed) { + throw new Error( + "Feed.onDownloaded: error - couldn't retrieve feed from cache" + ); + } + + // If the server sends a Last-Modified header, store the property on the + // feed so we can use it when making future requests, to avoid downloading + // and parsing feeds that have not changed. Don't update if merely checking + // the url, as for subscribe move/copy, as a subsequent refresh may get a 304. + // Save the response and persist it only upon successful completion of the + // refresh cycle (i.e. not if the request is cancelled). + let lastModifiedHeader = request.getResponseHeader("Last-Modified"); + feed.mLastModified = + lastModifiedHeader && feed.parseItems ? lastModifiedHeader : null; + + feed.fileSize = aEvent.loaded; + + // The download callback is called asynchronously when parse() is done. + feed.parse(); + }, + + onProgress(aEvent) { + let request = aEvent.target; + let url = request.channel.originalURI.spec; + let feed = FeedCache.getFeed(url); + + if (feed.downloadCallback) { + feed.downloadCallback.onProgress( + feed, + aEvent.loaded, + aEvent.total, + aEvent.lengthComputable + ); + } + }, + + onDownloadError(aEvent) { + let request = aEvent.target; + let url = request.channel.originalURI.spec; + let feed = FeedCache.getFeed(url); + if (feed.downloadCallback) { + // Generic network or 'not found' error initially. + let error = lazy.FeedUtils.kNewsBlogRequestFailure; + // Certain errors should disable the feed. + let disable = false; + + if (request.status == 304) { + // If the http status code is 304, the feed has not been modified + // since we last downloaded it and does not need to be parsed. + error = lazy.FeedUtils.kNewsBlogNoNewItems; + } else { + let [errType, errName] = + lazy.FeedUtils.createTCPErrorFromFailedXHR(request); + lazy.FeedUtils.log.info( + "Feed.onDownloaded: request errType:errName:statusCode - " + + errType + + ":" + + errName + + ":" + + request.status + ); + if (errType == "SecurityCertificate") { + // This is the code for nsINSSErrorsService.ERROR_CLASS_BAD_CERT + // overridable security certificate errors. + error = lazy.FeedUtils.kNewsBlogBadCertError; + } + + if (request.status == 401 || request.status == 403) { + // Unauthorized or Forbidden. + error = lazy.FeedUtils.kNewsBlogNoAuthError; + } + + if ( + request.status != 0 || + error == lazy.FeedUtils.kNewsBlogBadCertError || + errName == "DomainNotFoundError" + ) { + disable = true; + } + } + + feed.downloadCallback.downloaded(feed, error, disable); + } + + FeedCache.removeFeed(url); + }, + + onParseError(aFeed) { + if (!aFeed) { + return; + } + + aFeed.mInvalidFeed = true; + if (aFeed.downloadCallback) { + aFeed.downloadCallback.downloaded( + aFeed, + lazy.FeedUtils.kNewsBlogInvalidFeed, + true + ); + } + + FeedCache.removeFeed(aFeed.url); + }, + + onUrlChange(aFeed, aOldUrl) { + if (!aFeed) { + return; + } + + // Simulate a cancel after a url update; next cycle will check the new url. + aFeed.mInvalidFeed = true; + if (aFeed.downloadCallback) { + aFeed.downloadCallback.downloaded(aFeed, lazy.FeedUtils.kNewsBlogCancel); + } + + FeedCache.removeFeed(aOldUrl); + }, + + // nsIUrlListener methods for getDatabaseWithReparse(). + OnStartRunningUrl(aUrl) {}, + OnStopRunningUrl(aUrl, aExitCode) { + if (Components.isSuccessCode(aExitCode)) { + lazy.FeedUtils.log.debug( + "Feed.OnStopRunningUrl: rebuilt msgDatabase for " + + this.folder.name + + " - " + + this.folder.filePath.path + ); + } else { + lazy.FeedUtils.log.error( + "Feed.OnStopRunningUrl: rebuild msgDatabase failed, " + + "error " + + aExitCode + + ", for " + + this.folder.name + + " - " + + this.folder.filePath.path + ); + } + // Continue. + this.storeNextItem(); + }, + + get title() { + return lazy.FeedUtils.getSubscriptionAttr( + this.url, + this.server, + "title", + "" + ); + }, + + set title(aNewTitle) { + if (!aNewTitle) { + return; + } + lazy.FeedUtils.setSubscriptionAttr( + this.url, + this.server, + "title", + aNewTitle + ); + }, + + get lastModified() { + return lazy.FeedUtils.getSubscriptionAttr( + this.url, + this.server, + "lastModified", + "" + ); + }, + + set lastModified(aLastModified) { + lazy.FeedUtils.setSubscriptionAttr( + this.url, + this.server, + "lastModified", + aLastModified + ); + }, + + get quickMode() { + let defaultValue = this.server.getBoolValue("quickMode"); + return lazy.FeedUtils.getSubscriptionAttr( + this.url, + this.server, + "quickMode", + defaultValue + ); + }, + + set quickMode(aNewQuickMode) { + lazy.FeedUtils.setSubscriptionAttr( + this.url, + this.server, + "quickMode", + aNewQuickMode + ); + }, + + get options() { + let options = lazy.FeedUtils.getSubscriptionAttr( + this.url, + this.server, + "options", + null + ); + if (options && options.version == lazy.FeedUtils._optionsDefault.version) { + return options; + } + + let newOptions = lazy.FeedUtils.newOptions(options); + this.options = newOptions; + return newOptions; + }, + + set options(aOptions) { + let newOptions = aOptions ? aOptions : lazy.FeedUtils.optionsTemplate; + lazy.FeedUtils.setSubscriptionAttr( + this.url, + this.server, + "options", + newOptions + ); + }, + + get link() { + return lazy.FeedUtils.getSubscriptionAttr( + this.url, + this.server, + "link", + "" + ); + }, + + set link(aNewLink) { + if (!aNewLink) { + return; + } + lazy.FeedUtils.setSubscriptionAttr(this.url, this.server, "link", aNewLink); + }, + + parse() { + // Create a feed parser which will parse the feed. + let parser = new lazy.FeedParser(); + this.itemsToStore = parser.parseFeed(this, this.request.responseXML); + parser = null; + + if (this.mInvalidFeed) { + this.request = null; + this.mInvalidFeed = false; + return; + } + + this.itemsToStoreIndex = 0; + this.itemsStored = 0; + + // At this point, if we have items to potentially store and an existing + // folder, ensure the folder's msgDatabase is openable for new message + // processing. If not, reparse with an async nsIUrlListener |this| to + // continue once the reparse is complete. + if ( + this.itemsToStore.length > 0 && + this.folder && + !lazy.FeedUtils.isMsgDatabaseOpenable(this.folder, true, this) + ) { + return; + } + + // We have an msgDatabase; storeNextItem() will iterate through the parsed + // items, storing each one. + this.storeNextItem(); + }, + + /** + * Clear the 'valid' field of all feeditems associated with this feed. + * + * @returns {void} + */ + invalidateItems() { + let ds = lazy.FeedUtils.getItemsDS(this.server); + for (let id in ds.data) { + let item = ds.data[id]; + if (item.feedURLs.includes(this.url)) { + item.valid = false; + lazy.FeedUtils.log.trace("Feed.invalidateItems: item - " + id); + } + } + ds.saveSoon(); + }, + + /** + * Discards invalid items (in the feed item store) associated with the + * feed. There's a delay - invalid items are kept around for a set time + * before being purged. + * + * @param {Boolean} aDeleteFeed - is the feed being deleted (bypasses + * the delay time). + * @returns {void} + */ + removeInvalidItems(aDeleteFeed) { + let ds = lazy.FeedUtils.getItemsDS(this.server); + lazy.FeedUtils.log.debug("Feed.removeInvalidItems: for url - " + this.url); + + let currentTime = new Date().getTime(); + for (let id in ds.data) { + let item = ds.data[id]; + // skip valid items and ones not part of this feed. + if (!item.feedURLs.includes(this.url) || item.valid) { + continue; + } + let lastSeenTime = item.lastSeenTime || 0; + + if ( + currentTime - lastSeenTime < lazy.FeedUtils.INVALID_ITEM_PURGE_DELAY && + !aDeleteFeed + ) { + // Don't immediately purge items in active feeds; do so for deleted feeds. + continue; + } + + lazy.FeedUtils.log.trace("Feed.removeInvalidItems: item - " + id); + // Detach the item from this feed (it could be shared by multiple feeds). + item.feedURLs = item.feedURLs.filter(url => url != this.url); + if (item.feedURLs.length > 0) { + lazy.FeedUtils.log.debug( + "Feed.removeInvalidItems: " + + id + + " is from more than one feed; only the reference to" + + " this feed removed" + ); + } else { + delete ds.data[id]; + } + } + ds.saveSoon(); + }, + + createFolder() { + if (this.folder) { + return; + } + + try { + this.folder = this.server.rootMsgFolder + .QueryInterface(Ci.nsIMsgLocalMailFolder) + .createLocalSubfolder(this.folderName); + } catch (ex) { + // An error creating. + lazy.FeedUtils.log.info( + "Feed.createFolder: error creating folder - '" + + this.folderName + + "' in parent folder " + + this.server.rootMsgFolder.filePath.path + + " -- " + + ex + ); + // But its remnants are still there, clean up. + let xfolder = this.server.rootMsgFolder.getChildNamed(this.folderName); + this.server.rootMsgFolder.propagateDelete(xfolder, true); + } + }, + + // Gets the next item from itemsToStore and forces that item to be stored + // to the folder. If more items are left to be stored, fires a timer for + // the next one, otherwise triggers a download done notification to the UI. + storeNextItem() { + if (lazy.FeedUtils.CANCEL_REQUESTED) { + lazy.FeedUtils.CANCEL_REQUESTED = false; + this.cleanupParsingState(this, lazy.FeedUtils.kNewsBlogCancel); + return; + } + + if (this.itemsToStore.length == 0) { + let code = lazy.FeedUtils.kNewsBlogSuccess; + this.createFolder(); + if (!this.folder) { + code = lazy.FeedUtils.kNewsBlogFileError; + } + + this.cleanupParsingState(this, code); + return; + } + + let item = this.itemsToStore[this.itemsToStoreIndex]; + + if (item.store()) { + this.itemsStored++; + } + + if (!this.folder) { + this.cleanupParsingState(this, lazy.FeedUtils.kNewsBlogFileError); + return; + } + + this.itemsToStoreIndex++; + + // If the listener is tracking progress for each item, report it here. + if ( + item.feed.downloadCallback && + item.feed.downloadCallback.onFeedItemStored + ) { + item.feed.downloadCallback.onFeedItemStored( + item.feed, + this.itemsToStoreIndex, + this.itemsToStore.length + ); + } + + // Eventually we'll report individual progress here. + + if (this.itemsToStoreIndex < this.itemsToStore.length) { + if (!this.storeItemsTimer) { + this.storeItemsTimer = Cc["@mozilla.org/timer;1"].createInstance( + Ci.nsITimer + ); + } + + this.storeItemsTimer.initWithCallback( + this, + 50, + Ci.nsITimer.TYPE_ONE_SHOT + ); + } else { + // We have just finished downloading one or more feed items into the + // destination folder; if the folder is still listed as having new + // messages in it, then we should set the biff state on the folder so the + // right RDF UI changes happen in the folder pane to indicate new mail. + if (item.feed.folder.hasNewMessages) { + item.feed.folder.biffState = Ci.nsIMsgFolder.nsMsgBiffState_NewMail; + // Run the bayesian spam filter, if enabled. + item.feed.folder.callFilterPlugins(null); + } + + this.cleanupParsingState(this, lazy.FeedUtils.kNewsBlogSuccess); + } + }, + + cleanupParsingState(aFeed, aCode) { + // Now that we are done parsing the feed, remove the feed from the cache. + FeedCache.removeFeed(aFeed.url); + + if (aFeed.parseItems) { + // Do this only if we're in parse/store mode. + aFeed.removeInvalidItems(false); + + if (aCode == lazy.FeedUtils.kNewsBlogSuccess && aFeed.mLastModified) { + aFeed.lastModified = aFeed.mLastModified; + } + + // Flush any feed item changes to disk. + let ds = lazy.FeedUtils.getItemsDS(aFeed.server); + ds.saveSoon(); + lazy.FeedUtils.log.debug( + "Feed.cleanupParsingState: items stored - " + this.itemsStored + ); + } + + // Force the xml http request to go away. This helps reduce some nasty + // assertions on shut down. + this.request = null; + this.itemsToStore = []; + this.itemsToStoreIndex = 0; + this.storeItemsTimer = null; + + if (aFeed.downloadCallback) { + aFeed.downloadCallback.downloaded(aFeed, aCode); + } + }, + + // nsITimerCallback + notify(aTimer) { + this.storeNextItem(); + }, +}; diff --git a/comm/mailnews/extensions/newsblog/FeedItem.jsm b/comm/mailnews/extensions/newsblog/FeedItem.jsm new file mode 100644 index 0000000000..40a0424a0d --- /dev/null +++ b/comm/mailnews/extensions/newsblog/FeedItem.jsm @@ -0,0 +1,490 @@ +/* -*- Mode: JavaScript; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const EXPORTED_SYMBOLS = ["FeedItem", "FeedEnclosure"]; + +const { MailServices } = ChromeUtils.import( + "resource:///modules/MailServices.jsm" +); + +const lazy = {}; +ChromeUtils.defineModuleGetter( + lazy, + "FeedUtils", + "resource:///modules/FeedUtils.jsm" +); + +function FeedItem() { + this.mDate = lazy.FeedUtils.getValidRFC5322Date(); + this.mParserUtils = Cc["@mozilla.org/parserutils;1"].getService( + Ci.nsIParserUtils + ); +} + +FeedItem.prototype = { + // Only for IETF Atom. + xmlContentBase: null, + id: null, + feed: null, + description: null, + content: null, + enclosures: [], + title: null, + // Author must be angle bracket enclosed to function as an addr-spec, in the + // absence of an addr-spec portion of an RFC5322 email address, as other + // functionality (gloda search) depends on this. + author: "<anonymous>", + inReplyTo: "", + keywords: [], + mURL: null, + characterSet: "UTF-8", + + ENCLOSURE_BOUNDARY_PREFIX: "--------------", // 14 dashes + ENCLOSURE_HEADER_BOUNDARY_PREFIX: "------------", // 12 dashes + MESSAGE_TEMPLATE: + "\n" + + "<!DOCTYPE html>\n" + + "<html>\n" + + " <head>\n" + + " <title>%TITLE%</title>\n" + + ' <base href="%BASE%">\n' + + " </head>\n" + + ' <body id="msgFeedSummaryBody" selected="false">\n' + + " %CONTENT%\n" + + " </body>\n" + + "</html>\n", + + get url() { + return this.mURL; + }, + + set url(aVal) { + try { + this.mURL = Services.io.newURI(aVal).spec; + } catch (ex) { + // The url as published or constructed can be a non url. It's used as a + // feeditem identifier in feeditems.rdf, as a messageId, and as an href + // and for the content-base header. Save as is; ensure not null. + this.mURL = aVal ? aVal : ""; + } + }, + + get date() { + return this.mDate; + }, + + set date(aVal) { + this.mDate = aVal; + }, + + get identity() { + return this.feed.name + ": " + this.title + " (" + this.id + ")"; + }, + + normalizeMessageID(messageID) { + // Escape occurrences of message ID meta characters <, >, and @. + messageID.replace(/</g, "%3C"); + messageID.replace(/>/g, "%3E"); + messageID.replace(/@/g, "%40"); + messageID = "<" + messageID.trim() + "@localhost.localdomain>"; + + lazy.FeedUtils.log.trace( + "FeedItem.normalizeMessageID: messageID - " + messageID + ); + return messageID; + }, + + get contentBase() { + if (this.xmlContentBase) { + return this.xmlContentBase; + } + + return this.mURL; + }, + + /** + * Writes the item to the folder as a message and updates the feeditems db. + * + * @returns {void} + */ + store() { + // this.title and this.content contain HTML. + // this.mUrl and this.contentBase contain plain text. + + let stored = false; + let ds = lazy.FeedUtils.getItemsDS(this.feed.server); + let resource = this.findStoredResource(); + if (!this.feed.folder) { + return stored; + } + + if (resource == null) { + resource = { + feedURLs: [this.feed.url], + lastSeenTime: 0, + valid: false, + stored: false, + }; + ds.data[this.id] = resource; + if (!this.content) { + lazy.FeedUtils.log.trace( + "FeedItem.store: " + + this.identity + + " no content; storing description or title" + ); + this.content = this.description || this.title; + } + + let content = this.MESSAGE_TEMPLATE; + content = content.replace(/%TITLE%/, this.title); + content = content.replace(/%BASE%/, this.htmlEscape(this.contentBase)); + content = content.replace(/%CONTENT%/, this.content); + this.content = content; + this.writeToFolder(); + this.markStored(resource); + stored = true; + } + + this.markValid(resource); + ds.saveSoon(); + return stored; + }, + + findStoredResource() { + // Checks to see if the item has already been stored in its feed's + // message folder. + lazy.FeedUtils.log.trace( + "FeedItem.findStoredResource: checking if stored - " + this.identity + ); + + let server = this.feed.server; + let folder = this.feed.folder; + + if (!folder) { + lazy.FeedUtils.log.debug( + "FeedItem.findStoredResource: folder '" + + this.feed.folderName + + "' doesn't exist; creating as child of " + + server.rootMsgFolder.prettyName + + "\n" + ); + this.feed.createFolder(); + return null; + } + + let ds = lazy.FeedUtils.getItemsDS(server); + let item = ds.data[this.id]; + if (!item || !item.stored) { + lazy.FeedUtils.log.trace("FeedItem.findStoredResource: not stored"); + return null; + } + + lazy.FeedUtils.log.trace("FeedItem.findStoredResource: already stored"); + return item; + }, + + markValid(resource) { + resource.lastSeenTime = new Date().getTime(); + // Items can be in multiple feeds. + if (!resource.feedURLs.includes(this.feed.url)) { + resource.feedURLs.push(this.feed.url); + } + resource.valid = true; + }, + + markStored(resource) { + // Items can be in multiple feeds. + if (!resource.feedURLs.includes(this.feed.url)) { + resource.feedURLs.push(this.feed.url); + } + resource.stored = true; + }, + + writeToFolder() { + lazy.FeedUtils.log.trace( + "FeedItem.writeToFolder: " + + this.identity + + " writing to message folder " + + this.feed.name + ); + // The subject may contain HTML entities. Convert these to their unencoded + // state. i.e. & becomes '&'. + let title = this.title; + title = this.mParserUtils.convertToPlainText( + title, + Ci.nsIDocumentEncoder.OutputSelectionOnly | + Ci.nsIDocumentEncoder.OutputAbsoluteLinks, + 0 + ); + + // Compress white space in the subject to make it look better. Trim + // leading/trailing spaces to prevent mbox header folding issue at just + // the right subject length. + this.title = title.replace(/[\t\r\n]+/g, " ").trim(); + + // If the date looks like it's in W3C-DTF format, convert it into + // an IETF standard date. Otherwise assume it's in IETF format. + if (this.mDate.search(/^\d\d\d\d/) != -1) { + this.mDate = new Date(this.mDate).toUTCString(); + } + + // If there is an inreplyto value, create the headers. + let inreplytoHdrsStr = this.inReplyTo + ? "References: " + + this.inReplyTo + + "\n" + + "In-Reply-To: " + + this.inReplyTo + + "\n" + : ""; + + // Support multiple authors in From. + let fromStr = this.createHeaderStrFromArray("From: ", this.author); + + // If there are keywords (categories), create the headers. + let keywordsStr = this.createHeaderStrFromArray( + "Keywords: ", + this.keywords + ); + + // Escape occurrences of "From " at the beginning of lines of + // content per the mbox standard, since "From " denotes a new + // message, and add a line break so we know the last line has one. + this.content = this.content.replace(/([\r\n]+)(>*From )/g, "$1>$2"); + this.content += "\n"; + + let source = + "From - " + + this.mDate + + "\n" + + "X-Mozilla-Status: 0000\n" + + "X-Mozilla-Status2: 00000000\n" + + "X-Mozilla-Keys: " + + " ".repeat(80) + + "\n" + + "Received: by localhost; " + + lazy.FeedUtils.getValidRFC5322Date() + + "\n" + + "Date: " + + this.mDate + + "\n" + + "Message-Id: " + + this.normalizeMessageID(this.id) + + "\n" + + fromStr + + "MIME-Version: 1.0\n" + + "Subject: " + + this.title + + "\n" + + inreplytoHdrsStr + + keywordsStr + + "Content-Transfer-Encoding: 8bit\n" + + "Content-Base: " + + this.mURL + + "\n"; + + if (this.enclosures.length) { + let boundaryID = source.length; + source += + 'Content-Type: multipart/mixed; boundary="' + + this.ENCLOSURE_HEADER_BOUNDARY_PREFIX + + boundaryID + + '"\n\n' + + "This is a multi-part message in MIME format.\n" + + this.ENCLOSURE_BOUNDARY_PREFIX + + boundaryID + + "\n" + + "Content-Type: text/html; charset=" + + this.characterSet + + "\n" + + "Content-Transfer-Encoding: 8bit\n" + + this.content; + + this.enclosures.forEach(function (enclosure) { + source += enclosure.convertToAttachment(boundaryID); + }); + + source += this.ENCLOSURE_BOUNDARY_PREFIX + boundaryID + "--\n\n\n"; + } else { + source += + "Content-Type: text/html; charset=" + + this.characterSet + + "\n" + + this.content; + } + + lazy.FeedUtils.log.trace( + "FeedItem.writeToFolder: " + + this.identity + + " is " + + source.length + + " characters long" + ); + + // Get the folder and database storing the feed's messages and headers. + let folder = this.feed.folder.QueryInterface(Ci.nsIMsgLocalMailFolder); + let msgFolder = folder.QueryInterface(Ci.nsIMsgFolder); + msgFolder.gettingNewMessages = true; + // Source is a unicode js string, as UTF-16, and we want to save a + // char * cpp |string| as UTF-8 bytes. The source xml doc encoding is utf8. + source = unescape(encodeURIComponent(source)); + let msgDBHdr = folder.addMessage(source); + msgDBHdr.orFlags(Ci.nsMsgMessageFlags.FeedMsg); + msgFolder.gettingNewMessages = false; + this.tagItem(msgDBHdr, this.keywords); + }, + + /** + * Create a header string from an array. Intended for comma separated headers + * like From or Keywords. In the case of a longer than RFC5322 recommended + * line length, create multiple folded lines (easier to parse than multiple + * headers). + * + * @param {string} headerName - Name of the header. + * @param {string[]} headerItemsArray - An Array of strings to concatenate. + * + * @returns {String} - The header string. + */ + createHeaderStrFromArray(headerName, headerItemsArray) { + let headerStr = ""; + if (!headerItemsArray || headerItemsArray.length == 0) { + return headerStr; + } + + const HEADER = headerName; + const LINELENGTH = 78; + const MAXLINELENGTH = 990; + let items = [].concat(headerItemsArray); + let lines = []; + headerStr = HEADER; + while (items.length) { + let item = items.shift(); + if ( + headerStr.length + item.length > LINELENGTH && + headerStr.length > HEADER.length + ) { + lines.push(headerStr); + headerStr = " ".repeat(HEADER.length); + } + + headerStr += + headerStr.length + item.length > MAXLINELENGTH + ? item.substr(0, MAXLINELENGTH - headerStr.length) + "…, " + : item + ", "; + } + + headerStr = headerStr.replace(/,\s$/, "\n"); + lines.push(headerStr); + headerStr = lines.join("\n"); + + return headerStr; + }, + + /** + * Autotag messages. + * + * @param {nsIMsgDBHdr} aMsgDBHdr - message to tag + * @param {Array} aKeywords - keywords (tags) + * @returns {void} + */ + tagItem(aMsgDBHdr, aKeywords) { + let category = this.feed.options.category; + if (!aKeywords.length || !category.enabled) { + return; + } + + let prefix = category.prefixEnabled ? category.prefix : ""; + let rtl = Services.prefs.getIntPref("bidi.direction") == 2; + + let keys = []; + for (let keyword of aKeywords) { + keyword = rtl ? keyword + prefix : prefix + keyword; + let keyForTag = MailServices.tags.getKeyForTag(keyword); + if (!keyForTag) { + // Add the tag if it doesn't exist. + MailServices.tags.addTag(keyword, "", lazy.FeedUtils.AUTOTAG); + keyForTag = MailServices.tags.getKeyForTag(keyword); + } + + // Add the tag key to the keys array. + keys.push(keyForTag); + } + + if (keys.length) { + // Add the keys to the message. + aMsgDBHdr.folder.addKeywordsToMessages([aMsgDBHdr], keys.join(" ")); + } + }, + + htmlEscape(s) { + s = s.replace(/&/g, "&"); + s = s.replace(/>/g, ">"); + s = s.replace(/</g, "<"); + s = s.replace(/'/g, "'"); + s = s.replace(/"/g, """); + return s; + }, +}; + +// A feed enclosure is to RSS what an attachment is for e-mail. We make +// enclosures look like attachments in the UI. +function FeedEnclosure(aURL, aContentType, aLength, aTitle) { + this.mURL = aURL; + // Store a reasonable mimetype if content-type is not present. + this.mContentType = aContentType || "application/unknown"; + this.mLength = aLength; + this.mTitle = aTitle; + + // Generate a fileName from the URL. + if (this.mURL) { + try { + let uri = Services.io.newURI(this.mURL).QueryInterface(Ci.nsIURL); + this.mFileName = uri.fileName; + // Determine mimetype from extension if content-type is not present. + if (!aContentType) { + let contentType = Cc["@mozilla.org/mime;1"] + .getService(Ci.nsIMIMEService) + .getTypeFromExtension(uri.fileExtension); + this.mContentType = contentType; + } + } catch (ex) { + this.mFileName = this.mURL; + } + } +} + +FeedEnclosure.prototype = { + mURL: "", + mContentType: "", + mLength: 0, + mFileName: "", + mTitle: "", + ENCLOSURE_BOUNDARY_PREFIX: "--------------", // 14 dashes + + // Returns a string that looks like an e-mail attachment which represents + // the enclosure. + convertToAttachment(aBoundaryID) { + return ( + "\n" + + this.ENCLOSURE_BOUNDARY_PREFIX + + aBoundaryID + + "\n" + + "Content-Type: " + + this.mContentType + + '; name="' + + (this.mTitle || this.mFileName) + + (this.mLength ? '"; size=' + this.mLength : '"') + + "\n" + + "X-Mozilla-External-Attachment-URL: " + + this.mURL + + "\n" + + 'Content-Disposition: attachment; filename="' + + this.mFileName + + '"\n\n' + + lazy.FeedUtils.strings.GetStringFromName("externalAttachmentMsg") + + "\n" + ); + }, +}; diff --git a/comm/mailnews/extensions/newsblog/FeedParser.jsm b/comm/mailnews/extensions/newsblog/FeedParser.jsm new file mode 100644 index 0000000000..863d5789fe --- /dev/null +++ b/comm/mailnews/extensions/newsblog/FeedParser.jsm @@ -0,0 +1,1496 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const EXPORTED_SYMBOLS = ["FeedParser"]; + +const lazy = {}; + +ChromeUtils.defineModuleGetter( + lazy, + "FeedItem", + "resource:///modules/FeedItem.jsm" +); +ChromeUtils.defineModuleGetter( + lazy, + "FeedEnclosure", + "resource:///modules/FeedItem.jsm" +); +ChromeUtils.defineModuleGetter( + lazy, + "FeedUtils", + "resource:///modules/FeedUtils.jsm" +); + +/** + * The feed parser. Depends on FeedItem.js, Feed.js. + * + * @class + */ +function FeedParser() { + this.parsedItems = []; + this.mSerializer = new XMLSerializer(); +} + +FeedParser.prototype = { + /** + * parseFeed() returns an array of parsed items ready for processing. It is + * currently a synchronous operation. If there is an error parsing the feed, + * parseFeed returns an empty feed in addition to calling aFeed.onParseError. + * + * @param {Feed} aFeed - The Feed object. + * @param {XMLDocument} aDOM - The document to parse. + * @returns {Array} - array of items, or empty array for error returns or + * nothing to do condition. + */ + parseFeed(aFeed, aDOM) { + if (!XMLDocument.isInstance(aDOM)) { + // No xml doc. + aFeed.onParseError(aFeed); + return []; + } + + let doc = aDOM.documentElement; + if (doc.namespaceURI == lazy.FeedUtils.MOZ_PARSERERROR_NS) { + // Gecko caught a basic parsing error. + let errStr = + doc.firstChild.textContent + "\n" + doc.firstElementChild.textContent; + lazy.FeedUtils.log.info("FeedParser.parseFeed: - " + errStr); + aFeed.onParseError(aFeed); + return []; + } else if (aDOM.querySelector("redirect")) { + // Check for RSS2.0 redirect document. + let channel = aDOM.querySelector("redirect"); + if (this.isPermanentRedirect(aFeed, channel, null)) { + return []; + } + + aFeed.onParseError(aFeed); + return []; + } else if ( + doc.namespaceURI == lazy.FeedUtils.RDF_SYNTAX_NS && + doc.getElementsByTagNameNS(lazy.FeedUtils.RSS_NS, "channel")[0] + ) { + aFeed.mFeedType = "RSS_1.xRDF"; + lazy.FeedUtils.log.debug( + "FeedParser.parseFeed: type:url - " + + aFeed.mFeedType + + " : " + + aFeed.url + ); + + return this.parseAsRSS1(aFeed, aDOM); + } else if (doc.namespaceURI == lazy.FeedUtils.ATOM_03_NS) { + aFeed.mFeedType = "ATOM_0.3"; + lazy.FeedUtils.log.debug( + "FeedParser.parseFeed: type:url - " + + aFeed.mFeedType + + " : " + + aFeed.url + ); + return this.parseAsAtom(aFeed, aDOM); + } else if (doc.namespaceURI == lazy.FeedUtils.ATOM_IETF_NS) { + aFeed.mFeedType = "ATOM_IETF"; + lazy.FeedUtils.log.debug( + "FeedParser.parseFeed: type:url - " + + aFeed.mFeedType + + " : " + + aFeed.url + ); + return this.parseAsAtomIETF(aFeed, aDOM); + } else if ( + doc.getElementsByTagNameNS(lazy.FeedUtils.RSS_090_NS, "channel")[0] + ) { + aFeed.mFeedType = "RSS_0.90"; + lazy.FeedUtils.log.debug( + "FeedParser.parseFeed: type:url - " + + aFeed.mFeedType + + " : " + + aFeed.url + ); + return this.parseAsRSS2(aFeed, aDOM); + } + + // Parse as RSS 0.9x. In theory even RSS 1.0 feeds could be parsed by + // the 0.9x parser if the RSS namespace were the default. + let rssVer = doc.localName == "rss" ? doc.getAttribute("version") : null; + if (rssVer) { + aFeed.mFeedType = "RSS_" + rssVer; + } else { + aFeed.mFeedType = "RSS_0.9x?"; + } + lazy.FeedUtils.log.debug( + "FeedParser.parseFeed: type:url - " + aFeed.mFeedType + " : " + aFeed.url + ); + return this.parseAsRSS2(aFeed, aDOM); + }, + + parseAsRSS2(aFeed, aDOM) { + // Get the first channel (assuming there is only one per RSS File). + let channel = aDOM.querySelector("channel"); + if (!channel) { + aFeed.onParseError(aFeed); + return []; + } + + // Usually the empty string, unless this is RSS .90. + let nsURI = channel.namespaceURI || ""; + + if (this.isPermanentRedirect(aFeed, null, channel)) { + return []; + } + + let tags = this.childrenByTagNameNS(channel, nsURI, "title"); + aFeed.title = aFeed.title || this.getNodeValue(tags ? tags[0] : null); + tags = this.childrenByTagNameNS(channel, nsURI, "description"); + aFeed.description = this.getNodeValueFormatted(tags ? tags[0] : null); + tags = this.childrenByTagNameNS(channel, nsURI, "link"); + aFeed.link = this.validLink(this.getNodeValue(tags ? tags[0] : null)); + + if (!(aFeed.title || aFeed.description)) { + lazy.FeedUtils.log.error( + "FeedParser.parseAsRSS2: missing mandatory element " + + "<title> and <description>" + ); + // The RSS2 spec requires a <link> as well, but we can do without it + // so ignore the case of (valid) link missing. + aFeed.onParseError(aFeed); + return []; + } + + if (!aFeed.parseItems) { + return []; + } + + this.findSyUpdateTags(aFeed, channel); + + aFeed.invalidateItems(); + // XXX use getElementsByTagNameNS for now; childrenByTagNameNS would be + // better, but RSS .90 is still with us. + let itemNodes = aDOM.getElementsByTagNameNS(nsURI, "item"); + itemNodes = itemNodes ? itemNodes : []; + lazy.FeedUtils.log.debug( + "FeedParser.parseAsRSS2: items to parse - " + itemNodes.length + ); + + for (let itemNode of itemNodes) { + if (!itemNode.childElementCount) { + continue; + } + + let item = new lazy.FeedItem(); + item.feed = aFeed; + item.enclosures = []; + item.keywords = []; + + tags = this.childrenByTagNameNS( + itemNode, + lazy.FeedUtils.FEEDBURNER_NS, + "origLink" + ); + let link = this.validLink(this.getNodeValue(tags ? tags[0] : null)); + if (!link) { + tags = this.childrenByTagNameNS(itemNode, nsURI, "link"); + link = this.validLink(this.getNodeValue(tags ? tags[0] : null)); + } + tags = this.childrenByTagNameNS(itemNode, nsURI, "guid"); + let guidNode = tags ? tags[0] : null; + + let guid; + let isPermaLink = false; + if (guidNode) { + guid = this.getNodeValue(guidNode); + // isPermaLink is true if the value is "true" or if the attribute is + // not present; all other values, including "false" and "False" and + // for that matter "TRuE" and "meatcake" are false. + if ( + !guidNode.hasAttribute("isPermaLink") || + guidNode.getAttribute("isPermaLink") == "true" + ) { + isPermaLink = true; + } + // If attribute isPermaLink is missing, it is good to check the validity + // of <guid> value as an URL to avoid linking to non-URL strings. + if (!guidNode.hasAttribute("isPermaLink")) { + try { + Services.io.newURI(guid); + if (Services.io.extractScheme(guid) == "tag") { + isPermaLink = false; + } + } catch (ex) { + isPermaLink = false; + } + } + + item.id = guid; + } + + let guidLink = this.validLink(guid); + if (isPermaLink && guidLink) { + item.url = guidLink; + } else if (link) { + item.url = link; + } else { + item.url = null; + } + + tags = this.childrenByTagNameNS(itemNode, nsURI, "description"); + item.description = this.getNodeValueFormatted(tags ? tags[0] : null); + tags = this.childrenByTagNameNS(itemNode, nsURI, "title"); + item.title = this.getNodeValue(tags ? tags[0] : null); + if (!(item.title || item.description)) { + lazy.FeedUtils.log.info( + "FeedParser.parseAsRSS2: <item> missing mandatory " + + "element, either <title> or <description>; skipping" + ); + continue; + } + + if (!item.id) { + // At this point, if there is no guid, uniqueness cannot be guaranteed + // by any of link or date (optional) or title (optional unless there + // is no description). Use a big chunk of description; minimize dupes + // with url and title if present. + item.id = + (item.url || item.feed.url) + + "#" + + item.title + + "#" + + (this.stripTags( + item.description ? item.description.substr(0, 150) : null + ) || item.title); + item.id = item.id.replace(/[\n\r\t\s]+/g, " "); + } + + // Escape html entities in <title>, which are unescaped as textContent + // values. If the title is used as content, it will remain escaped; if + // it is used as the title, it will be unescaped upon store. Bug 1240603. + // The <description> tag must follow escaping examples found in + // http://www.rssboard.org/rss-encoding-examples, i.e. single escape angle + // brackets for tags, which are removed if used as title, and double + // escape entities for presentation in title. + // Better: always use <title>. Best: use Atom. + if (!item.title) { + item.title = this.stripTags(item.description).substr(0, 150); + } else { + item.title = item.htmlEscape(item.title); + } + + tags = this.childrenByTagNameNS(itemNode, nsURI, "author"); + if (!tags) { + tags = this.childrenByTagNameNS( + itemNode, + lazy.FeedUtils.DC_NS, + "creator" + ); + } + let author = this.getNodeValue(tags ? tags[0] : null) || aFeed.title; + author = this.cleanAuthorName(author); + item.author = author ? ["<" + author + ">"] : item.author; + + tags = this.childrenByTagNameNS(itemNode, nsURI, "pubDate"); + if (!tags || !this.getNodeValue(tags[0])) { + tags = this.childrenByTagNameNS(itemNode, lazy.FeedUtils.DC_NS, "date"); + } + item.date = this.getNodeValue(tags ? tags[0] : null) || item.date; + + // If the date is invalid, users will see the beginning of the epoch + // unless we reset it here, so they'll see the current time instead. + // This is typical aggregator behavior. + if (item.date) { + item.date = item.date.trim(); + if (!lazy.FeedUtils.isValidRFC822Date(item.date)) { + // XXX Use this on the other formats as well. + item.date = this.dateRescue(item.date); + } + } + + tags = this.childrenByTagNameNS( + itemNode, + lazy.FeedUtils.RSS_CONTENT_NS, + "encoded" + ); + item.content = this.getNodeValueFormatted(tags ? tags[0] : null); + + // Handle <enclosures> and <media:content>, which may be in a + // <media:group> (if present). + tags = this.childrenByTagNameNS(itemNode, nsURI, "enclosure"); + let encUrls = []; + if (tags) { + for (let tag of tags) { + let url = this.validLink(tag.getAttribute("url")); + if (url && !encUrls.includes(url)) { + let type = this.removeUnprintableASCII(tag.getAttribute("type")); + let length = this.removeUnprintableASCII( + tag.getAttribute("length") + ); + item.enclosures.push(new lazy.FeedEnclosure(url, type, length)); + encUrls.push(url); + } + } + } + + tags = itemNode.getElementsByTagNameNS(lazy.FeedUtils.MRSS_NS, "content"); + if (tags) { + for (let tag of tags) { + let url = this.validLink(tag.getAttribute("url")); + if (url && !encUrls.includes(url)) { + let type = this.removeUnprintableASCII(tag.getAttribute("type")); + let fileSize = this.removeUnprintableASCII( + tag.getAttribute("fileSize") + ); + item.enclosures.push(new lazy.FeedEnclosure(url, type, fileSize)); + } + } + } + + // The <origEnclosureLink> tag has no specification, especially regarding + // whether more than one tag is allowed and, if so, how tags would + // relate to previously declared (and well specified) enclosure urls. + // The common usage is to include 1 origEnclosureLink, in addition to + // the specified enclosure tags for 1 enclosure. Thus, we will replace the + // first enclosure's, if found, url with the first <origEnclosureLink> + // url only or else add the <origEnclosureLink> url. + tags = this.childrenByTagNameNS( + itemNode, + lazy.FeedUtils.FEEDBURNER_NS, + "origEnclosureLink" + ); + let origEncUrl = this.validLink(this.getNodeValue(tags ? tags[0] : null)); + if (origEncUrl) { + if (item.enclosures.length) { + item.enclosures[0].mURL = origEncUrl; + } else { + item.enclosures.push(new lazy.FeedEnclosure(origEncUrl)); + } + } + + // Support <category> and autotagging. + tags = this.childrenByTagNameNS(itemNode, nsURI, "category"); + if (tags) { + for (let tag of tags) { + let term = this.getNodeValue(tag); + term = term ? this.xmlUnescape(term.replace(/,/g, ";")) : null; + if (term && !item.keywords.includes(term)) { + item.keywords.push(term); + } + } + } + + this.parsedItems.push(item); + } + + return this.parsedItems; + }, + + /** + * Extracts feed details and (optionally) items from an RSS1 + * feed which has already been XML-parsed as an XMLDocument. + * The feed items are extracted only if feed.parseItems is set. + * + * Technically RSS1 is supposed to be treated as RDFXML, but in practice + * no feed parser anywhere ever does this, and feeds in the wild are + * pretty shakey on their RDF encoding too. So we just treat it as raw + * XML and pick out the bits we want. + * + * @param {Feed} feed - The Feed object. + * @param {XMLDocument} doc - The document to parse. + * @returns {Array} - array of FeedItems or empty array for error returns or + * nothing to do condition (ie unset feed.parseItems). + */ + parseAsRSS1(feed, doc) { + let channel = doc.querySelector("channel"); + if (!channel) { + feed.onParseError(feed); + return []; + } + + if (this.isPermanentRedirect(feed, null, channel)) { + return []; + } + + let titleNode = this.childByTagNameNS( + channel, + lazy.FeedUtils.RSS_NS, + "title" + ); + // If user entered a title manually, retain it. + feed.title = feed.title || this.getNodeValue(titleNode) || feed.url; + + let descNode = this.childByTagNameNS( + channel, + lazy.FeedUtils.RSS_NS, + "description" + ); + feed.description = this.getNodeValueFormatted(descNode) || ""; + + let linkNode = this.childByTagNameNS( + channel, + lazy.FeedUtils.RSS_NS, + "link" + ); + feed.link = this.validLink(this.getNodeValue(linkNode)) || feed.url; + + if (!(feed.title || feed.description) || !feed.link) { + lazy.FeedUtils.log.error( + "FeedParser.parseAsRSS1: missing mandatory element " + + "<title> and <description>, or <link>" + ); + feed.onParseError(feed); + return []; + } + + // If we're only interested in the overall feed description, we're done. + if (!feed.parseItems) { + return []; + } + + this.findSyUpdateTags(feed, channel); + + feed.invalidateItems(); + + // Now process all the individual items in the feed. + let itemNodes = doc.getElementsByTagNameNS(lazy.FeedUtils.RSS_NS, "item"); + itemNodes = itemNodes ? itemNodes : []; + + for (let itemNode of itemNodes) { + let item = new lazy.FeedItem(); + item.feed = feed; + + // Prefer the value of the link tag to the item URI since the URI could be + // a relative URN. + let itemURI = itemNode.getAttribute("about") || ""; + itemURI = this.removeUnprintableASCII(itemURI.trim()); + let linkNode = this.childByTagNameNS( + itemNode, + lazy.FeedUtils.RSS_NS, + "link" + ); + item.id = this.getNodeValue(linkNode) || itemURI; + item.url = this.validLink(item.id); + + let descNode = this.childByTagNameNS( + itemNode, + lazy.FeedUtils.RSS_NS, + "description" + ); + item.description = this.getNodeValueFormatted(descNode); + + let titleNode = this.childByTagNameNS( + itemNode, + lazy.FeedUtils.RSS_NS, + "title" + ); + let subjectNode = this.childByTagNameNS( + itemNode, + lazy.FeedUtils.DC_NS, + "subject" + ); + + item.title = + this.getNodeValue(titleNode) || this.getNodeValue(subjectNode); + if (!item.title && item.description) { + item.title = this.stripTags(item.description).substr(0, 150); + } + if (!item.url || !item.title) { + lazy.FeedUtils.log.info( + "FeedParser.parseAsRSS1: <item> missing mandatory " + + "element <item rdf:about> and <link>, or <title> and " + + "no <description>; skipping" + ); + continue; + } + + // TODO XXX: ignores multiple authors. + let authorNode = this.childByTagNameNS( + itemNode, + lazy.FeedUtils.DC_NS, + "creator" + ); + let channelCreatorNode = this.childByTagNameNS( + channel, + lazy.FeedUtils.DC_NS, + "creator" + ); + let author = + this.getNodeValue(authorNode) || + this.getNodeValue(channelCreatorNode) || + feed.title; + author = this.cleanAuthorName(author); + item.author = author ? ["<" + author + ">"] : item.author; + + let dateNode = this.childByTagNameNS( + itemNode, + lazy.FeedUtils.DC_NS, + "date" + ); + item.date = this.getNodeValue(dateNode) || item.date; + + let contentNode = this.childByTagNameNS( + itemNode, + lazy.FeedUtils.RSS_CONTENT_NS, + "encoded" + ); + item.content = this.getNodeValueFormatted(contentNode); + + this.parsedItems.push(item); + } + lazy.FeedUtils.log.debug( + "FeedParser.parseAsRSS1: items parsed - " + this.parsedItems.length + ); + + return this.parsedItems; + }, + + // TODO: deprecate ATOM_03_NS. + parseAsAtom(aFeed, aDOM) { + // Get the first channel (assuming there is only one per Atom File). + let channel = aDOM.querySelector("feed"); + if (!channel) { + aFeed.onParseError(aFeed); + return []; + } + + if (this.isPermanentRedirect(aFeed, null, channel)) { + return []; + } + + let tags = this.childrenByTagNameNS( + channel, + lazy.FeedUtils.ATOM_03_NS, + "title" + ); + aFeed.title = + aFeed.title || this.stripTags(this.getNodeValue(tags ? tags[0] : null)); + tags = this.childrenByTagNameNS( + channel, + lazy.FeedUtils.ATOM_03_NS, + "tagline" + ); + aFeed.description = this.getNodeValueFormatted(tags ? tags[0] : null); + tags = this.childrenByTagNameNS(channel, lazy.FeedUtils.ATOM_03_NS, "link"); + aFeed.link = this.validLink(this.findAtomLink("alternate", tags)); + + if (!aFeed.title) { + lazy.FeedUtils.log.error( + "FeedParser.parseAsAtom: missing mandatory element <title>" + ); + aFeed.onParseError(aFeed); + return []; + } + + if (!aFeed.parseItems) { + return []; + } + + this.findSyUpdateTags(aFeed, channel); + + aFeed.invalidateItems(); + let items = this.childrenByTagNameNS( + channel, + lazy.FeedUtils.ATOM_03_NS, + "entry" + ); + items = items ? items : []; + lazy.FeedUtils.log.debug( + "FeedParser.parseAsAtom: items to parse - " + items.length + ); + + for (let itemNode of items) { + if (!itemNode.childElementCount) { + continue; + } + + let item = new lazy.FeedItem(); + item.feed = aFeed; + + tags = this.childrenByTagNameNS( + itemNode, + lazy.FeedUtils.ATOM_03_NS, + "link" + ); + item.url = this.validLink(this.findAtomLink("alternate", tags)); + + tags = this.childrenByTagNameNS( + itemNode, + lazy.FeedUtils.ATOM_03_NS, + "id" + ); + item.id = this.getNodeValue(tags ? tags[0] : null); + tags = this.childrenByTagNameNS( + itemNode, + lazy.FeedUtils.ATOM_03_NS, + "summary" + ); + item.description = this.getNodeValueFormatted(tags ? tags[0] : null); + tags = this.childrenByTagNameNS( + itemNode, + lazy.FeedUtils.ATOM_03_NS, + "title" + ); + item.title = + this.getNodeValue(tags ? tags[0] : null) || + (item.description ? item.description.substr(0, 150) : null); + if (!item.title || !item.id) { + // We're lenient about other mandatory tags, but insist on these. + lazy.FeedUtils.log.info( + "FeedParser.parseAsAtom: <entry> missing mandatory " + + "element <id>, or <title> and no <summary>; skipping" + ); + continue; + } + + tags = this.childrenByTagNameNS( + itemNode, + lazy.FeedUtils.ATOM_03_NS, + "author" + ); + if (!tags) { + tags = this.childrenByTagNameNS( + itemNode, + lazy.FeedUtils.ATOM_03_NS, + "contributor" + ); + } + if (!tags) { + tags = this.childrenByTagNameNS( + channel, + lazy.FeedUtils.ATOM_03_NS, + "author" + ); + } + + let authorEl = tags ? tags[0] : null; + + let author = ""; + if (authorEl) { + tags = this.childrenByTagNameNS( + authorEl, + lazy.FeedUtils.ATOM_03_NS, + "name" + ); + let name = this.getNodeValue(tags ? tags[0] : null); + tags = this.childrenByTagNameNS( + authorEl, + lazy.FeedUtils.ATOM_03_NS, + "email" + ); + let email = this.getNodeValue(tags ? tags[0] : null); + if (name) { + author = name + (email ? " <" + email + ">" : ""); + } else if (email) { + author = email; + } + } + + item.author = author || item.author || aFeed.title; + + tags = this.childrenByTagNameNS( + itemNode, + lazy.FeedUtils.ATOM_03_NS, + "modified" + ); + if (!tags || !this.getNodeValue(tags[0])) { + tags = this.childrenByTagNameNS( + itemNode, + lazy.FeedUtils.ATOM_03_NS, + "issued" + ); + } + if (!tags || !this.getNodeValue(tags[0])) { + tags = this.childrenByTagNameNS( + channel, + lazy.FeedUtils.ATOM_03_NS, + "created" + ); + } + + item.date = this.getNodeValue(tags ? tags[0] : null) || item.date; + + // XXX We should get the xml:base attribute from the content tag as well + // and use it as the base HREF of the message. + // XXX Atom feeds can have multiple content elements; we should differentiate + // between them and pick the best one. + // Some Atom feeds wrap the content in a CTYPE declaration; others use + // a namespace to identify the tags as HTML; and a few are buggy and put + // HTML tags in without declaring their namespace so they look like Atom. + // We deal with the first two but not the third. + tags = this.childrenByTagNameNS( + itemNode, + lazy.FeedUtils.ATOM_03_NS, + "content" + ); + let contentNode = tags ? tags[0] : null; + + let content; + if (contentNode) { + content = ""; + for (let node of contentNode.childNodes) { + if (node.nodeType == node.CDATA_SECTION_NODE) { + content += node.data; + } else { + content += this.mSerializer.serializeToString(node); + } + } + + if (contentNode.getAttribute("mode") == "escaped") { + content = content.replace(/</g, "<"); + content = content.replace(/>/g, ">"); + content = content.replace(/&/g, "&"); + } + + if (content == "") { + content = null; + } + } + + item.content = content; + this.parsedItems.push(item); + } + + return this.parsedItems; + }, + + parseAsAtomIETF(aFeed, aDOM) { + // Get the first channel (assuming there is only one per Atom File). + let channel = this.childrenByTagNameNS( + aDOM, + lazy.FeedUtils.ATOM_IETF_NS, + "feed" + )[0]; + if (!channel) { + aFeed.onParseError(aFeed); + return []; + } + + if (this.isPermanentRedirect(aFeed, null, channel)) { + return []; + } + + let contentBase = channel.getAttribute("xml:base"); + + let tags = this.childrenByTagNameNS( + channel, + lazy.FeedUtils.ATOM_IETF_NS, + "title" + ); + aFeed.title = + aFeed.title || + this.stripTags(this.serializeTextConstruct(tags ? tags[0] : null)); + + tags = this.childrenByTagNameNS( + channel, + lazy.FeedUtils.ATOM_IETF_NS, + "subtitle" + ); + aFeed.description = this.serializeTextConstruct(tags ? tags[0] : null); + + // Per spec, aFeed.link and contentBase may both end up null here. + tags = this.childrenByTagNameNS( + channel, + lazy.FeedUtils.ATOM_IETF_NS, + "link" + ); + aFeed.link = + this.findAtomLink("self", tags, contentBase) || + this.findAtomLink("alternate", tags, contentBase); + aFeed.link = this.validLink(aFeed.link); + if (!contentBase) { + contentBase = aFeed.link; + } + + if (!aFeed.title) { + lazy.FeedUtils.log.error( + "FeedParser.parseAsAtomIETF: missing mandatory element <title>" + ); + aFeed.onParseError(aFeed); + return []; + } + + if (!aFeed.parseItems) { + return []; + } + + this.findSyUpdateTags(aFeed, channel); + + aFeed.invalidateItems(); + let items = this.childrenByTagNameNS( + channel, + lazy.FeedUtils.ATOM_IETF_NS, + "entry" + ); + items = items ? items : []; + lazy.FeedUtils.log.debug( + "FeedParser.parseAsAtomIETF: items to parse - " + items.length + ); + + for (let itemNode of items) { + if (!itemNode.childElementCount) { + continue; + } + + let item = new lazy.FeedItem(); + item.feed = aFeed; + item.enclosures = []; + item.keywords = []; + + contentBase = itemNode.getAttribute("xml:base") || contentBase; + + tags = this.childrenByTagNameNS( + itemNode, + lazy.FeedUtils.ATOM_IETF_NS, + "source" + ); + let source = tags ? tags[0] : null; + + // Per spec, item.link and contentBase may both end up null here. + // If <content> is also not present, then <link rel="alternate"> is MUST + // but we're lenient. + tags = this.childrenByTagNameNS( + itemNode, + lazy.FeedUtils.FEEDBURNER_NS, + "origLink" + ); + item.url = this.validLink(this.getNodeValue(tags ? tags[0] : null)); + if (!item.url) { + tags = this.childrenByTagNameNS( + itemNode, + lazy.FeedUtils.ATOM_IETF_NS, + "link" + ); + item.url = + this.validLink(this.findAtomLink("alternate", tags, contentBase)) || + aFeed.link; + } + if (!contentBase) { + contentBase = item.url; + } + + tags = this.childrenByTagNameNS( + itemNode, + lazy.FeedUtils.ATOM_IETF_NS, + "id" + ); + item.id = this.getNodeValue(tags ? tags[0] : null); + tags = this.childrenByTagNameNS( + itemNode, + lazy.FeedUtils.ATOM_IETF_NS, + "summary" + ); + item.description = this.serializeTextConstruct(tags ? tags[0] : null); + tags = this.childrenByTagNameNS( + itemNode, + lazy.FeedUtils.ATOM_IETF_NS, + "title" + ); + if (!tags || !this.getNodeValue(tags[0])) { + tags = this.childrenByTagNameNS( + source, + lazy.FeedUtils.ATOM_IETF_NS, + "title" + ); + } + item.title = this.stripTags( + this.serializeTextConstruct(tags ? tags[0] : null) || + (item.description ? item.description.substr(0, 150) : null) + ); + if (!item.title || !item.id) { + // We're lenient about other mandatory tags, but insist on these. + lazy.FeedUtils.log.info( + "FeedParser.parseAsAtomIETF: <entry> missing mandatory " + + "element <id>, or <title> and no <summary>; skipping" + ); + continue; + } + + // Support multiple authors. + tags = this.childrenByTagNameNS( + itemNode, + lazy.FeedUtils.ATOM_IETF_NS, + "author" + ); + if (!tags) { + tags = this.childrenByTagNameNS( + source, + lazy.FeedUtils.ATOM_IETF_NS, + "author" + ); + } + if (!tags) { + tags = this.childrenByTagNameNS( + channel, + lazy.FeedUtils.ATOM_IETF_NS, + "author" + ); + } + + let authorTags = tags || []; + let authors = []; + for (let authorTag of authorTags) { + let author = ""; + tags = this.childrenByTagNameNS( + authorTag, + lazy.FeedUtils.ATOM_IETF_NS, + "name" + ); + let name = this.getNodeValue(tags ? tags[0] : null); + tags = this.childrenByTagNameNS( + authorTag, + lazy.FeedUtils.ATOM_IETF_NS, + "email" + ); + let email = this.getNodeValue(tags ? tags[0] : null); + if (name) { + name = this.cleanAuthorName(name); + if (email) { + if (!email.match(/^<.*>$/)) { + email = " <" + email + ">"; + } + author = name + email; + } else { + author = "<" + name + ">"; + } + } else if (email) { + author = email; + } + + if (author) { + authors.push(author); + } + } + + if (authors.length == 0) { + tags = this.childrenByTagNameNS( + channel, + lazy.FeedUtils.DC_NS, + "publisher" + ); + let author = this.getNodeValue(tags ? tags[0] : null) || aFeed.title; + author = this.cleanAuthorName(author); + item.author = author ? ["<" + author + ">"] : item.author; + } else { + item.author = authors; + } + lazy.FeedUtils.log.trace( + "FeedParser.parseAsAtomIETF: author(s) - " + item.author + ); + + tags = this.childrenByTagNameNS( + itemNode, + lazy.FeedUtils.ATOM_IETF_NS, + "updated" + ); + if (!tags || !this.getNodeValue(tags[0])) { + tags = this.childrenByTagNameNS( + itemNode, + lazy.FeedUtils.ATOM_IETF_NS, + "published" + ); + } + if (!tags || !this.getNodeValue(tags[0])) { + tags = this.childrenByTagNameNS( + source, + lazy.FeedUtils.ATOM_IETF_NS, + "published" + ); + } + item.date = this.getNodeValue(tags ? tags[0] : null) || item.date; + + tags = this.childrenByTagNameNS( + itemNode, + lazy.FeedUtils.ATOM_IETF_NS, + "content" + ); + item.content = this.serializeTextConstruct(tags ? tags[0] : null); + + // Ensure relative links can be resolved and Content-Base set to an + // absolute url for the entry. But it's not mandatory that a url is found + // for Content-Base, per spec. + if (item.content) { + item.xmlContentBase = + (tags && tags[0].getAttribute("xml:base")) || contentBase; + } else if (item.description) { + tags = this.childrenByTagNameNS( + itemNode, + lazy.FeedUtils.ATOM_IETF_NS, + "summary" + ); + item.xmlContentBase = + (tags && tags[0].getAttribute("xml:base")) || contentBase; + } else { + item.xmlContentBase = contentBase; + } + + item.xmlContentBase = this.validLink(item.xmlContentBase); + + // Handle <link rel="enclosure"> (if present). + tags = this.childrenByTagNameNS( + itemNode, + lazy.FeedUtils.ATOM_IETF_NS, + "link" + ); + let encUrls = []; + if (tags) { + for (let tag of tags) { + let url = + tag.getAttribute("rel") == "enclosure" + ? (tag.getAttribute("href") || "").trim() + : null; + url = this.validLink(url); + if (url && !encUrls.includes(url)) { + let type = this.removeUnprintableASCII(tag.getAttribute("type")); + let length = this.removeUnprintableASCII( + tag.getAttribute("length") + ); + let title = this.removeUnprintableASCII(tag.getAttribute("title")); + item.enclosures.push( + new lazy.FeedEnclosure(url, type, length, title) + ); + encUrls.push(url); + } + } + } + + tags = this.childrenByTagNameNS( + itemNode, + lazy.FeedUtils.FEEDBURNER_NS, + "origEnclosureLink" + ); + let origEncUrl = this.validLink(this.getNodeValue(tags ? tags[0] : null)); + if (origEncUrl) { + if (item.enclosures.length) { + item.enclosures[0].mURL = origEncUrl; + } else { + item.enclosures.push(new lazy.FeedEnclosure(origEncUrl)); + } + } + + // Handle atom threading extension, RFC4685. There may be 1 or more tags, + // and each must contain a ref attribute with 1 Message-Id equivalent + // value. This is the only attr of interest in the spec for presentation. + tags = this.childrenByTagNameNS( + itemNode, + lazy.FeedUtils.ATOM_THREAD_NS, + "in-reply-to" + ); + if (tags) { + for (let tag of tags) { + let ref = this.removeUnprintableASCII(tag.getAttribute("ref")); + if (ref) { + item.inReplyTo += item.normalizeMessageID(ref) + " "; + } + } + item.inReplyTo = item.inReplyTo.trimRight(); + } + + // Support <category> and autotagging. + tags = this.childrenByTagNameNS( + itemNode, + lazy.FeedUtils.ATOM_IETF_NS, + "category" + ); + if (tags) { + for (let tag of tags) { + let term = this.removeUnprintableASCII(tag.getAttribute("term")); + term = term ? this.xmlUnescape(term.replace(/,/g, ";")).trim() : null; + if (term && !item.keywords.includes(term)) { + item.keywords.push(term); + } + } + } + + this.parsedItems.push(item); + } + + return this.parsedItems; + }, + + isPermanentRedirect(aFeed, aRedirDocChannel, aFeedChannel) { + // If subscribing to a new feed, do not check redirect tags. + if (!aFeed.downloadCallback || aFeed.downloadCallback.mSubscribeMode) { + return false; + } + + let tags, tagName, newUrl; + let oldUrl = aFeed.url; + + // Check for RSS2.0 redirect document <newLocation> tag. + if (aRedirDocChannel) { + tagName = "newLocation"; + tags = this.childrenByTagNameNS(aRedirDocChannel, "", tagName); + newUrl = this.getNodeValue(tags ? tags[0] : null); + } + + // Check for <itunes:new-feed-url> tag. + if (aFeedChannel) { + tagName = "new-feed-url"; + tags = this.childrenByTagNameNS( + aFeedChannel, + lazy.FeedUtils.ITUNES_NS, + tagName + ); + newUrl = this.getNodeValue(tags ? tags[0] : null); + tagName = "itunes:" + tagName; + } + + if ( + newUrl && + newUrl != oldUrl && + lazy.FeedUtils.isValidScheme(newUrl) && + lazy.FeedUtils.changeUrlForFeed(aFeed, newUrl) + ) { + lazy.FeedUtils.log.info( + "FeedParser.isPermanentRedirect: found <" + + tagName + + "> tag; updated feed url from: " + + oldUrl + + " to: " + + newUrl + + " in folder: " + + lazy.FeedUtils.getFolderPrettyPath(aFeed.folder) + ); + aFeed.onUrlChange(aFeed, oldUrl); + return true; + } + + return false; + }, + + serializeTextConstruct(textElement) { + let content = ""; + if (textElement) { + let textType = textElement.getAttribute("type"); + + // Atom spec says consider it "text" if not present. + if (!textType) { + textType = "text"; + } + + // There could be some strange content type we don't handle. + if (textType != "text" && textType != "html" && textType != "xhtml") { + return null; + } + + for (let node of textElement.childNodes) { + if (node.nodeType == node.CDATA_SECTION_NODE) { + content += this.xmlEscape(node.data); + } else { + content += this.mSerializer.serializeToString(node); + } + } + + if (textType == "html") { + content = this.xmlUnescape(content); + } + + content = content.trim(); + } + + // Other parts of the code depend on this being null if there's no content. + return content ? content : null; + }, + + /** + * Return a cleaned up author name value. + * + * @param {string} authorString - A string. + * @returns {String} - A clean string value. + */ + cleanAuthorName(authorString) { + if (!authorString) { + return ""; + } + lazy.FeedUtils.log.trace( + "FeedParser.cleanAuthor: author1 - " + authorString + ); + let author = authorString + .replace(/[\n\r\t]+/g, " ") + .replace(/"/g, '\\"') + .trim(); + // If the name contains special chars, quote it. + if (author.match(/[<>@,"]/)) { + author = '"' + author + '"'; + } + lazy.FeedUtils.log.trace("FeedParser.cleanAuthor: author2 - " + author); + + return author; + }, + + /** + * Return a cleaned up node value. This is intended for values that are not + * multiline and not formatted. A sequence of tab or newline is converted to + * a space and unprintable ascii is removed. + * + * @param {Node} node - A DOM node. + * @returns {String} - A clean string value or null. + */ + getNodeValue(node) { + let nodeValue = this.getNodeValueRaw(node); + if (!nodeValue) { + return null; + } + + nodeValue = nodeValue.replace(/[\n\r\t]+/g, " "); + return this.removeUnprintableASCII(nodeValue); + }, + + /** + * Return a cleaned up formatted node value, meaning CR/LF/TAB are retained + * while all other unprintable ascii is removed. This is intended for values + * that are multiline and formatted, such as content or description tags. + * + * @param {Node} node - A DOM node. + * @returns {String} - A clean string value or null. + */ + getNodeValueFormatted(node) { + let nodeValue = this.getNodeValueRaw(node); + if (!nodeValue) { + return null; + } + + return this.removeUnprintableASCIIexCRLFTAB(nodeValue); + }, + + /** + * Return a raw node value, as received. This should be sanitized as + * appropriate. + * + * @param {Node} node - A DOM node. + * @returns {String} - A string value or null. + */ + getNodeValueRaw(node) { + if (node && node.textContent) { + return node.textContent.trim(); + } + + if (node && node.firstChild) { + let ret = ""; + for (let child = node.firstChild; child; child = child.nextSibling) { + let value = this.getNodeValueRaw(child); + if (value) { + ret += value; + } + } + + if (ret) { + return ret.trim(); + } + } + + return null; + }, + + // Finds elements that are direct children of the first arg. + childrenByTagNameNS(aElement, aNamespace, aTagName) { + if (!aElement) { + return null; + } + + let matches = aElement.getElementsByTagNameNS(aNamespace, aTagName); + let matchingChildren = []; + for (let match of matches) { + if (match.parentNode == aElement) { + matchingChildren.push(match); + } + } + + return matchingChildren.length ? matchingChildren : null; + }, + + /** + * Returns first matching descendent of element, or null. + * + * @param {Element} element - DOM element to search. + * @param {string} namespace - Namespace of the search tag. + * @param {String} tagName - Tag to search for. + * @returns {Element|null} - Matching element, or null. + */ + childByTagNameNS(element, namespace, tagName) { + if (!element) { + return null; + } + // Handily, item() returns null for out-of-bounds access. + return element.getElementsByTagNameNS(namespace, tagName).item(0); + }, + + /** + * Ensure <link> type tags start with http[s]://, ftp:// or magnet: + * for values stored in mail headers (content-base and remote enclosures), + * particularly to prevent data: uris, javascript, and other spoofing. + * + * @param {string} link - An intended http url string. + * @returns {String} - A clean string starting with http, ftp or magnet, + * else null. + */ + validLink(link) { + if (/^((https?|ftp):\/\/|magnet:)/.test(link)) { + return this.removeUnprintableASCII(link.trim()); + } + + return null; + }, + + /** + * Return an absolute link for <entry> relative links. If xml:base is + * present in a <feed> attribute or child <link> element attribute, use it; + * otherwise the Feed.link will be the relevant <feed> child <link> value + * and will be the |baseURI| for <entry> child <link>s if there is no further + * xml:base, which may be an attribute of any element. + * + * @param {string} linkRel - the <link> rel attribute value to find. + * @param {NodeList} linkElements - the nodelist of <links> to search in. + * @param {string} baseURI - the url to use when resolving relative + * links to absolute values. + * @returns {String} or null - absolute url for a <link>, or null if the + * rel type is not found. + */ + findAtomLink(linkRel, linkElements, baseURI) { + if (!linkElements) { + return null; + } + + // XXX Need to check for MIME type and hreflang. + for (let alink of linkElements) { + if ( + alink && + // If there's a link rel. + ((alink.getAttribute("rel") && alink.getAttribute("rel") == linkRel) || + // If there isn't, assume 'alternate'. + (!alink.getAttribute("rel") && linkRel == "alternate")) && + alink.getAttribute("href") + ) { + // Atom links are interpreted relative to xml:base. + let href = alink.getAttribute("href"); + baseURI = alink.getAttribute("xml:base") || baseURI || href; + try { + return Services.io.newURI(baseURI).resolve(href); + } catch (ex) {} + } + } + + return null; + }, + + /** + * Find RSS Syndication extension tags. + * http://web.resource.org/rss/1.0/modules/syndication/ + * + * @param {Feed} aFeed - the feed object. + * @param {Node | String} aChannel - dom node for the <channel>. + * @returns {void} + */ + findSyUpdateTags(aFeed, aChannel) { + let tag, updatePeriod, updateFrequency, updateBase; + tag = this.childrenByTagNameNS( + aChannel, + lazy.FeedUtils.RSS_SY_NS, + "updatePeriod" + ); + updatePeriod = this.getNodeValue(tag ? tag[0] : null) || ""; + tag = this.childrenByTagNameNS( + aChannel, + lazy.FeedUtils.RSS_SY_NS, + "updateFrequency" + ); + updateFrequency = this.getNodeValue(tag ? tag[0] : null) || ""; + tag = this.childrenByTagNameNS( + aChannel, + lazy.FeedUtils.RSS_SY_NS, + "updateBase" + ); + updateBase = this.getNodeValue(tag ? tag[0] : null) || ""; + lazy.FeedUtils.log.debug( + "FeedParser.findSyUpdateTags: updatePeriod:updateFrequency - " + + updatePeriod + + ":" + + updateFrequency + ); + + if (updatePeriod) { + if (lazy.FeedUtils.RSS_SY_UNITS.includes(updatePeriod.toLowerCase())) { + updatePeriod = updatePeriod.toLowerCase(); + } else { + updatePeriod = "daily"; + } + } + + updateFrequency = isNaN(updateFrequency) ? 1 : updateFrequency; + + let options = aFeed.options; + if ( + options.updates.updatePeriod == updatePeriod && + options.updates.updateFrequency == updateFrequency && + options.updates.updateBase == updateBase + ) { + return; + } + + options.updates.updatePeriod = updatePeriod; + options.updates.updateFrequency = updateFrequency; + options.updates.updateBase = updateBase; + aFeed.options = options; + }, + + /** + * Remove unprintable ascii, particularly CR/LF, for non formatted tag values. + * + * @param {string} s - String to clean. + * @returns {String} - Cleaned string. + */ + removeUnprintableASCII(s) { + /* eslint-disable-next-line no-control-regex */ + return s ? s.replace(/[\x00-\x1F\x7F]+/g, "") : ""; + }, + + /** + * Remove unprintable ascii, except CR/LF/TAB, for formatted tag values. + * + * @param {string} s - String to clean. + * @returns {String} - Cleaned string. + */ + removeUnprintableASCIIexCRLFTAB(s) { + /* eslint-disable-next-line no-control-regex */ + return s ? s.replace(/[\x00-\x08\x0B-\x0C\x0E-\x1F\x7F]+/g, "") : ""; + }, + + stripTags(someHTML) { + return someHTML ? someHTML.replace(/<[^>]+>/g, "") : someHTML; + }, + + xmlUnescape(s) { + s = s.replace(/</g, "<"); + s = s.replace(/>/g, ">"); + s = s.replace(/&/g, "&"); + return s; + }, + + xmlEscape(s) { + s = s.replace(/&/g, "&"); + s = s.replace(/>/g, ">"); + s = s.replace(/</g, "<"); + return s; + }, + + dateRescue(dateString) { + // Deal with various kinds of invalid dates. + if (!isNaN(parseInt(dateString))) { + // It's an integer, so maybe it's a timestamp. + let d = new Date(parseInt(dateString) * 1000); + let now = new Date(); + let yeardiff = now.getFullYear() - d.getFullYear(); + lazy.FeedUtils.log.trace( + "FeedParser.dateRescue: Rescue Timestamp date - " + + d.toString() + + " ,year diff - " + + yeardiff + ); + if (yeardiff >= 0 && yeardiff < 3) { + // It's quite likely the correct date. + return d.toString(); + } + } + + // Could be an ISO8601/W3C date. If not, get the current time. + return lazy.FeedUtils.getValidRFC5322Date(dateString); + }, +}; diff --git a/comm/mailnews/extensions/newsblog/FeedUtils.jsm b/comm/mailnews/extensions/newsblog/FeedUtils.jsm new file mode 100644 index 0000000000..b881c358d8 --- /dev/null +++ b/comm/mailnews/extensions/newsblog/FeedUtils.jsm @@ -0,0 +1,2136 @@ +/* -*- Mode: JavaScript; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const EXPORTED_SYMBOLS = ["FeedUtils"]; + +const { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); +const { MailServices } = ChromeUtils.import( + "resource:///modules/MailServices.jsm" +); + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + JSONFile: "resource://gre/modules/JSONFile.sys.mjs", + PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs", +}); + +XPCOMUtils.defineLazyModuleGetters(lazy, { + Feed: "resource:///modules/Feed.jsm", + jsmime: "resource:///modules/jsmime.jsm", + MailUtils: "resource:///modules/MailUtils.jsm", +}); + +var FeedUtils = { + MOZ_PARSERERROR_NS: "http://www.mozilla.org/newlayout/xml/parsererror.xml", + + RDF_SYNTAX_NS: "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + RDF_SYNTAX_TYPE: "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", + RSS_090_NS: "http://my.netscape.com/rdf/simple/0.9/", + + RSS_NS: "http://purl.org/rss/1.0/", + + RSS_CONTENT_NS: "http://purl.org/rss/1.0/modules/content/", + + RSS_SY_NS: "http://purl.org/rss/1.0/modules/syndication/", + RSS_SY_UNITS: ["hourly", "daily", "weekly", "monthly", "yearly"], + kBiffUnitsMinutes: "min", + kBiffUnitsDays: "d", + + DC_NS: "http://purl.org/dc/elements/1.1/", + + MRSS_NS: "http://search.yahoo.com/mrss/", + FEEDBURNER_NS: "http://rssnamespace.org/feedburner/ext/1.0", + ITUNES_NS: "http://www.itunes.com/dtds/podcast-1.0.dtd", + + FZ_NS: "urn:forumzilla:", + FZ_ITEM_NS: "urn:feeditem:", + + // Atom constants + ATOM_03_NS: "http://purl.org/atom/ns#", + ATOM_IETF_NS: "http://www.w3.org/2005/Atom", + ATOM_THREAD_NS: "http://purl.org/syndication/thread/1.0", + + // Accept content mimetype preferences for feeds. + REQUEST_ACCEPT: + "application/atom+xml," + + "application/rss+xml;q=0.9," + + "application/rdf+xml;q=0.8," + + "application/xml;q=0.7,text/xml;q=0.7," + + "*/*;q=0.1", + // Timeout for nonresponse to request, 30 seconds. + REQUEST_TIMEOUT: 30 * 1000, + + MILLISECONDS_PER_DAY: 24 * 60 * 60 * 1000, + + // Maximum number of concurrent in progress feeds, across all accounts. + kMaxConcurrentFeeds: 25, + get MAX_CONCURRENT_FEEDS() { + let pref = "rss.max_concurrent_feeds"; + if (Services.prefs.prefHasUserValue(pref)) { + return Services.prefs.getIntPref(pref); + } + + Services.prefs.setIntPref(pref, FeedUtils.kMaxConcurrentFeeds); + return FeedUtils.kMaxConcurrentFeeds; + }, + + // The amount of time, specified in milliseconds, to leave an item in the + // feeditems cache after the item has disappeared from the publisher's + // file. The default delay is one day. + kInvalidItemPurgeDelayDays: 1, + get INVALID_ITEM_PURGE_DELAY() { + let pref = "rss.invalid_item_purge_delay_days"; + if (Services.prefs.prefHasUserValue(pref)) { + return Services.prefs.getIntPref(pref) * this.MILLISECONDS_PER_DAY; + } + + Services.prefs.setIntPref(pref, FeedUtils.kInvalidItemPurgeDelayDays); + return FeedUtils.kInvalidItemPurgeDelayDays * this.MILLISECONDS_PER_DAY; + }, + + // Polling interval to check individual feed update interval preference. + kBiffPollMinutes: 1, + kNewsBlogSuccess: 0, + // Usually means there was an error trying to parse the feed. + kNewsBlogInvalidFeed: 1, + // Generic networking failure when trying to download the feed. + kNewsBlogRequestFailure: 2, + kNewsBlogFeedIsBusy: 3, + // For 304 Not Modified; There are no new articles for this feed. + kNewsBlogNoNewItems: 4, + kNewsBlogCancel: 5, + kNewsBlogFileError: 6, + // Invalid certificate, for overridable user exception errors. + kNewsBlogBadCertError: 7, + // For 401 Unauthorized or 403 Forbidden. + kNewsBlogNoAuthError: 8, + + CANCEL_REQUESTED: false, + AUTOTAG: "~AUTOTAG", + + FEED_ACCOUNT_TYPES: ["rss"], + + /** + * Get all rss account servers rootFolders. + * + * @returns {nsIMsgIncomingServer}[] - Array of servers (empty array if none). + */ + getAllRssServerRootFolders() { + let rssRootFolders = []; + for (let server of MailServices.accounts.allServers) { + if (server && server.type == "rss") { + rssRootFolders.push(server.rootFolder); + } + } + + // By default, Tb sorts by hostname, ie Feeds, Feeds-1, and not by alpha + // prettyName. Do the same as a stock install to match folderpane order. + rssRootFolders.sort(function (a, b) { + return a.hostname > b.hostname; + }); + + return rssRootFolders; + }, + + /** + * Create rss account. + * + * @param {String} aName - Optional account name to override default. + * @returns {nsIMsgAccount} - The creaged account. + */ + createRssAccount(aName) { + let userName = "nobody"; + let hostName = "Feeds"; + let hostNamePref = hostName; + let server; + let serverType = "rss"; + let defaultName = FeedUtils.strings.GetStringFromName("feeds-accountname"); + let i = 2; + while (MailServices.accounts.findServer(userName, hostName, serverType)) { + // If "Feeds" exists, try "Feeds-2", then "Feeds-3", etc. + hostName = hostNamePref + "-" + i++; + } + + server = MailServices.accounts.createIncomingServer( + userName, + hostName, + serverType + ); + server.biffMinutes = FeedUtils.kBiffPollMinutes; + server.prettyName = aName ? aName : defaultName; + server.valid = true; + let account = MailServices.accounts.createAccount(); + account.incomingServer = server; + // Initialize the feed_options now. + this.getOptionsAcct(server); + + // Ensure the Trash folder db (.msf) is created otherwise folder/message + // deletes will throw until restart creates it. + server.msgStore.discoverSubFolders(server.rootMsgFolder, false); + + // Save new accounts in case of a crash. + try { + MailServices.accounts.saveAccountInfo(); + } catch (ex) { + this.log.error( + "FeedUtils.createRssAccount: error on saveAccountInfo - " + ex + ); + } + + this.log.debug( + "FeedUtils.createRssAccount: " + + account.incomingServer.rootFolder.prettyName + ); + + return account; + }, + + /** + * Helper routine that checks our subscriptions list array and returns + * true if the url is already in our list. This is used to prevent the + * user from subscribing to the same feed multiple times for the same server. + * + * @param {string} aUrl - The url. + * @param {nsIMsgIncomingServer} aServer - Account server. + * @returns {Boolean} - true if exists else false. + */ + feedAlreadyExists(aUrl, aServer) { + let ds = this.getSubscriptionsDS(aServer); + let sub = ds.data.find(x => x.url == aUrl); + if (sub === undefined) { + return false; + } + let folder = sub.destFolder; + this.log.info( + "FeedUtils.feedAlreadyExists: feed url " + + aUrl + + " subscribed in folder url " + + decodeURI(folder) + ); + + return true; + }, + + /** + * Download a feed url on biff or get new messages. + * + * @param {nsIMsgFolder} aFolder - The folder. + * @param {nsIUrlListener} aUrlListener - Feed url. + * @param {Boolean} aIsBiff - true if biff, false if manual get. + * @param {nsIDOMWindow} aMsgWindow - The window. + * + * @returns {Promise<void>} When all feeds downloading have been set off. + */ + async downloadFeed(aFolder, aUrlListener, aIsBiff, aMsgWindow) { + FeedUtils.log.debug( + "downloadFeed: account isBiff:isOffline - " + + aIsBiff + + " : " + + Services.io.offline + ); + // User set. + if (Services.io.offline) { + return; + } + + // No network connection. Unfortunately, this is only set if the event is + // received by Tb from the OS (ie it must already be running) and doesn't + // necessarily mean connectivity to the internet, only the nearest network + // point. But it's something. + if (!Services.io.connectivity) { + FeedUtils.log.warn("downloadFeed: network connection unavailable"); + return; + } + + // We don't yet support the ability to check for new articles while we are + // in the middle of subscribing to a feed. For now, abort the check for + // new feeds. + if (FeedUtils.progressNotifier.mSubscribeMode) { + FeedUtils.log.warn( + "downloadFeed: Aborting RSS New Mail Check. " + + "Feed subscription in progress\n" + ); + return; + } + + let forceDownload = !aIsBiff; + let inStartup = false; + if (aFolder.isServer) { + // The lastUpdateTime is |null| only at session startup/initialization. + // Note: feed processing does not impact startup, as the biff poll + // will go off in about kBiffPollMinutes (1) and process each feed + // according to its own lastUpdatTime/update frequency. + if (FeedUtils.getStatus(aFolder, aFolder.URI).lastUpdateTime === null) { + inStartup = true; + } + + FeedUtils.setStatus(aFolder, aFolder.URI, "lastUpdateTime", Date.now()); + } + + let allFolders = aFolder.descendants; + if (!aFolder.isServer) { + // Add the base folder; it does not get returned by .descendants. Do not + // add the account folder as it doesn't have the feedUrl property or even + // a msgDatabase necessarily. + allFolders.unshift(aFolder); + } + + let folder; + async function* feeder() { + for (let i = 0; i < allFolders.length; i++) { + folder = allFolders[i]; + FeedUtils.log.debug( + "downloadFeed: START x/# folderName:folderPath - " + + (i + 1) + + "/" + + allFolders.length + + " " + + folder.name + + " : " + + folder.filePath.path + ); + + let feedUrlArray = FeedUtils.getFeedUrlsInFolder(folder); + // Continue if there are no feedUrls for the folder in the feeds + // database. All folders in Trash are skipped. + if (!feedUrlArray) { + continue; + } + + FeedUtils.log.debug( + "downloadFeed: CONTINUE foldername:urlArray - " + + folder.name + + " : " + + feedUrlArray + ); + + // We need to kick off a download for each feed. + let now = Date.now(); + for (let url of feedUrlArray) { + // Check whether this feed should be updated; if forceDownload is true + // skip the per feed check. + let status = FeedUtils.getStatus(folder, url); + if (!forceDownload) { + // Also skip if user paused, or error paused (but not inStartup; + // go check the feed again), or update interval hasn't expired. + if ( + status.enabled === false || + (status.enabled === null && !inStartup) || + now - status.lastUpdateTime < status.updateMinutes * 60000 + ) { + FeedUtils.log.debug( + "downloadFeed: SKIP feed, " + + "aIsBiff:enabled:minsSinceLastUpdate::url - " + + aIsBiff + + " : " + + status.enabled + + " : " + + Math.round((now - status.lastUpdateTime) / 60) / 1000 + + " :: " + + url + ); + continue; + } + } + // Update feed icons only once every 24h, tops. + if ( + forceDownload || + now - status.lastUpdateTime >= 24 * 60 * 60 * 1000 + ) { + await FeedUtils.getFavicon(folder, url); + } + + // Create a feed object. + let feed = new lazy.Feed(url, folder); + + // init can be called multiple times. Checks if it should actually + // init itself. + FeedUtils.progressNotifier.init(aMsgWindow, false); + + // Bump our pending feed download count. From now on, all feeds will + // be resolved and finish with progressNotifier.downloaded(). Any + // early returns must call downloaded() so mNumPendingFeedDownloads + // is decremented and notification/status feedback is reset. + FeedUtils.progressNotifier.mNumPendingFeedDownloads++; + + // If the current active count exceeds the max desired, exit from + // the current poll cycle. Only throttle for a background biff; for + // a user manual get messages, do them all. + if ( + aIsBiff && + FeedUtils.progressNotifier.mNumPendingFeedDownloads > + FeedUtils.MAX_CONCURRENT_FEEDS + ) { + FeedUtils.log.debug( + "downloadFeed: RETURN active feeds count is greater " + + "than the max - " + + FeedUtils.MAX_CONCURRENT_FEEDS + ); + FeedUtils.progressNotifier.downloaded( + feed, + FeedUtils.kNewsBlogFeedIsBusy + ); + return; + } + + // Set status info and download. + FeedUtils.log.debug("downloadFeed: DOWNLOAD feed url - " + url); + FeedUtils.setStatus( + folder, + url, + "code", + FeedUtils.kNewsBlogFeedIsBusy + ); + feed.download(true, FeedUtils.progressNotifier); + + Services.tm.mainThread.dispatch(function () { + try { + let done = getFeed.next().done; + if (done) { + // Finished with all feeds in base aFolder and its subfolders. + FeedUtils.log.debug( + "downloadFeed: Finished with folder - " + aFolder.name + ); + folder = null; + allFolders = null; + } + } catch (ex) { + FeedUtils.log.error("downloadFeed: error - " + ex); + FeedUtils.progressNotifier.downloaded( + feed, + FeedUtils.kNewsBlogFeedIsBusy + ); + } + }, Ci.nsIThread.DISPATCH_NORMAL); + + yield undefined; + } + } + } + + let getFeed = await feeder(); + try { + let done = getFeed.next().done; + if (done) { + // Nothing to do. + FeedUtils.log.debug( + "downloadFeed: Nothing to do in folder - " + aFolder.name + ); + folder = null; + allFolders = null; + } + } catch (ex) { + FeedUtils.log.error("downloadFeed: error - " + ex); + FeedUtils.progressNotifier.downloaded( + { folder: aFolder, url: "" }, + FeedUtils.kNewsBlogFeedIsBusy + ); + } + }, + + /** + * Subscribe a new feed url. + * + * @param {String} aUrl - Feed url. + * @param {nsIMsgFolder} aFolder - Folder. + * + * @returns {void} + */ + subscribeToFeed(aUrl, aFolder) { + // We don't support the ability to subscribe to several feeds at once yet. + // For now, abort the subscription if we are already in the middle of + // subscribing to a feed via drag and drop. + if (FeedUtils.progressNotifier.mNumPendingFeedDownloads > 0) { + FeedUtils.log.warn( + "subscribeToFeed: Aborting RSS subscription. " + + "Feed downloads already in progress\n" + ); + return; + } + + // If aFolder is null, then use the root folder for the first RSS account. + if (!aFolder) { + aFolder = FeedUtils.getAllRssServerRootFolders()[0]; + } + + // If the user has no Feeds account yet, create one. + if (!aFolder) { + aFolder = FeedUtils.createRssAccount().incomingServer.rootFolder; + } + + // If aUrl is a feed url, then it is either of the form + // feed://example.org/feed.xml or feed:https://example.org/feed.xml. + // Replace feed:// with http:// per the spec, then strip off feed: + // for the second case. + aUrl = aUrl.replace(/^feed:\x2f\x2f/i, "http://"); + aUrl = aUrl.replace(/^feed:/i, ""); + + let msgWindow = Services.wm.getMostRecentWindow("mail:3pane").msgWindow; + + // Make sure we aren't already subscribed to this feed before we attempt + // to subscribe to it. + if (FeedUtils.feedAlreadyExists(aUrl, aFolder.server)) { + msgWindow.statusFeedback.showStatusString( + FeedUtils.strings.GetStringFromName("subscribe-feedAlreadySubscribed") + ); + return; + } + + let feed = new lazy.Feed(aUrl, aFolder); + // Default setting for new feeds per account settings. + feed.quickMode = feed.server.getBoolValue("quickMode"); + feed.options = FeedUtils.getOptionsAcct(feed.server); + + FeedUtils.progressNotifier.init(msgWindow, true); + FeedUtils.progressNotifier.mNumPendingFeedDownloads++; + feed.download(true, FeedUtils.progressNotifier); + }, + + /** + * Enable or disable updates for all subscriptions in a folder, or all + * subscriptions in an account if the folder is the account folder. + * A folder's subfolders' feeds are not included. + * + * @param {nsIMsgFolder} aFolder - Folder or account folder (server). + * @param {boolean} aPause - To pause or not to pause. + * @param {Boolean} aBiffNow - If aPause is false, and aBiffNow is true + * do the biff immediately. + * @returns {void} + */ + pauseFeedFolderUpdates(aFolder, aPause, aBiffNow) { + if (aFolder.isServer) { + let serverFolder = aFolder.server.rootFolder; + // Remove server from biff first. If enabling biff, this will make the + // latest biffMinutes take effect now rather than waiting for the timer + // to expire. + aFolder.server.doBiff = false; + if (!aPause) { + aFolder.server.doBiff = true; + } + + FeedUtils.setStatus(serverFolder, serverFolder.URI, "enabled", !aPause); + if (!aPause && aBiffNow) { + aFolder.server.performBiff(null); + } + + return; + } + + let feedUrls = FeedUtils.getFeedUrlsInFolder(aFolder); + if (!feedUrls) { + return; + } + + for (let feedUrl of feedUrls) { + let feed = new lazy.Feed(feedUrl, aFolder); + let options = feed.options; + options.updates.enabled = !aPause; + feed.options = options; + FeedUtils.setStatus(aFolder, feedUrl, "enabled", !aPause); + FeedUtils.log.debug( + "pauseFeedFolderUpdates: enabled:url " + !aPause + ": " + feedUrl + ); + } + + let win = Services.wm.getMostRecentWindow("Mail:News-BlogSubscriptions"); + if (win) { + let curItem = win.FeedSubscriptions.mView.currentItem; + win.FeedSubscriptions.refreshSubscriptionView(); + if (curItem.container) { + win.FeedSubscriptions.selectFolder(curItem.folder); + } else { + let feed = new lazy.Feed(curItem.url, curItem.parentFolder); + win.FeedSubscriptions.selectFeed(feed); + } + } + }, + + /** + * Add a feed record to the feeds database and update the folder's feedUrl + * property. + * + * @param {Feed} aFeed - Our feed object. + * + * @returns {void} + */ + addFeed(aFeed) { + // Find or create subscription entry. + let ds = this.getSubscriptionsDS(aFeed.server); + let sub = ds.data.find(x => x.url == aFeed.url); + if (sub === undefined) { + sub = {}; + ds.data.push(sub); + } + sub.url = aFeed.url; + sub.destFolder = aFeed.folder.URI; + if (aFeed.title) { + sub.title = aFeed.title; + } + ds.saveSoon(); + + // Update folderpane. + Services.obs.notifyObservers(aFeed.folder, "folder-properties-changed"); + }, + + /** + * Delete a feed record from the feeds database and update the folder's + * feedUrl property. + * + * @param {Feed} aFeed - Our feed object. + * + * @returns {void} + */ + deleteFeed(aFeed) { + // Remove items associated with this feed from the items db. + aFeed.invalidateItems(); + aFeed.removeInvalidItems(true); + + // Remove the entry in the subscriptions db. + let ds = this.getSubscriptionsDS(aFeed.server); + ds.data = ds.data.filter(x => x.url != aFeed.url); + ds.saveSoon(); + + // Update folderpane. + Services.obs.notifyObservers(aFeed.folder, "folder-properties-changed"); + }, + + /** + * Change an existing feed's url. + * + * @param {Feed} aFeed - The feed object. + * @param {string} aNewUrl - New url. + * + * @returns {Boolean} - true if successful, else false. + */ + changeUrlForFeed(aFeed, aNewUrl) { + if (!aFeed || !aFeed.folder || !aNewUrl) { + return false; + } + + if (this.feedAlreadyExists(aNewUrl, aFeed.server)) { + this.log.info( + "FeedUtils.changeUrlForFeed: new feed url " + + aNewUrl + + " already subscribed in account " + + aFeed.server.prettyName + ); + return false; + } + + let title = aFeed.title; + let link = aFeed.link; + let quickMode = aFeed.quickMode; + let options = aFeed.options; + + this.deleteFeed(aFeed); + aFeed.url = aNewUrl; + aFeed.title = title; + aFeed.link = link; + aFeed.quickMode = quickMode; + aFeed.options = options; + this.addFeed(aFeed); + + let win = Services.wm.getMostRecentWindow("Mail:News-BlogSubscriptions"); + if (win) { + win.FeedSubscriptions.refreshSubscriptionView(aFeed.folder, aNewUrl); + } + + return true; + }, + + /** + * Determine if a message is a feed message. Prior to Tb15, a message had to + * be in an rss acount type folder. In Tb15 and later, a flag is set on the + * message itself upon initial store; the message can be moved to any folder. + * + * @param {nsIMsgDBHdr} aMsgHdr - The message. + * + * @returns {Boolean} - true if message is a feed, false if not. + */ + isFeedMessage(aMsgHdr) { + return Boolean( + aMsgHdr instanceof Ci.nsIMsgDBHdr && + (aMsgHdr.flags & Ci.nsMsgMessageFlags.FeedMsg || + this.isFeedFolder(aMsgHdr.folder)) + ); + }, + + /** + * Determine if a folder is a feed acount folder. Trash or a folder in Trash + * should be checked with FeedUtils.isInTrash() if required. + * + * @param {nsIMsgFolder} aFolder - The folder. + * + * @returns {Boolean} - true if folder's server.type is in FEED_ACCOUNT_TYPES, + * false if not. + */ + isFeedFolder(aFolder) { + return Boolean( + aFolder instanceof Ci.nsIMsgFolder && + this.FEED_ACCOUNT_TYPES.includes(aFolder.server.type) + ); + }, + + /** + * Get the list of feed urls for a folder. + * + * @param {nsIMsgFolder} aFolder - The folder. + * + * @returns {String}[] - Array of urls, or null if none. + */ + getFeedUrlsInFolder(aFolder) { + if ( + !aFolder || + aFolder.isServer || + aFolder.server.type != "rss" || + aFolder.getFlag(Ci.nsMsgFolderFlags.Trash) || + aFolder.getFlag(Ci.nsMsgFolderFlags.Virtual) || + !aFolder.filePath.exists() + ) { + // There are never any feedUrls in the account/non-feed/trash/virtual + // folders or in a ghost folder (nonexistent on disk yet found in + // aFolder.subFolders). + return null; + } + + let feedUrlArray = []; + + // Get the list from the feeds database. + try { + let ds = this.getSubscriptionsDS(aFolder.server); + for (const sub of ds.data) { + if (sub.destFolder == aFolder.URI) { + feedUrlArray.push(sub.url); + } + } + } catch (ex) { + this.log.error("getFeedUrlsInFolder: feeds db error - " + ex); + this.log.error( + "getFeedUrlsInFolder: feeds db error for account - " + + aFolder.server.serverURI + + " : " + + aFolder.server.prettyName + ); + } + + return feedUrlArray.length ? feedUrlArray : null; + }, + + /** + * Check if the folder's msgDatabase is openable, reparse if desired. + * + * @param {nsIMsgFolder} aFolder - The folder. + * @param {boolean} aReparse - Reparse if true. + * @param {nsIUrlListener} aUrlListener - Object implementing nsIUrlListener. + * + * @returns {Boolean} - true if msgDb is available, else false + */ + isMsgDatabaseOpenable(aFolder, aReparse, aUrlListener) { + let msgDb; + try { + msgDb = Cc["@mozilla.org/msgDatabase/msgDBService;1"] + .getService(Ci.nsIMsgDBService) + .openFolderDB(aFolder, true); + } catch (ex) {} + + if (msgDb) { + return true; + } + + if (!aReparse) { + return false; + } + + // Force a reparse. + FeedUtils.log.debug( + "checkMsgDb: rebuild msgDatabase for " + + aFolder.name + + " - " + + aFolder.filePath.path + ); + try { + // Ignore error returns. + aFolder + .QueryInterface(Ci.nsIMsgLocalMailFolder) + .getDatabaseWithReparse(aUrlListener, null); + } catch (ex) {} + + return false; + }, + + /** + * Return properties for nsITreeView getCellProperties, for a tree row item in + * folderpane or subscribe dialog tree. + * + * @param {nsIMsgFolder} aFolder - Folder or a feed url's parent folder. + * @param {string} aFeedUrl - Feed url for a feed row, null for folder. + * + * @returns {string} - Space separated properties. + */ + getFolderProperties(aFolder, aFeedUrl) { + let folder = aFolder; + let feedUrls = aFeedUrl ? [aFeedUrl] : this.getFeedUrlsInFolder(aFolder); + if (!feedUrls && !folder.isServer) { + return ""; + } + + let serverEnabled = this.getStatus( + folder.server.rootFolder, + folder.server.rootFolder.URI + ).enabled; + if (folder.isServer) { + return !serverEnabled ? " isPaused" : ""; + } + + let properties = aFeedUrl ? " isFeed-true" : " isFeedFolder-true"; + let hasError, + isBusy, + numPaused = 0; + for (let feedUrl of feedUrls) { + let feedStatus = this.getStatus(folder, feedUrl); + if ( + feedStatus.code == FeedUtils.kNewsBlogInvalidFeed || + feedStatus.code == FeedUtils.kNewsBlogRequestFailure || + feedStatus.code == FeedUtils.kNewsBlogBadCertError || + feedStatus.code == FeedUtils.kNewsBlogNoAuthError + ) { + hasError = true; + } + if (feedStatus.code == FeedUtils.kNewsBlogFeedIsBusy) { + isBusy = true; + } + if (!feedStatus.enabled) { + numPaused++; + } + } + + properties += hasError ? " hasError" : ""; + properties += isBusy ? " isBusy" : ""; + properties += numPaused == feedUrls.length ? " isPaused" : ""; + + return properties; + }, + + /** + * Get a cached feed or folder status. + * + * @param {nsIMsgFolder} aFolder - Folder. + * @param {string} aUrl - Url key (feed url or folder URI). + * + * @returns {String} aValue - The value. + */ + getStatus(aFolder, aUrl) { + if (!aFolder || !aUrl) { + return null; + } + + let serverKey = aFolder.server.serverURI; + if (!this[serverKey]) { + this[serverKey] = {}; + } + + if (!this[serverKey][aUrl]) { + // Seed the status object. + this[serverKey][aUrl] = {}; + this[serverKey][aUrl].status = this.statusTemplate; + if (FeedUtils.isValidScheme(aUrl)) { + // Seed persisted status properties for feed urls. + let feed = new lazy.Feed(aUrl, aFolder); + this[serverKey][aUrl].status.enabled = feed.options.updates.enabled; + this[serverKey][aUrl].status.updateMinutes = + feed.options.updates.updateMinutes; + this[serverKey][aUrl].status.lastUpdateTime = + feed.options.updates.lastUpdateTime; + feed = null; + } else { + // Seed persisted status properties for servers. + let optionsAcct = FeedUtils.getOptionsAcct(aFolder.server); + this[serverKey][aUrl].status.enabled = optionsAcct.doBiff; + } + FeedUtils.log.debug("getStatus: seed url - " + aUrl); + } + + return this[serverKey][aUrl].status; + }, + + /** + * Update a feed or folder status and refresh folderpane. + * + * @param {nsIMsgFolder} aFolder - Folder. + * @param {string} aUrl - Url key (feed url or folder URI). + * @param {string} aProperty - Url status property. + * @param {string} aValue - Value. + * + * @returns {void} + */ + setStatus(aFolder, aUrl, aProperty, aValue) { + if (!aFolder || !aUrl || !aProperty) { + return; + } + + if ( + !this[aFolder.server.serverURI] || + !this[aFolder.server.serverURI][aUrl] + ) { + // Not yet seeded, so do it. + this.getStatus(aFolder, aUrl); + } + + this[aFolder.server.serverURI][aUrl].status[aProperty] = aValue; + + Services.obs.notifyObservers(aFolder, "folder-properties-changed"); + + let win = Services.wm.getMostRecentWindow("Mail:News-BlogSubscriptions"); + if (win) { + win.FeedSubscriptions.mView.tree.invalidate(); + } + }, + + /** + * Get the favicon for a feed folder subscription url (first one) or a feed + * message url. The favicon service caches it in memory if places history is + * not enabled. + * + * @param {?nsIMsgFolder} folder - The feed folder or null if url set. + * @param {?string} feedURL - A url (feed, message, other) or null if folder set. + * + * @returns {Promise<string>} - The favicon url or empty string. + */ + async getFavicon(folder, feedURL) { + if ( + !Services.prefs.getBoolPref("browser.chrome.site_icons") || + !Services.prefs.getBoolPref("browser.chrome.favicons") || + !Services.prefs.getBoolPref("places.history.enabled") + ) { + return ""; + } + + let url = feedURL; + if (!url) { + // Get the proposed iconUrl from the folder's first subscribed feed's + // <link>. + url = this.getFeedUrlsInFolder(folder)[0]; + if (!url) { + return ""; + } + feedURL = url; + } + + if (folder) { + let feed = new lazy.Feed(url, folder); + url = feed.link && feed.link.startsWith("http") ? feed.link : url; + } + + /** + * Convert a Blob to data URL. + * @param {Blob} blob - Blob to convert. + * @returns {string} data URL. + */ + let blobToBase64 = blob => { + return new Promise((resolve, reject) => { + let reader = new FileReader(); + reader.onloadend = () => { + if (reader.result.endsWith("base64,")) { + reject(new Error(`Invalid blob encountered.`)); + return; + } + resolve(reader.result); + }; + reader.readAsDataURL(blob); + }); + }; + + /** + * Try getting favicon from url. + * @param {string} url - The favicon url. + * @returns {Blob} - Existing favicon. + */ + let fetchFavicon = async url => { + let response = await fetch(url); + if (!response.ok) { + throw new Error(`No favicon for url ${url}`); + } + if (!/^image\//i.test(response.headers.get("Content-Type"))) { + throw new Error(`Non-image favicon for ${url}`); + } + return response.blob(); + }; + + /** + * Try getting favicon from the a html page. + * @param {string} page - The page url to check. + * @returns {Blob} - Found favicon. + */ + let discoverFaviconURL = async page => { + let response = await fetch(page); + if (!response.ok) { + throw new Error(`No favicon for page ${page}`); + } + if (!/^text\/html/i.test(response.headers.get("Content-Type"))) { + throw new Error(`No page to get favicon from for ${page}`); + } + let doc = new DOMParser().parseFromString( + await response.text(), + "text/html" + ); + let iconLink = doc.querySelector( + `link[href][rel~='icon']:is([sizes~='any'],[sizes~='16x16' i],[sizes~='32x32' i],:not([sizes])` + ); + if (!iconLink) { + throw new Error(`No iconLink discovered for page=${page}`); + } + if (/^https?:/.test(iconLink.href)) { + return iconLink.href; + } + if (iconLink.href.at(0) != "/") { + iconLink.href = "/" + iconLink.href; + } + return new URL(page).origin + iconLink.href; + }; + + let uri = Services.io.newURI(url); + let iconURL = await fetchFavicon(uri.prePath + "/favicon.ico") + .then(blobToBase64) + .catch(e => { + return discoverFaviconURL(url) + .catch(() => discoverFaviconURL(uri.prePath)) + .then(fetchFavicon) + .then(blobToBase64) + .catch(() => ""); + }); + + if (!iconURL) { + return ""; + } + + // setAndFetchFaviconForPage needs the url to be in the database first. + await lazy.PlacesUtils.history.insert({ + url: feedURL, + visits: [ + { + date: new Date(), + }, + ], + }); + return new Promise(resolve => { + // All good. Now store iconURL for future usage. + lazy.PlacesUtils.favicons.setAndFetchFaviconForPage( + Services.io.newURI(feedURL), + Services.io.newURI(iconURL), + true, + lazy.PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE, + faviconURI => { + resolve(faviconURI.spec); + }, + Services.scriptSecurityManager.getSystemPrincipal() + ); + }); + }, + + /** + * Update the feeds database for rename and move/copy folder name changes. + * + * @param {nsIMsgFolder} aFolder - The folder, new if rename or target of + * move/copy folder (new parent). + * @param {nsIMsgFolder} aOrigFolder - Original folder. + * @param {String} aAction - "move" or "copy" or "rename". + * + * @returns {void} + */ + updateSubscriptionsDS(aFolder, aOrigFolder, aAction) { + this.log.debug( + "FeedUtils.updateSubscriptionsDS: " + + "\nfolder changed - " + + aAction + + "\nnew folder - " + + aFolder.filePath.path + + "\norig folder - " + + aOrigFolder.filePath.path + ); + + this.log.debug( + `updateSubscriptions(${aFolder.name}, ${aOrigFolder.name}, ${aAction})` + ); + + if (aFolder.server.type != "rss" || FeedUtils.isInTrash(aOrigFolder)) { + // Target not a feed account folder; nothing to do, or move/rename in + // trash; no subscriptions already. + return; + } + + let newFolder = aFolder; + let newParentURI = aFolder.URI; + let origParentURI = aOrigFolder.URI; + if (aAction == "move" || aAction == "copy") { + // Get the new folder. Don't process the entire parent (new dest folder)! + newFolder = aFolder.getChildNamed(aOrigFolder.name); + origParentURI = aOrigFolder.parent + ? aOrigFolder.parent.URI + : aOrigFolder.rootFolder.URI; + } + + this.updateFolderChangeInFeedsDS(newFolder, aOrigFolder, null, null); + + // There may be subfolders, but we only get a single notification; iterate + // over all descendent folders of the folder whose location has changed. + for (let newSubFolder of newFolder.descendants) { + FeedUtils.updateFolderChangeInFeedsDS( + newSubFolder, + aOrigFolder, + newParentURI, + origParentURI + ); + } + }, + + /** + * Update the feeds database with the new folder's or subfolder's location + * for rename and move/copy name changes. The feeds subscriptions db is + * also synced on cross account folder copies. Note that if a copied folder's + * url exists in the new account, its active subscription will be switched to + * the folder being copied, to enforce the one unique url per account design. + * + * @param {nsIMsgFolder} aFolder - New folder. + * @param {nsIMsgFolder} aOrigFolder - Original folder. + * @param {string} aNewAncestorURI - For subfolders, ancestor new folder. + * @param {String} aOrigAncestorURI - For subfolders, ancestor original folder. + * + * @returns {void} + */ + updateFolderChangeInFeedsDS( + aFolder, + aOrigFolder, + aNewAncestorURI, + aOrigAncestorURI + ) { + this.log.debug( + "updateFolderChangeInFeedsDS: " + + "\naFolder - " + + aFolder.URI + + "\naOrigFolder - " + + aOrigFolder.URI + + "\naOrigAncestor - " + + aOrigAncestorURI + + "\naNewAncestor - " + + aNewAncestorURI + ); + + // Get the original folder's URI. + let folderURI = aFolder.URI; + let origURI = + aNewAncestorURI && aOrigAncestorURI + ? folderURI.replace(aNewAncestorURI, aOrigAncestorURI) + : aOrigFolder.URI; + this.log.debug("updateFolderChangeInFeedsDS: urls origURI - " + origURI); + + // Get affected feed subscriptions - all the ones in the original folder. + let origDS = this.getSubscriptionsDS(aOrigFolder.server); + let affectedSubs = origDS.data.filter(sub => sub.destFolder == origURI); + if (affectedSubs.length == 0) { + this.log.debug("updateFolderChangeInFeedsDS: no feedUrls in this folder"); + return; + } + + if (this.isInTrash(aFolder)) { + // Moving to trash. Unsubscribe. + affectedSubs.forEach(function (sub) { + let feed = new lazy.Feed(sub.url, aFolder); + FeedUtils.deleteFeed(feed); + }); + // note: deleteFeed() calls saveSoon(), so we don't need to. + } else if (aFolder.server == aOrigFolder.server) { + // Staying in same account - just update destFolder as required + for (let sub of affectedSubs) { + sub.destFolder = folderURI; + } + origDS.saveSoon(); + } else { + // Moving between accounts. + let destDS = this.getSubscriptionsDS(aFolder.server); + for (let sub of affectedSubs) { + // Move to the new subscription db (replacing any existing entry). + origDS.data = origDS.data.filter(x => x.url != sub.url); + destDS.data = destDS.data.filter(x => x.url != sub.url); + sub.destFolder = folderURI; + destDS.data.push(sub); + } + + origDS.saveSoon(); + destDS.saveSoon(); + } + }, + + /** + * When subscribing to feeds by dnd on, or adding a url to, the account + * folder (only), or creating folder structure via opml import, a subfolder is + * autocreated and thus the derived/given name must be sanitized to prevent + * filesystem errors. Hashing invalid chars based on OS rather than filesystem + * is not strictly correct. + * + * @param {nsIMsgFolder} aParentFolder - Parent folder. + * @param {String} aProposedName - Proposed name. + * @param {String} aDefaultName - Default name if proposed sanitizes to + * blank, caller ensures sane value. + * @param {Boolean} aUnique - If true, return a unique indexed name. + * + * @returns {String} - Sanitized unique name. + */ + getSanitizedFolderName(aParentFolder, aProposedName, aDefaultName, aUnique) { + // Clean up the name for the strictest fs (fat) and to ensure portability. + // 1) Replace line breaks and tabs '\n\r\t' with a space. + // 2) Remove nonprintable ascii. + // 3) Remove invalid win chars '* | \ / : < > ? "'. + // 4) Remove all '.' as starting/ending with one is trouble on osx/win. + // 5) No leading/trailing spaces. + /* eslint-disable no-control-regex */ + let folderName = aProposedName + .replace(/[\n\r\t]+/g, " ") + .replace(/[\x00-\x1F]+/g, "") + .replace(/[*|\\\/:<>?"]+/g, "") + .replace(/[\.]+/g, "") + .trim(); + /* eslint-enable no-control-regex */ + + // Prefix with __ if name is: + // 1) a reserved win filename. + // 2) an undeletable/unrenameable special folder name (bug 259184). + if ( + folderName + .toUpperCase() + .match(/^COM\d$|^LPT\d$|^CON$|PRN$|^AUX$|^NUL$|^CLOCK\$/) || + folderName + .toUpperCase() + .match(/^INBOX$|^OUTBOX$|^UNSENT MESSAGES$|^TRASH$/) + ) { + folderName = "__" + folderName; + } + + // Use a default if no name is found. + if (!folderName) { + folderName = aDefaultName; + } + + if (!aUnique) { + return folderName; + } + + // Now ensure the folder name is not a dupe; if so append index. + let folderNameBase = folderName; + let i = 2; + while (aParentFolder.containsChildNamed(folderName)) { + folderName = folderNameBase + "-" + i++; + } + + return folderName; + }, + + /** + * This object contains feed/account status info. + */ + _statusDefault: { + // Derived from persisted value. + enabled: null, + // Last update result code, a kNewsBlog* value. + code: 0, + updateMinutes: null, + // JS Date; startup state is null indicating no update since startup. + lastUpdateTime: null, + }, + + get statusTemplate() { + // Copy the object. + return JSON.parse(JSON.stringify(this._statusDefault)); + }, + + /** + * This object will contain all persisted feed specific properties. + */ + _optionsDefault: { + version: 2, + updates: { + enabled: true, + // User set. + updateMinutes: 100, + // User set: "min"=minutes, "d"=days + updateUnits: "min", + // JS Date. + lastUpdateTime: null, + // The last time a new message was stored. JS Date. + lastDownloadTime: null, + // Publisher recommended from the feed. + updatePeriod: null, + updateFrequency: 1, + updateBase: null, + }, + // Autotag and <category> handling options. + category: { + enabled: false, + prefixEnabled: false, + prefix: null, + }, + }, + + get optionsTemplate() { + // Copy the object. + return JSON.parse(JSON.stringify(this._optionsDefault)); + }, + + getOptionsAcct(aServer) { + let optionsAcct; + let optionsAcctPref = "mail.server." + aServer.key + ".feed_options"; + let check_new_mail = "mail.server." + aServer.key + ".check_new_mail"; + let check_time = "mail.server." + aServer.key + ".check_time"; + + // Biff enabled or not. Make sure pref exists. + if (!Services.prefs.prefHasUserValue(check_new_mail)) { + Services.prefs.setBoolPref(check_new_mail, true); + } + + // System polling interval. Make sure pref exists. + if (!Services.prefs.prefHasUserValue(check_time)) { + Services.prefs.setIntPref(check_time, FeedUtils.kBiffPollMinutes); + } + + try { + optionsAcct = JSON.parse(Services.prefs.getCharPref(optionsAcctPref)); + // Add the server specific biff enabled state. + optionsAcct.doBiff = Services.prefs.getBoolPref(check_new_mail); + } catch (ex) {} + + if (optionsAcct && optionsAcct.version == this._optionsDefault.version) { + return optionsAcct; + } + + // Init account updates options if new or upgrading to version in + // |_optionsDefault.version|. + if (!optionsAcct || optionsAcct.version < this._optionsDefault.version) { + this.initAcct(aServer); + } + + let newOptions = this.newOptions(optionsAcct); + this.setOptionsAcct(aServer, newOptions); + newOptions.doBiff = Services.prefs.getBoolPref(check_new_mail); + return newOptions; + }, + + setOptionsAcct(aServer, aOptions) { + let optionsAcctPref = "mail.server." + aServer.key + ".feed_options"; + aOptions = aOptions || this.optionsTemplate; + Services.prefs.setCharPref(optionsAcctPref, JSON.stringify(aOptions)); + }, + + initAcct(aServer) { + let serverPrefStr = "mail.server." + aServer.key; + // System polling interval. Effective after current interval expires on + // change; no user facing ui. + Services.prefs.setIntPref( + serverPrefStr + ".check_time", + FeedUtils.kBiffPollMinutes + ); + + // If this pref is false, polling on biff is disabled and account updates + // are paused; ui in account server settings and folderpane context menu + // (Pause All Updates). Checking Enable updates or unchecking Pause takes + // effect immediately. Here on startup, we just ensure the polling interval + // above is reset immediately. + let doBiff = Services.prefs.getBoolPref(serverPrefStr + ".check_new_mail"); + FeedUtils.log.debug( + "initAcct: " + aServer.prettyName + " doBiff - " + doBiff + ); + this.pauseFeedFolderUpdates(aServer.rootFolder, !doBiff, false); + }, + + newOptions(aCurrentOptions) { + if (!aCurrentOptions) { + return this.optionsTemplate; + } + + // Options version has changed; meld current template with existing + // aCurrentOptions settings, removing items gone from the template while + // preserving user settings for retained template items. + let newOptions = this.optionsTemplate; + this.Mixins.meld(aCurrentOptions, false, true).into(newOptions); + newOptions.version = this.optionsTemplate.version; + return newOptions; + }, + + /** + * A generalized recursive melder of two objects. Getters/setters not included. + */ + Mixins: { + meld(source, keep, replace) { + function meldin(source, target, keep, replace) { + for (let attribute in source) { + // Recurse for objects. + if ( + typeof source[attribute] == "object" && + typeof target[attribute] == "object" + ) { + meldin(source[attribute], target[attribute], keep, replace); + } else { + // Use attribute values from source for the target, unless + // replace is false. + if (attribute in target && !replace) { + continue; + } + // Don't copy attribute from source to target if it is not in the + // target, unless keep is true. + if (!(attribute in target) && !keep) { + continue; + } + + target[attribute] = source[attribute]; + } + } + } + return { + source, + into(target) { + meldin(this.source, target, keep, replace); + }, + }; + }, + }, + + /** + * Returns a reference to the feed subscriptions store for the given server + * (the feeds.json data). + * + * @param {nsIMsgIncomingServer} aServer - server to fetch item data for. + * @returns {JSONFile} - a JSONFile holding the array of feed subscriptions + * in its data field. + */ + getSubscriptionsDS(aServer) { + if (this[aServer.serverURI] && this[aServer.serverURI].FeedsDS) { + return this[aServer.serverURI].FeedsDS; + } + + let rssServer = aServer.QueryInterface(Ci.nsIRssIncomingServer); + let feedsFile = rssServer.subscriptionsPath; // Path to feeds.json + let exists = feedsFile.exists(); + let ds = new lazy.JSONFile({ + path: feedsFile.path, + backupTo: feedsFile.path + ".backup", + }); + ds.ensureDataReady(); + if (!this[aServer.serverURI]) { + this[aServer.serverURI] = {}; + } + this[aServer.serverURI].FeedsDS = ds; + if (!exists) { + // No feeds.json, so we need to initialise. + ds.data = []; + } + return ds; + }, + + /** + * Fetch an attribute for a subscribed feed. + * + * @param {string} feedURL - URL of the feed. + * @param {nsIMsgIncomingServer} server - Server holding the subscription. + * @param {String} attrName - Name of attribute to fetch. + * @param {undefined} defaultValue - Value to return if not found. + * + * @returns {undefined} - the fetched value (defaultValue if the + * subscription or attribute doesn't exist). + */ + getSubscriptionAttr(feedURL, server, attrName, defaultValue) { + let ds = this.getSubscriptionsDS(server); + let sub = ds.data.find(feed => feed.url == feedURL); + if (sub === undefined || sub[attrName] === undefined) { + return defaultValue; + } + return sub[attrName]; + }, + + /** + * Set an attribute for a feed in the subscriptions store. + * NOTE: If the feed is not already in the store, it will be + * added. + * + * @param {string} feedURL - URL of the feed. + * @param {nsIMsgIncomingServer} server - server holding subscription. + * @param {String} attrName - Name of attribute to fetch. + * @param {undefined} value - Value to store. + * + * @returns {void} + */ + setSubscriptionAttr(feedURL, server, attrName, value) { + let ds = this.getSubscriptionsDS(server); + let sub = ds.data.find(feed => feed.url == feedURL); + if (sub === undefined) { + // Add a new entry. + sub = { url: feedURL }; + ds.data.push(sub); + } + sub[attrName] = value; + ds.saveSoon(); + }, + + /** + * Returns a reference to the feeditems store for the given server + * (the feeditems.json data). + * + * @param {nsIMsgIncomingServer} aServer - server to fetch item data for. + * @returns {JSONFile} - JSONFile with data field containing a collection + * of feeditems indexed by item url. + */ + getItemsDS(aServer) { + if (this[aServer.serverURI] && this[aServer.serverURI].FeedItemsDS) { + return this[aServer.serverURI].FeedItemsDS; + } + + let rssServer = aServer.QueryInterface(Ci.nsIRssIncomingServer); + let itemsFile = rssServer.feedItemsPath; // Path to feeditems.json + let exists = itemsFile.exists(); + let ds = new lazy.JSONFile({ + path: itemsFile.path, + backupTo: itemsFile.path + ".backup", + }); + ds.ensureDataReady(); + if (!this[aServer.serverURI]) { + this[aServer.serverURI] = {}; + } + this[aServer.serverURI].FeedItemsDS = ds; + if (!exists) { + // No feeditems.json, need to initialise our data. + ds.data = {}; + } + return ds; + }, + + /** + * Dragging something from somewhere. It may be a nice x-moz-url or from a + * browser or app that provides a less nice dataTransfer object in the event. + * Extract the url and if it passes the scheme test, try to subscribe. + * + * @param {nsISupports} aDataTransfer - The dnd event's dataTransfer. + * + * @returns {nsIURI} or null - A uri if valid, null if none. + */ + getFeedUriFromDataTransfer(aDataTransfer) { + let dt = aDataTransfer; + let types = ["text/x-moz-url-data", "text/x-moz-url"]; + let validUri = false; + let uri; + + if (dt.getData(types[0])) { + // The url is the data. + uri = Services.io.newURI(dt.mozGetDataAt(types[0], 0)); + validUri = this.isValidScheme(uri); + this.log.trace( + "getFeedUriFromDataTransfer: dropEffect:type:value - " + + dt.dropEffect + + " : " + + types[0] + + " : " + + uri.spec + ); + } else if (dt.getData(types[1])) { + // The url is the first part of the data, the second part is random. + uri = Services.io.newURI(dt.mozGetDataAt(types[1], 0).split("\n")[0]); + validUri = this.isValidScheme(uri); + this.log.trace( + "getFeedUriFromDataTransfer: dropEffect:type:value - " + + dt.dropEffect + + " : " + + types[0] + + " : " + + uri.spec + ); + } else { + // Go through the types and see if there's a url; get the first one. + for (let i = 0; i < dt.types.length; i++) { + let spec = dt.mozGetDataAt(dt.types[i], 0); + this.log.trace( + "getFeedUriFromDataTransfer: dropEffect:index:type:value - " + + dt.dropEffect + + " : " + + i + + " : " + + dt.types[i] + + " : " + + spec + ); + try { + uri = Services.io.newURI(spec); + validUri = this.isValidScheme(uri); + } catch (ex) {} + + if (validUri) { + break; + } + } + } + + return validUri ? uri : null; + }, + + /** + * Returns security/certificate/network error details for an XMLHTTPRequest. + * + * @param {XMLHTTPRequest} xhr - The xhr request. + * + * @returns {Array}[{String} errType or null, {String} errName or null] + * - Array with 2 error codes, (nulls if not determined). + */ + createTCPErrorFromFailedXHR(xhr) { + let status = xhr.channel.QueryInterface(Ci.nsIRequest).status; + + let errType = null; + let errName = null; + if ((status & 0xff0000) === 0x5a0000) { + // Security module. + const nsINSSErrorsService = Ci.nsINSSErrorsService; + let nssErrorsService = + Cc["@mozilla.org/nss_errors_service;1"].getService(nsINSSErrorsService); + let errorClass; + + // getErrorClass()) will throw a generic NS_ERROR_FAILURE if the error + // code is somehow not in the set of covered errors. + try { + errorClass = nssErrorsService.getErrorClass(status); + } catch (ex) { + // Catch security protocol exception. + errorClass = "SecurityProtocol"; + } + + if (errorClass == nsINSSErrorsService.ERROR_CLASS_BAD_CERT) { + errType = "SecurityCertificate"; + } else { + errType = "SecurityProtocol"; + } + + // NSS_SEC errors (happen below the base value because of negative vals). + if ( + (status & 0xffff) < + Math.abs(nsINSSErrorsService.NSS_SEC_ERROR_BASE) + ) { + // The bases are actually negative, so in our positive numeric space, + // we need to subtract the base off our value. + let nssErr = + Math.abs(nsINSSErrorsService.NSS_SEC_ERROR_BASE) - (status & 0xffff); + + switch (nssErr) { + case 11: // SEC_ERROR_EXPIRED_CERTIFICATE, sec(11) + errName = "SecurityExpiredCertificateError"; + break; + case 12: // SEC_ERROR_REVOKED_CERTIFICATE, sec(12) + errName = "SecurityRevokedCertificateError"; + break; + + // Per bsmith, we will be unable to tell these errors apart very soon, + // so it makes sense to just folder them all together already. + case 13: // SEC_ERROR_UNKNOWN_ISSUER, sec(13) + case 20: // SEC_ERROR_UNTRUSTED_ISSUER, sec(20) + case 21: // SEC_ERROR_UNTRUSTED_CERT, sec(21) + case 36: // SEC_ERROR_CA_CERT_INVALID, sec(36) + errName = "SecurityUntrustedCertificateIssuerError"; + break; + case 90: // SEC_ERROR_INADEQUATE_KEY_USAGE, sec(90) + errName = "SecurityInadequateKeyUsageError"; + break; + case 176: // SEC_ERROR_CERT_SIGNATURE_ALGORITHM_DISABLED, sec(176) + errName = "SecurityCertificateSignatureAlgorithmDisabledError"; + break; + default: + errName = "SecurityError"; + break; + } + } else { + // Calculating the difference. + let sslErr = + Math.abs(nsINSSErrorsService.NSS_SSL_ERROR_BASE) - (status & 0xffff); + + switch (sslErr) { + case 3: // SSL_ERROR_NO_CERTIFICATE, ssl(3) + errName = "SecurityNoCertificateError"; + break; + case 4: // SSL_ERROR_BAD_CERTIFICATE, ssl(4) + errName = "SecurityBadCertificateError"; + break; + case 8: // SSL_ERROR_UNSUPPORTED_CERTIFICATE_TYPE, ssl(8) + errName = "SecurityUnsupportedCertificateTypeError"; + break; + case 9: // SSL_ERROR_UNSUPPORTED_VERSION, ssl(9) + errName = "SecurityUnsupportedTLSVersionError"; + break; + case 12: // SSL_ERROR_BAD_CERT_DOMAIN, ssl(12) + errName = "SecurityCertificateDomainMismatchError"; + break; + default: + errName = "SecurityError"; + break; + } + } + } else { + errType = "Network"; + switch (status) { + // Connect to host:port failed. + case 0x804b000c: // NS_ERROR_CONNECTION_REFUSED, network(13) + errName = "ConnectionRefusedError"; + break; + // network timeout error. + case 0x804b000e: // NS_ERROR_NET_TIMEOUT, network(14) + errName = "NetworkTimeoutError"; + break; + // Hostname lookup failed. + case 0x804b001e: // NS_ERROR_UNKNOWN_HOST, network(30) + errName = "DomainNotFoundError"; + break; + case 0x804b0047: // NS_ERROR_NET_INTERRUPT, network(71) + errName = "NetworkInterruptError"; + break; + default: + errName = "NetworkError"; + break; + } + } + + return [errType, errName]; + }, + + /** + * Returns if a uri/url is valid to subscribe. + * + * @param {nsIURI} aUri or {String} aUrl - The Uri/Url. + * + * @returns {boolean} - true if a valid scheme, false if not. + */ + _validSchemes: ["http", "https", "file"], + isValidScheme(aUri) { + if (!(aUri instanceof Ci.nsIURI)) { + try { + aUri = Services.io.newURI(aUri); + } catch (ex) { + return false; + } + } + + return this._validSchemes.includes(aUri.scheme); + }, + + /** + * Is a folder Trash or in Trash. + * + * @param {nsIMsgFolder} aFolder - The folder. + * + * @returns {Boolean} - true if folder is Trash else false. + */ + isInTrash(aFolder) { + let trashFolder = aFolder.rootFolder.getFolderWithFlags( + Ci.nsMsgFolderFlags.Trash + ); + if ( + trashFolder && + (trashFolder == aFolder || trashFolder.isAncestorOf(aFolder)) + ) { + return true; + } + + return false; + }, + + /** + * Return a folder path string constructed from individual folder UTF8 names + * stored as properties (not possible hashes used to construct disk foldername). + * + * @param {nsIMsgFolder} aFolder - The folder. + * + * @returns {String} prettyName or null - Name or null if not a disk folder. + */ + getFolderPrettyPath(aFolder) { + let msgFolder = lazy.MailUtils.getExistingFolder(aFolder.URI); + if (!msgFolder) { + // Not a real folder uri. + return null; + } + + if (msgFolder.URI == msgFolder.server.serverURI) { + return msgFolder.server.prettyName; + } + + // Server part first. + let pathParts = [msgFolder.server.prettyName]; + let rawPathParts = msgFolder.URI.split(msgFolder.server.serverURI + "/"); + let folderURI = msgFolder.server.serverURI; + rawPathParts = rawPathParts[1].split("/"); + for (let i = 0; i < rawPathParts.length - 1; i++) { + // Two or more folders deep parts here. + folderURI += "/" + rawPathParts[i]; + msgFolder = lazy.MailUtils.getExistingFolder(folderURI); + pathParts.push(msgFolder.name); + } + + // Leaf folder last. + pathParts.push(aFolder.name); + return pathParts.join("/"); + }, + + /** + * Date validator for feeds. + * + * @param {string} aDate - Date string. + * + * @returns {Boolean} - true if passes regex test, false if not. + */ + isValidRFC822Date(aDate) { + const FZ_RFC822_RE = + "^(((Mon)|(Tue)|(Wed)|(Thu)|(Fri)|(Sat)|(Sun)), *)?\\d\\d?" + + " +((Jan)|(Feb)|(Mar)|(Apr)|(May)|(Jun)|(Jul)|(Aug)|(Sep)|(Oct)|(Nov)|(Dec))" + + " +\\d\\d(\\d\\d)? +\\d\\d:\\d\\d(:\\d\\d)? +(([+-]?\\d\\d\\d\\d)|(UT)|(GMT)" + + "|(EST)|(EDT)|(CST)|(CDT)|(MST)|(MDT)|(PST)|(PDT)|\\w)$"; + let regex = new RegExp(FZ_RFC822_RE); + return regex.test(aDate); + }, + + /** + * Create rfc5322 date. + * + * @param {string} aDateString - Optional date string; if null or invalid + * date, get the current datetime. + * + * @returns {String} - An rfc5322 date string. + */ + getValidRFC5322Date(aDateString) { + let d = new Date(aDateString || new Date().getTime()); + d = isNaN(d.getTime()) ? new Date() : d; + return lazy.jsmime.headeremitter + .emitStructuredHeader("Date", d, {}) + .substring(6) + .trim(); + }, + + /** + * Progress glue code. Acts as a go between the RSS back end and the mail + * window front end determined by the aMsgWindow parameter passed into + * nsINewsBlogFeedDownloader. + */ + progressNotifier: { + mSubscribeMode: false, + mMsgWindow: null, + mStatusFeedback: null, + mFeeds: {}, + // Keeps track of the total number of feeds we have been asked to download. + // This number may not reflect the # of entries in our mFeeds array because + // not all feeds may have reported in for the first time. + mNumPendingFeedDownloads: 0, + + /** + * @param {?nsIMsgWindow} aMsgWindow - Associated aMsgWindow if any. + * @param {boolean} aSubscribeMode - Whether we're in subscribe mode. + * @returns {void} + */ + init(aMsgWindow, aSubscribeMode) { + if (this.mNumPendingFeedDownloads == 0) { + // If we aren't already in the middle of downloading feed items. + this.mStatusFeedback = aMsgWindow ? aMsgWindow.statusFeedback : null; + this.mSubscribeMode = aSubscribeMode; + this.mMsgWindow = aMsgWindow; + + if (this.mStatusFeedback) { + this.mStatusFeedback.startMeteors(); + this.mStatusFeedback.showStatusString( + FeedUtils.strings.GetStringFromName( + aSubscribeMode + ? "subscribe-validating-feed" + : "newsblog-getNewMsgsCheck" + ) + ); + } + } + }, + + /** + * Called on final success or error resolution of a feed download and + * parsing. If aDisable is true, the error shouldn't be retried continually + * and the url should be verified by the user. A bad html response code or + * cert error will cause the url to be disabled, while general network + * connectivity errors applying to all urls will not. + * + * @param {Feed} feed - The Feed object, or a synthetic object that must + * contain members {nsIMsgFolder folder, String url}. + * @param {Integer} aErrorCode - The resolution code, a kNewsBlog* value. + * @param {Boolean} aDisable - If true, disable/pause the feed. + * + * @returns {void} + */ + downloaded(feed, aErrorCode, aDisable) { + let folderName = feed.folder + ? feed.folder.name + : feed.server.rootFolder.prettyName; + FeedUtils.log.debug( + "downloaded: " + + (this.mSubscribeMode ? "Subscribe " : "Update ") + + "errorCode:folderName:feedUrl - " + + aErrorCode + + " : " + + folderName + + " : " + + feed.url + ); + if (this.mSubscribeMode) { + if (aErrorCode == FeedUtils.kNewsBlogSuccess) { + // Add the feed to the databases. + FeedUtils.addFeed(feed); + + // Nice touch: notify so the window ca select the folder that now + // contains the newly subscribed feed. + // This is particularly nice if we just finished subscribing + // to a feed URL that the operating system gave us. + Services.obs.notifyObservers(feed.folder, "folder-subscribed"); + + // Check for an existing feed subscriptions window and update it. + let subscriptionsWindow = Services.wm.getMostRecentWindow( + "Mail:News-BlogSubscriptions" + ); + if (subscriptionsWindow) { + subscriptionsWindow.FeedSubscriptions.FolderListener.folderAdded( + feed.folder + ); + } + } else if (feed && feed.url && feed.server) { + // Non success. Remove intermediate traces from the feeds database. + FeedUtils.deleteFeed(feed); + } + } + + if (aErrorCode != FeedUtils.kNewsBlogFeedIsBusy) { + if ( + aErrorCode == FeedUtils.kNewsBlogSuccess || + aErrorCode == FeedUtils.kNewsBlogNoNewItems + ) { + // Update lastUpdateTime only if successful normal processing. + let options = feed.options; + let now = Date.now(); + options.updates.lastUpdateTime = now; + if (feed.itemsStored) { + options.updates.lastDownloadTime = now; + } + + // If a previously disabled due to error feed is successful, set + // enabled state on, as that was the desired user setting. + if (options.updates.enabled == null) { + options.updates.enabled = true; + FeedUtils.setStatus(feed.folder, feed.url, "enabled", true); + } + + feed.options = options; + FeedUtils.setStatus(feed.folder, feed.url, "lastUpdateTime", now); + } else if (aDisable) { + if ( + Services.prefs.getBoolPref("rss.disable_feeds_on_update_failure") + ) { + // Do not keep retrying feeds with error states. Set persisted state + // to |null| to indicate error disable (and not user disable), but + // only if the feed is user enabled. + let options = feed.options; + if (options.updates.enabled) { + options.updates.enabled = null; + } + + feed.options = options; + FeedUtils.setStatus(feed.folder, feed.url, "enabled", false); + FeedUtils.log.warn( + "downloaded: updates disabled due to error, " + + "check the url - " + + feed.url + ); + } else { + FeedUtils.log.warn( + "downloaded: update failed, check the url - " + feed.url + ); + } + } + + if (!this.mSubscribeMode) { + FeedUtils.setStatus(feed.folder, feed.url, "code", aErrorCode); + + if ( + feed.folder && + !FeedUtils.getFolderProperties(feed.folder).includes("isBusy") + ) { + // Free msgDatabase after new mail biff is set; if busy let the next + // result do the freeing. Otherwise new messages won't be indicated. + // This feed may belong to a folder with multiple other feeds, some + // of which may not yet be finished, so free only if the folder is + // no longer busy. + feed.folder.msgDatabase = null; + FeedUtils.log.debug( + "downloaded: msgDatabase freed - " + feed.folder.name + ); + } + } + } + + let message = ""; + switch (aErrorCode) { + case FeedUtils.kNewsBlogSuccess: + case FeedUtils.kNewsBlogFeedIsBusy: + message = ""; + break; + case FeedUtils.kNewsBlogNoNewItems: + message = + feed.url + + ". " + + FeedUtils.strings.GetStringFromName( + "newsblog-noNewArticlesForFeed" + ); + break; + case FeedUtils.kNewsBlogInvalidFeed: + message = FeedUtils.strings.formatStringFromName( + "newsblog-feedNotValid", + [feed.url] + ); + break; + case FeedUtils.kNewsBlogRequestFailure: + message = FeedUtils.strings.formatStringFromName( + "newsblog-networkError", + [feed.url] + ); + break; + case FeedUtils.kNewsBlogFileError: + message = FeedUtils.strings.GetStringFromName( + "subscribe-errorOpeningFile" + ); + break; + case FeedUtils.kNewsBlogBadCertError: + let host = Services.io.newURI(feed.url).host; + message = FeedUtils.strings.formatStringFromName( + "newsblog-badCertError", + [host] + ); + break; + case FeedUtils.kNewsBlogNoAuthError: + message = FeedUtils.strings.formatStringFromName( + "newsblog-noAuthError", + [feed.url] + ); + break; + } + + if (message) { + let location = + FeedUtils.getFolderPrettyPath(feed.folder || feed.server.rootFolder) + + " -> "; + FeedUtils.log.info( + "downloaded: " + + (this.mSubscribeMode ? "Subscribe: " : "Update: ") + + location + + message + ); + } + + if (this.mStatusFeedback) { + this.mStatusFeedback.showStatusString(message); + this.mStatusFeedback.stopMeteors(); + } + + this.mNumPendingFeedDownloads--; + + if (this.mNumPendingFeedDownloads == 0) { + this.mFeeds = {}; + this.mSubscribeMode = false; + FeedUtils.log.debug("downloaded: all pending downloads finished"); + + // Should we do this on a timer so the text sticks around for a little + // while? It doesn't look like we do it on a timer for newsgroups so + // we'll follow that model. Don't clear the status text if we just + // dumped an error to the status bar! + if (aErrorCode == FeedUtils.kNewsBlogSuccess && this.mStatusFeedback) { + this.mStatusFeedback.showStatusString(""); + } + } + + feed = null; + }, + + /** + * This gets called after the RSS parser finishes storing a feed item to + * disk. aCurrentFeedItems is an integer corresponding to how many feed + * items have been downloaded so far. aMaxFeedItems is an integer + * corresponding to the total number of feed items to download. + * + * @param {Feed} feed - The Feed object. + * @param {Integer} aCurrentFeedItems - Number downloaded so far. + * @param {Integer} aMaxFeedItems - Total number to download. + * + * @returns {void} + */ + onFeedItemStored(feed, aCurrentFeedItems, aMaxFeedItems) { + // We currently don't do anything here. Eventually we may add status + // text about the number of new feed articles received. + + if (this.mSubscribeMode && this.mStatusFeedback) { + // If we are subscribing to a feed, show feed download progress. + this.mStatusFeedback.showStatusString( + FeedUtils.strings.formatStringFromName("subscribe-gettingFeedItems", [ + aCurrentFeedItems, + aMaxFeedItems, + ]) + ); + this.onProgress(feed, aCurrentFeedItems, aMaxFeedItems); + } + }, + + onProgress(feed, aProgress, aProgressMax, aLengthComputable) { + if (feed.url in this.mFeeds) { + // Have we already seen this feed? + this.mFeeds[feed.url].currentProgress = aProgress; + } else { + this.mFeeds[feed.url] = { + currentProgress: aProgress, + maxProgress: aProgressMax, + }; + } + + this.updateProgressBar(); + }, + + updateProgressBar() { + let currentProgress = 0; + let maxProgress = 0; + for (let index in this.mFeeds) { + currentProgress += this.mFeeds[index].currentProgress; + maxProgress += this.mFeeds[index].maxProgress; + } + + // If we start seeing weird "jumping" behavior where the progress bar + // goes below a threshold then above it again, then we can factor a + // fudge factor here based on the number of feeds that have not reported + // yet and the avg progress we've already received for existing feeds. + // Fortunately the progressmeter is on a timer and only updates every so + // often. For the most part all of our request have initial progress + // before the UI actually picks up a progress value. + if (this.mStatusFeedback) { + let progress = (currentProgress * 100) / maxProgress; + this.mStatusFeedback.showProgress(progress); + } + }, + }, +}; + +XPCOMUtils.defineLazyGetter(FeedUtils, "log", function () { + return console.createInstance({ + prefix: "feeds", + maxLogLevelPref: "feeds.loglevel", + }); +}); + +XPCOMUtils.defineLazyGetter(FeedUtils, "strings", function () { + return Services.strings.createBundle( + "chrome://messenger-newsblog/locale/newsblog.properties" + ); +}); + +XPCOMUtils.defineLazyGetter(FeedUtils, "stringsPrefs", function () { + return Services.strings.createBundle( + "chrome://messenger/locale/prefs.properties" + ); +}); diff --git a/comm/mailnews/extensions/newsblog/NewsBlog.jsm b/comm/mailnews/extensions/newsblog/NewsBlog.jsm new file mode 100644 index 0000000000..b1a1a22536 --- /dev/null +++ b/comm/mailnews/extensions/newsblog/NewsBlog.jsm @@ -0,0 +1,28 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +var EXPORTED_SYMBOLS = ["FeedDownloader", "FeedAcctMgrExtension"]; + +var { FeedUtils } = ChromeUtils.import("resource:///modules/FeedUtils.jsm"); + +function FeedDownloader() {} +FeedDownloader.prototype = { + downloadFeed(aFolder, aUrlListener, aIsBiff, aMsgWindow) { + FeedUtils.downloadFeed(aFolder, aUrlListener, aIsBiff, aMsgWindow); + }, + updateSubscriptionsDS(aFolder, aOrigFolder, aAction) { + FeedUtils.updateSubscriptionsDS(aFolder, aOrigFolder, aAction); + }, + + QueryInterface: ChromeUtils.generateQI(["nsINewsBlogFeedDownloader"]), +}; + +function FeedAcctMgrExtension() {} +FeedAcctMgrExtension.prototype = { + name: "newsblog", + chromePackageName: "messenger-newsblog", + showPanel: server => false, + + QueryInterface: ChromeUtils.generateQI(["nsIMsgAccountManagerExtension"]), +}; diff --git a/comm/mailnews/extensions/newsblog/am-newsblog.js b/comm/mailnews/extensions/newsblog/am-newsblog.js new file mode 100644 index 0000000000..eacf2a3ae5 --- /dev/null +++ b/comm/mailnews/extensions/newsblog/am-newsblog.js @@ -0,0 +1,128 @@ +/* -*- Mode: JavaScript; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* import-globals-from ../../base/prefs/content/am-prefs.js */ + +var { FeedUtils } = ChromeUtils.import("resource:///modules/FeedUtils.jsm"); + +var gAccount, + gUpdateEnabled, + gUpdateValue, + gBiffUnits, + gAutotagEnable, + gAutotagUsePrefix, + gAutotagPrefix; + +/** + * Initialize am-newsblog account settings page when it gets shown. + * Update an account's main settings title etc. + * + * @returns {void} + */ +function onInit() { + setAccountTitle(); + + let optionsAcct = FeedUtils.getOptionsAcct(gAccount.incomingServer); + document.getElementById("doBiff").checked = optionsAcct.doBiff; + + gUpdateEnabled = document.getElementById("updateEnabled"); + gUpdateValue = document.getElementById("updateValue"); + gBiffUnits = document.getElementById("biffUnits"); + gAutotagEnable = document.getElementById("autotagEnable"); + gAutotagUsePrefix = document.getElementById("autotagUsePrefix"); + gAutotagPrefix = document.getElementById("autotagPrefix"); + + gUpdateEnabled.checked = optionsAcct.updates.enabled; + gBiffUnits.value = optionsAcct.updates.updateUnits; + let minutes = + optionsAcct.updates.updateUnits == FeedUtils.kBiffUnitsMinutes + ? optionsAcct.updates.updateMinutes + : optionsAcct.updates.updateMinutes / (24 * 60); + gUpdateValue.value = Number(minutes); + onCheckItem("updateValue", ["updateEnabled"]); + onCheckItem("biffMinutes", ["updateEnabled"]); + onCheckItem("biffDays", ["updateEnabled"]); + + gAutotagEnable.checked = optionsAcct.category.enabled; + gAutotagUsePrefix.disabled = !gAutotagEnable.checked; + gAutotagUsePrefix.checked = optionsAcct.category.prefixEnabled; + gAutotagPrefix.disabled = + gAutotagUsePrefix.disabled || !gAutotagUsePrefix.checked; + gAutotagPrefix.value = optionsAcct.category.prefix; +} + +function onPreInit(account, accountValues) { + gAccount = account; +} + +/** + * Handle the blur event of the #server.prettyName pref input. + * Update account name in account manager tree and account settings' main title. + * + * @param {Event} event - Blur event from the pretty name input. + * @returns {void} + */ +function serverPrettyNameOnBlur(event) { + parent.setAccountLabel(gAccount.key, event.target.value); + setAccountTitle(); +} + +/** + * Update an account's main settings title with the account name if applicable. + * + * @returns {void} + */ +function setAccountTitle() { + let accountName = document.getElementById("server.prettyName"); + let title = document.querySelector("#am-newsblog-title .dialogheader-title"); + let titleValue = title.getAttribute("defaultTitle"); + if (accountName.value) { + titleValue += " - " + accountName.value; + } + + title.setAttribute("value", titleValue); + document.title = titleValue; +} + +function setPrefs(aNode) { + let optionsAcct = FeedUtils.getOptionsAcct(gAccount.incomingServer); + switch (aNode.id) { + case "doBiff": + FeedUtils.pauseFeedFolderUpdates( + gAccount.incomingServer.rootFolder, + !aNode.checked, + true + ); + break; + case "updateEnabled": + case "updateValue": + case "biffUnits": + optionsAcct.updates.enabled = gUpdateEnabled.checked; + onCheckItem("updateValue", ["updateEnabled"]); + onCheckItem("biffMinutes", ["updateEnabled"]); + onCheckItem("biffDays", ["updateEnabled"]); + let minutes = + gBiffUnits.value == FeedUtils.kBiffUnitsMinutes + ? gUpdateValue.value + : gUpdateValue.value * 24 * 60; + optionsAcct.updates.updateMinutes = Number(minutes); + optionsAcct.updates.updateUnits = gBiffUnits.value; + break; + case "autotagEnable": + optionsAcct.category.enabled = aNode.checked; + gAutotagUsePrefix.disabled = !aNode.checked; + gAutotagPrefix.disabled = !aNode.checked || !gAutotagUsePrefix.checked; + break; + case "autotagUsePrefix": + optionsAcct.category.prefixEnabled = aNode.checked; + gAutotagPrefix.disabled = aNode.disabled || !aNode.checked; + break; + case "autotagPrefix": + optionsAcct.category.prefix = aNode.value; + break; + } + + FeedUtils.setOptionsAcct(gAccount.incomingServer, optionsAcct); +} diff --git a/comm/mailnews/extensions/newsblog/am-newsblog.xhtml b/comm/mailnews/extensions/newsblog/am-newsblog.xhtml new file mode 100644 index 0000000000..e6ba0f6a5d --- /dev/null +++ b/comm/mailnews/extensions/newsblog/am-newsblog.xhtml @@ -0,0 +1,233 @@ +<?xml version="1.0"?> +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> + +<?xml-stylesheet href="chrome://messenger/skin/accountManage.css" type="text/css"?> +<?xml-stylesheet href="chrome://messenger-newsblog/skin/feed-subscriptions.css" type="text/css"?> + +<!DOCTYPE html [ <!ENTITY % newsblogDTD SYSTEM "chrome://messenger-newsblog/locale/am-newsblog.dtd"> +%newsblogDTD; +<!ENTITY % feedDTD SYSTEM "chrome://messenger-newsblog/locale/feed-subscriptions.dtd" > +%feedDTD; +<!ENTITY % accountNoIdentDTD SYSTEM "chrome://messenger/locale/am-serverwithnoidentities.dtd" > +%accountNoIdentDTD; +<!ENTITY % accountServerTopDTD SYSTEM "chrome://messenger/locale/am-server-top.dtd"> +%accountServerTopDTD; ]> + +<html + xmlns="http://www.w3.org/1999/xhtml" + xmlns:xul="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul" + xmlns:html="http://www.w3.org/1999/xhtml" +> + <head> + <title>&accountTitle.label;</title> + <script + defer="defer" + src="chrome://messenger/content/globalOverlay.js" + ></script> + <script + defer="defer" + src="chrome://global/content/editMenuOverlay.js" + ></script> + <script + defer="defer" + src="chrome://messenger/content/AccountManager.js" + ></script> + <script + defer="defer" + src="chrome://messenger-newsblog/content/am-newsblog.js" + ></script> + <script + defer="defer" + src="chrome://messenger-newsblog/content/newsblogOverlay.js" + ></script> + <script defer="defer" src="chrome://messenger/content/amUtils.js"></script> + <script defer="defer" src="chrome://messenger/content/am-prefs.js"></script> + <script> + // FIXME: move to script file. + window.addEventListener("load", event => { + parent.onPanelLoaded("am-newsblog.xhtml"); + }); + </script> + </head> + <html:body + xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul" + > + <vbox id="containerBox" flex="1"> + <hbox id="am-newsblog-title" class="dialogheader"> + <label class="dialogheader-title" defaultTitle="&accountTitle.label;" /> + </hbox> + + <description class="secDesc">&accountSettingsDesc.label;</description> + + <hbox class="input-container"> + <label + id="server.prettyNameLabel" + value="&accountName.label;" + accesskey="&accountName.accesskey;" + control="server.prettyName" + /> + <html:input + id="server.prettyName" + type="text" + wsm_persist="true" + class="input-inline" + aria-labelledby="server.prettyNameLabel" + onblur="serverPrettyNameOnBlur(event);" + prefstring="mail.server.%serverkey%.name" + /> + </hbox> + + <separator class="thin" /> + + <html:div> + <html:fieldset> + <html:legend>&serverSettings.label;</html:legend> + <checkbox + id="doBiff" + label="&biffAll.label;" + accesskey="&biffAll.accesskey;" + oncommand="setPrefs(this)" + /> + </html:fieldset> + </html:div> + + <separator class="thin" /> + + <html:div> + <html:fieldset> + <html:legend>&newFeedSettings.label;</html:legend> + + <hbox align="center"> + <checkbox + id="updateEnabled" + label="&biffStart.label;" + accesskey="&biffStart.accesskey;" + oncommand="setPrefs(this)" + /> + <html:input + id="updateValue" + type="number" + class="size3" + min="1" + aria-labelledby="updateEnabled updateValue biffMinutes biffDays" + onchange="setPrefs(this)" + /> + <radiogroup + id="biffUnits" + orient="horizontal" + oncommand="setPrefs(this)" + > + <radio + id="biffMinutes" + value="min" + label="&biffMinutes.label;" + accesskey="&biffMinutes.accesskey;" + /> + <radio + id="biffDays" + value="d" + label="&biffDays.label;" + accesskey="&biffDays.accesskey;" + /> + </radiogroup> + </hbox> + + <checkbox + id="server.quickMode" + wsm_persist="true" + genericattr="true" + label="&quickMode.label;" + accesskey="&quickMode.accesskey;" + preftype="bool" + prefattribute="value" + prefstring="mail.server.%serverkey%.quickMode" + /> + + <checkbox + id="autotagEnable" + accesskey="&autotagEnable.accesskey;" + label="&autotagEnable.label;" + oncommand="setPrefs(this)" + /> + <hbox class="input-container"> + <checkbox + id="autotagUsePrefix" + class="indent" + accesskey="&autotagUsePrefix.accesskey;" + label="&autotagUsePrefix.label;" + oncommand="setPrefs(this)" + /> + <html:input + id="autotagPrefix" + type="text" + class="input-inline" + aria-labelledby="autotagUsePrefix" + placeholder="&autoTagPrefix.placeholder;" + onchange="setPrefs(this)" + /> + </hbox> + </html:fieldset> + </html:div> + + <separator class="thin" /> + + <hbox pack="end"> + <button + label="&manageSubscriptions.label;" + accesskey="&manageSubscriptions.accesskey;" + oncommand="openSubscriptionsDialog(gAccount.incomingServer.rootFolder);" + /> + </hbox> + + <separator class="thin" /> + + <html:div> + <html:fieldset> + <html:legend>&messageStorage.label;</html:legend> + + <checkbox + id="server.emptyTrashOnExit" + wsm_persist="true" + label="&emptyTrashOnExit.label;" + accesskey="&emptyTrashOnExit.accesskey;" + prefattribute="value" + prefstring="mail.server.%serverkey%.empty_trash_on_exit" + /> + + <separator class="thin" /> + + <vbox> + <hbox align="center"> + <label + id="server.localPathLabel" + value="&localPath1.label;" + control="server.localPath" + /> + <hbox class="input-container" flex="1"> + <html:input + id="server.localPath" + type="text" + readonly="readonly" + class="uri-element input-inline" + aria-labelledby="server.localPathLabel" + wsm_persist="true" + datatype="nsIFile" + prefstring="mail.server.%serverkey%.directory" + /> + </hbox> + <button + id="browseForLocalFolder" + label="&browseFolder.label;" + filepickertitle="&localFolderPicker.label;" + accesskey="&browseFolder.accesskey;" + oncommand="BrowseForLocalFolders();" + /> + </hbox> + </vbox> + </html:fieldset> + </html:div> + </vbox> + </html:body> +</html> diff --git a/comm/mailnews/extensions/newsblog/components.conf b/comm/mailnews/extensions/newsblog/components.conf new file mode 100644 index 0000000000..0e2c4ce03a --- /dev/null +++ b/comm/mailnews/extensions/newsblog/components.conf @@ -0,0 +1,21 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +Classes = [ + { + "cid": "{5c124537-adca-4456-b2b5-641ab687d1f6}", + "contract_ids": ["@mozilla.org/newsblog-feed-downloader;1"], + "jsm": "resource:///modules/NewsBlog.jsm", + "constructor": "FeedDownloader", + }, + { + "cid": "{e109c05f-d304-4ca5-8c44-6de1bfaf1f74}", + "contract_ids": ["@mozilla.org/accountmanager/extension;1?name=newsblog"], + "jsm": "resource:///modules/NewsBlog.jsm", + "constructor": "FeedAcctMgrExtension", + "categories": {"mailnews-accountmanager-extensions": "newsblog"}, + }, +] diff --git a/comm/mailnews/extensions/newsblog/feed-subscriptions.js b/comm/mailnews/extensions/newsblog/feed-subscriptions.js new file mode 100644 index 0000000000..43ce61b11b --- /dev/null +++ b/comm/mailnews/extensions/newsblog/feed-subscriptions.js @@ -0,0 +1,3120 @@ +/* -*- Mode: JavaScript; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * @file + * GUI-side code for managing folder subscriptions. + */ + +var { Feed } = ChromeUtils.import("resource:///modules/Feed.jsm"); +var { FeedUtils } = ChromeUtils.import("resource:///modules/FeedUtils.jsm"); +var { MailServices } = ChromeUtils.import( + "resource:///modules/MailServices.jsm" +); +var { AppConstants } = ChromeUtils.importESModule( + "resource://gre/modules/AppConstants.sys.mjs" +); +var { FileUtils } = ChromeUtils.importESModule( + "resource://gre/modules/FileUtils.sys.mjs" +); +var { PluralForm } = ChromeUtils.importESModule( + "resource://gre/modules/PluralForm.sys.mjs" +); + +var FeedSubscriptions = { + get mMainWin() { + return Services.wm.getMostRecentWindow("mail:3pane"); + }, + + get mTree() { + return document.getElementById("rssSubscriptionsList"); + }, + + mFeedContainers: [], + mRSSServer: null, + mActionMode: null, + kSubscribeMode: 1, + kUpdateMode: 2, + kMoveMode: 3, + kCopyMode: 4, + kImportingOPML: 5, + kVerifyUrlMode: 6, + + get FOLDER_ACTIONS() { + return ( + Ci.nsIMsgFolderNotificationService.folderAdded | + Ci.nsIMsgFolderNotificationService.folderDeleted | + Ci.nsIMsgFolderNotificationService.folderRenamed | + Ci.nsIMsgFolderNotificationService.folderMoveCopyCompleted + ); + }, + + onLoad() { + // Extract the folder argument. + let folder; + if (window.arguments && window.arguments[0].folder) { + folder = window.arguments[0].folder; + } + + // Ensure dialog is fully loaded before selecting, to get visible row. + setTimeout(() => { + FeedSubscriptions.refreshSubscriptionView(folder); + }, 100); + let message = FeedUtils.strings.GetStringFromName("subscribe-loading"); + this.updateStatusItem("statusText", message); + + FeedUtils.CANCEL_REQUESTED = false; + + if (this.mMainWin) { + MailServices.mfn.addListener(this.FolderListener, this.FOLDER_ACTIONS); + } + }, + + onDialogAccept() { + let dismissDialog = true; + + // If we are in the middle of subscribing to a feed, inform the user that + // dismissing the dialog right now will abort the feed subscription. + if (this.mActionMode == this.kSubscribeMode) { + let pTitle = FeedUtils.strings.GetStringFromName( + "subscribe-cancelSubscriptionTitle" + ); + let pMessage = FeedUtils.strings.GetStringFromName( + "subscribe-cancelSubscription" + ); + dismissDialog = !Services.prompt.confirmEx( + window, + pTitle, + pMessage, + Ci.nsIPromptService.STD_YES_NO_BUTTONS, + null, + null, + null, + null, + {} + ); + } + + if (dismissDialog) { + FeedUtils.CANCEL_REQUESTED = this.mActionMode == this.kSubscribeMode; + if (this.mMainWin) { + MailServices.mfn.removeListener( + this.FolderListener, + this.FOLDER_ACTIONS + ); + } + } + + return dismissDialog; + }, + + refreshSubscriptionView(aSelectFolder, aSelectFeedUrl) { + let item = this.mView.currentItem; + this.loadSubscriptions(); + this.mTree.view = this.mView; + + if (aSelectFolder && !aSelectFeedUrl) { + this.selectFolder(aSelectFolder); + } else if (item) { + // If no folder to select, try to select the pre rebuild selection, in + // an existing window. For folderpane changes in a feed account. + let rootFolder = item.container + ? item.folder.rootFolder + : item.parentFolder.rootFolder; + if (item.container) { + if (!this.selectFolder(item.folder, { open: item.open })) { + // The item no longer exists, an ancestor folder was deleted or + // renamed/moved. + this.selectFolder(rootFolder); + } + } else { + let url = + item.parentFolder == aSelectFolder ? aSelectFeedUrl : item.url; + this.selectFeed({ folder: rootFolder, url }, null); + } + } + + this.mView.tree.ensureRowIsVisible(this.mView.selection.currentIndex); + this.clearStatusInfo(); + }, + + mView: { + kRowIndexUndefined: -1, + + get currentItem() { + // Get the current selection, if any. + let seln = this.selection; + let currentSelectionIndex = seln ? seln.currentIndex : null; + let item; + if (currentSelectionIndex != null) { + item = this.getItemAtIndex(currentSelectionIndex); + } + + return item; + }, + + /* nsITreeView */ + /* eslint-disable no-multi-spaces */ + tree: null, + + mRowCount: 0, + get rowCount() { + return this.mRowCount; + }, + + _selection: null, + get selection() { + return this._selection; + }, + set selection(val) { + this._selection = val; + }, + + setTree(aTree) { + this.tree = aTree; + }, + isSeparator(aRow) { + return false; + }, + isSorted() { + return false; + }, + isEditable(aRow, aColumn) { + return false; + }, + + getProgressMode(aRow, aCol) {}, + cycleHeader(aCol) {}, + cycleCell(aRow, aCol) {}, + selectionChanged() {}, + getRowProperties(aRow) { + return ""; + }, + getColumnProperties(aCol) { + return ""; + }, + getCellValue(aRow, aColumn) {}, + setCellValue(aRow, aColumn, aValue) {}, + setCellText(aRow, aColumn, aValue) {}, + /* eslint-enable no-multi-spaces */ + + getCellProperties(aRow, aColumn) { + let item = this.getItemAtIndex(aRow); + if (!item) { + return ""; + } + + if (AppConstants.MOZ_APP_NAME != "thunderbird") { + if (!item.folder) { + return "serverType-rss"; + } else if (item.folder.isServer) { + return "serverType-rss isServer-true"; + } + + return "livemark"; + } + + let folder = item.folder; + let properties = "folderNameCol"; + let mainWin = FeedSubscriptions.mMainWin; + if (!mainWin) { + let hasFeeds = FeedUtils.getFeedUrlsInFolder(folder); + if (!folder) { + properties += " isFeed-true"; + } else if (hasFeeds) { + properties += " isFeedFolder-true"; + } else if (folder.isServer) { + properties += " serverType-rss isServer-true"; + } + } else { + let url = folder ? null : item.url; + folder = folder || item.parentFolder; + properties = mainWin.FolderUtils.getFolderProperties(folder, item.open); + properties += mainWin.FeedUtils.getFolderProperties(folder, url); + if ( + this.selection.currentIndex == aRow && + url && + item.options.updates.enabled && + properties.includes("isPaused") + ) { + item.options.updates.enabled = false; + FeedSubscriptions.updateFeedData(item); + } + } + + item.properties = properties; + return properties; + }, + + isContainer(aRow) { + let item = this.getItemAtIndex(aRow); + return item ? item.container : false; + }, + + isContainerOpen(aRow) { + let item = this.getItemAtIndex(aRow); + return item ? item.open : false; + }, + + isContainerEmpty(aRow) { + let item = this.getItemAtIndex(aRow); + if (!item) { + return false; + } + + return item.children.length == 0; + }, + + getItemAtIndex(aRow) { + if (aRow < 0 || aRow >= FeedSubscriptions.mFeedContainers.length) { + return null; + } + + return FeedSubscriptions.mFeedContainers[aRow]; + }, + + getItemInViewIndex(aFolder) { + if (!aFolder || !(aFolder instanceof Ci.nsIMsgFolder)) { + return null; + } + + for (let index = 0; index < this.rowCount; index++) { + // Find the visible folder in the view. + let item = this.getItemAtIndex(index); + if (item && item.container && item.url == aFolder.URI) { + return index; + } + } + + return null; + }, + + removeItemAtIndex(aRow, aNoSelect) { + let itemToRemove = this.getItemAtIndex(aRow); + if (!itemToRemove) { + return; + } + + if (itemToRemove.container && itemToRemove.open) { + // Close it, if open container. + this.toggleOpenState(aRow); + } + + let parentIndex = this.getParentIndex(aRow); + let hasNextSibling = this.hasNextSibling(aRow, aRow); + if (parentIndex != this.kRowIndexUndefined) { + let parent = this.getItemAtIndex(parentIndex); + if (parent) { + for (let index = 0; index < parent.children.length; index++) { + if (parent.children[index] == itemToRemove) { + parent.children.splice(index, 1); + break; + } + } + } + } + + // Now remove it from our view. + FeedSubscriptions.mFeedContainers.splice(aRow, 1); + + // Now invalidate the correct tree rows. + this.mRowCount--; + this.tree.rowCountChanged(aRow, -1); + + // Now update the selection position, unless noSelect (selection is + // done later or not at all). If the item is the last child, select the + // parent. Otherwise select the next sibling. + if (!aNoSelect) { + if (aRow <= FeedSubscriptions.mFeedContainers.length) { + this.selection.select(hasNextSibling ? aRow : aRow - 1); + } else { + this.selection.clearSelection(); + } + } + + // Now refocus the tree. + FeedSubscriptions.mTree.focus(); + }, + + getCellText(aRow, aColumn) { + let item = this.getItemAtIndex(aRow); + return item && aColumn.id == "folderNameCol" ? item.name : ""; + }, + + getImageSrc(aRow, aCol) { + let item = this.getItemAtIndex(aRow); + if ((item.folder && item.folder.isServer) || item.open) { + return ""; + } + + if ( + !item.open && + (item.properties.includes("hasError") || + item.properties.includes("isBusy")) + ) { + return ""; + } + + if (item.favicon != null) { + return item.favicon; + } + + let callback = iconUrl => { + item.favicon = iconUrl; + if (item.folder) { + for (let child of item.children) { + if (!child.container) { + child.favicon = iconUrl; + break; + } + } + } + + this.selection.tree.invalidateRow(aRow); + }; + + // A closed non server folder. + if (item.folder) { + for (let child of item.children) { + if (!child.container) { + if (child.favicon != null) { + return child.favicon; + } + + setTimeout(async () => { + let iconUrl = await FeedUtils.getFavicon( + child.parentFolder, + child.url + ); + if (iconUrl) { + callback(iconUrl); + } + }, 0); + break; + } + } + } else { + // A feed. + setTimeout(async () => { + let iconUrl = await FeedUtils.getFavicon(item.parentFolder, item.url); + if (iconUrl) { + callback(iconUrl); + } + }, 0); + } + + // Store empty string to return default while favicons are retrieved. + return (item.favicon = ""); + }, + + canDrop(aRow, aOrientation) { + let dropResult = this.extractDragData(aRow); + return ( + aOrientation == Ci.nsITreeView.DROP_ON && + dropResult.canDrop && + (dropResult.dropUrl || + dropResult.dropOnIndex != this.kRowIndexUndefined) + ); + }, + + drop(aRow, aOrientation) { + let win = FeedSubscriptions; + let results = this.extractDragData(aRow); + if (!results.canDrop) { + return; + } + + // Preselect the drop folder. + this.selection.select(aRow); + + if (results.dropUrl) { + // Don't freeze the app that initiated the drop just because we are + // in a loop waiting for the user to dimisss the add feed dialog. + setTimeout(() => { + win.addFeed(results.dropUrl, null, true, null, win.kSubscribeMode); + }, 0); + + let folderItem = this.getItemAtIndex(aRow); + FeedUtils.log.debug( + "drop: folder, url - " + + folderItem.folder.name + + ", " + + results.dropUrl + ); + } else if (results.dropOnIndex != this.kRowIndexUndefined) { + win.moveCopyFeed(results.dropOnIndex, aRow, results.dropEffect); + } + }, + + // Helper function for drag and drop. + extractDragData(aRow) { + let dt = this._currentDataTransfer; + let dragDataResults = { + canDrop: false, + dropUrl: null, + dropOnIndex: this.kRowIndexUndefined, + dropEffect: dt.dropEffect, + }; + + if (dt.getData("text/x-moz-feed-index")) { + // Dragging a feed in the tree. + if (this.selection) { + dragDataResults.dropOnIndex = this.selection.currentIndex; + + let curItem = this.getItemAtIndex(this.selection.currentIndex); + let newItem = this.getItemAtIndex(aRow); + let curServer = + curItem && curItem.parentFolder + ? curItem.parentFolder.server + : null; + let newServer = + newItem && newItem.folder ? newItem.folder.server : null; + + // No copying within the same account and no moving to the account + // folder in the same account. + if ( + !( + curServer == newServer && + (dragDataResults.dropEffect == "copy" || + newItem.folder == curItem.parentFolder || + newItem.folder.isServer) + ) + ) { + dragDataResults.canDrop = true; + } + } + } else { + // Try to get a feed url. + let validUri = FeedUtils.getFeedUriFromDataTransfer(dt); + + if (validUri) { + dragDataResults.canDrop = true; + dragDataResults.dropUrl = validUri.spec; + } + } + + return dragDataResults; + }, + + getParentIndex(aRow) { + let item = this.getItemAtIndex(aRow); + + if (item) { + for (let index = aRow; index >= 0; index--) { + if (FeedSubscriptions.mFeedContainers[index].level < item.level) { + return index; + } + } + } + + return this.kRowIndexUndefined; + }, + + isIndexChildOfParentIndex(aRow, aChildRow) { + // For visible tree rows, not if items are children of closed folders. + let item = this.getItemAtIndex(aRow); + if (!item || aChildRow <= aRow) { + return false; + } + + let targetLevel = this.getItemAtIndex(aRow).level; + let rows = FeedSubscriptions.mFeedContainers; + + for (let i = aRow + 1; i < rows.length; i++) { + if (this.getItemAtIndex(i).level <= targetLevel) { + break; + } + if (aChildRow == i) { + return true; + } + } + + return false; + }, + + hasNextSibling(aRow, aAfterIndex) { + let targetLevel = this.getItemAtIndex(aRow).level; + let rows = FeedSubscriptions.mFeedContainers; + for (let i = aAfterIndex + 1; i < rows.length; i++) { + if (this.getItemAtIndex(i).level == targetLevel) { + return true; + } + if (this.getItemAtIndex(i).level < targetLevel) { + return false; + } + } + + return false; + }, + + hasPreviousSibling(aRow) { + let item = this.getItemAtIndex(aRow); + if (item && aRow) { + return this.getItemAtIndex(aRow - 1).level == item.level; + } + + return false; + }, + + getLevel(aRow) { + let item = this.getItemAtIndex(aRow); + if (!item) { + return 0; + } + + return item.level; + }, + + toggleOpenState(aRow) { + let item = this.getItemAtIndex(aRow); + if (!item) { + return; + } + + // Save off the current selection item. + let seln = this.selection; + let currentSelectionIndex = seln.currentIndex; + + let rowsChanged = this.toggle(aRow); + + // Now restore selection, ensuring selection is maintained on toggles. + if (currentSelectionIndex > aRow) { + seln.currentIndex = currentSelectionIndex + rowsChanged; + } else { + seln.select(currentSelectionIndex); + } + + seln.selectEventsSuppressed = false; + }, + + toggle(aRow) { + // Collapse the row, or build sub rows based on open states in the map. + let item = this.getItemAtIndex(aRow); + if (!item) { + return null; + } + + let rows = FeedSubscriptions.mFeedContainers; + let rowCount = 0; + let multiplier; + + function addDescendants(aItem) { + for (let i = 0; i < aItem.children.length; i++) { + rowCount++; + let child = aItem.children[i]; + rows.splice(aRow + rowCount, 0, child); + if (child.open) { + addDescendants(child); + } + } + } + + if (item.open) { + // Close the container. Add up all subfolders and their descendants + // who may be open. + multiplier = -1; + let nextRow = aRow + 1; + let nextItem = rows[nextRow]; + while (nextItem && nextItem.level > item.level) { + rowCount++; + nextItem = rows[++nextRow]; + } + + rows.splice(aRow + 1, rowCount); + } else { + // Open the container. Restore the open state of all subfolder and + // their descendants. + multiplier = 1; + addDescendants(item); + } + + let delta = multiplier * rowCount; + this.mRowCount += delta; + + item.open = !item.open; + // Suppress the select event caused by rowCountChanged. + this.selection.selectEventsSuppressed = true; + // Add or remove the children from our view. + this.tree.rowCountChanged(aRow, delta); + return delta; + }, + }, + + makeFolderObject(aFolder, aCurrentLevel) { + let defaultQuickMode = aFolder.server.getBoolValue("quickMode"); + let optionsAcct = aFolder.isServer + ? FeedUtils.getOptionsAcct(aFolder.server) + : null; + let open = + !aFolder.isServer && + aFolder.server == this.mRSSServer && + this.mActionMode == this.kImportingOPML; + let folderObject = { + children: [], + folder: aFolder, + name: aFolder.prettyName, + level: aCurrentLevel, + url: aFolder.URI, + quickMode: defaultQuickMode, + options: optionsAcct, + open, + container: true, + favicon: null, + }; + + // If a feed has any sub folders, add them to the list of children. + for (let folder of aFolder.subFolders) { + if ( + folder instanceof Ci.nsIMsgFolder && + !folder.getFlag(Ci.nsMsgFolderFlags.Trash) && + !folder.getFlag(Ci.nsMsgFolderFlags.Virtual) + ) { + folderObject.children.push( + this.makeFolderObject(folder, aCurrentLevel + 1) + ); + } + } + + let feeds = this.getFeedsInFolder(aFolder); + for (let feed of feeds) { + // Now add any feed urls for the folder. + folderObject.children.push( + this.makeFeedObject(feed, aFolder, aCurrentLevel + 1) + ); + } + + // Finally, set the folder's quickMode based on the its first feed's + // quickMode, since that is how the view determines summary mode, and now + // quickMode is updated to be the same for all feeds in a folder. + if (feeds && feeds[0]) { + folderObject.quickMode = feeds[0].quickMode; + } + + folderObject.children = this.folderItemSorter(folderObject.children); + + return folderObject; + }, + + folderItemSorter(aArray) { + return aArray + .sort(function (a, b) { + return a.name.toLowerCase() > b.name.toLowerCase(); + }) + .sort(function (a, b) { + return a.container < b.container; + }); + }, + + getFeedsInFolder(aFolder) { + let feeds = []; + let feedUrlArray = FeedUtils.getFeedUrlsInFolder(aFolder); + if (!feedUrlArray) { + // No feedUrls in this folder. + return feeds; + } + + for (let url of feedUrlArray) { + let feed = new Feed(url, aFolder); + feeds.push(feed); + } + + return feeds; + }, + + makeFeedObject(aFeed, aFolder, aLevel) { + // Look inside the data source for the feed properties. + let feed = { + children: [], + parentFolder: aFolder, + name: aFeed.title || aFeed.description || aFeed.url, + url: aFeed.url, + quickMode: aFeed.quickMode, + options: aFeed.options || FeedUtils.optionsTemplate, + level: aLevel, + open: false, + container: false, + favicon: null, + }; + return feed; + }, + + loadSubscriptions() { + // Put together an array of folders. Each feed account level folder is + // included as the root. + let numFolders = 0; + let feedContainers = []; + // Get all the feed account folders. + let feedRootFolders = FeedUtils.getAllRssServerRootFolders(); + + feedRootFolders.forEach(function (rootFolder) { + feedContainers.push(this.makeFolderObject(rootFolder, 0)); + numFolders++; + }, this); + + this.mFeedContainers = feedContainers; + this.mView.mRowCount = numFolders; + + FeedSubscriptions.mTree.focus(); + }, + + /** + * Find the folder in the tree. The search may be limited to subfolders of + * a known folder, or expanded to include the entire tree. This function is + * also used to insert/remove folders without rebuilding the tree view cache + * (to avoid position/toggle state loss). + * + * @param {nsIMsgFolder} aFolder - The folder to find. + * @param {object} aParms - The params object, containing: + * + * {Integer} parentIndex - index of folder to start the search; if + * null (default), the index of the folder's + * rootFolder will be used. + * {boolean} select - if true (default) the folder's ancestors + * will be opened and the folder selected. + * {boolean} open - if true (default) the folder is opened. + * {boolean} remove - delete the item from tree row cache if true, + * false (default) otherwise. + * {nsIMsgFolder} newFolder - if not null (default) the new folder, + * for add or rename. + * + * @returns {Boolean} found - true if found, false if not. + */ + selectFolder(aFolder, aParms) { + let folderURI = aFolder.URI; + let parentIndex = + aParms && "parentIndex" in aParms ? aParms.parentIndex : null; + let selectIt = aParms && "select" in aParms ? aParms.select : true; + let openIt = aParms && "open" in aParms ? aParms.open : true; + let removeIt = aParms && "remove" in aParms ? aParms.remove : false; + let newFolder = aParms && "newFolder" in aParms ? aParms.newFolder : null; + let startIndex, startItem; + let found = false; + + let firstVisRow, curFirstVisRow, curLastVisRow; + if (this.mView.tree) { + firstVisRow = this.mView.tree.getFirstVisibleRow(); + } + + if (parentIndex != null) { + // Use the parentIndex if given. + startIndex = parentIndex; + if (aFolder.isServer) { + // Fake item for account root folder. + startItem = { + name: "AccountRoot", + children: [this.mView.getItemAtIndex(startIndex)], + container: true, + open: false, + url: null, + level: -1, + }; + } else { + startItem = this.mView.getItemAtIndex(startIndex); + } + } else { + // Get the folder's root parent index. + let index = 0; + for (index; index < this.mView.rowCount; index++) { + let item = this.mView.getItemAtIndex(index); + if (item.url == aFolder.server.rootFolder.URI) { + break; + } + } + + startIndex = index; + if (aFolder.isServer) { + // Fake item for account root folder. + startItem = { + name: "AccountRoot", + children: [this.mView.getItemAtIndex(startIndex)], + container: true, + open: false, + url: null, + level: -1, + }; + } else { + startItem = this.mView.getItemAtIndex(startIndex); + } + } + + function containsFolder(aItem) { + // Search for the folder. If it's found, set the open state on all + // ancestor folders. A toggle() rebuilds the view rows to match the map. + if (aItem.url == folderURI) { + return (found = true); + } + + for (let i = 0; i < aItem.children.length; i++) { + if (aItem.children[i].container && containsFolder(aItem.children[i])) { + if (removeIt && aItem.children[i].url == folderURI) { + // Get all occurrences in the tree cache arrays. + FeedUtils.log.debug( + "selectFolder: delete in cache, " + + "parent:children:item:index - " + + aItem.name + + ":" + + aItem.children.length + + ":" + + aItem.children[i].name + + ":" + + i + ); + aItem.children.splice(i, 1); + FeedUtils.log.debug( + "selectFolder: deleted in cache, " + + "parent:children - " + + aItem.name + + ":" + + aItem.children.length + ); + removeIt = false; + return true; + } + if (newFolder) { + let newItem = FeedSubscriptions.makeFolderObject( + newFolder, + aItem.level + 1 + ); + newItem.open = aItem.children[i].open; + if (newFolder.isServer) { + FeedSubscriptions.mFeedContainers[startIndex] = newItem; + } else { + aItem.children[i] = newItem; + aItem.children = FeedSubscriptions.folderItemSorter( + aItem.children + ); + } + FeedUtils.log.trace( + "selectFolder: parentName:newFolderName:newFolderItem - " + + aItem.name + + ":" + + newItem.name + + ":" + + newItem.toSource() + ); + newFolder = null; + return true; + } + if (!found) { + // For the folder to find. + found = true; + aItem.children[i].open = openIt; + } else if (selectIt || openIt) { + // For ancestor folders. + aItem.children[i].open = true; + } + + return true; + } + } + + return false; + } + + if (startItem) { + // Find a folder with a specific parent. + containsFolder(startItem); + if (!found) { + return false; + } + + if (!selectIt) { + return true; + } + + if (startItem.open) { + this.mView.toggle(startIndex); + } + + this.mView.toggleOpenState(startIndex); + } + + for (let index = 0; index < this.mView.rowCount && selectIt; index++) { + // The desired folder is now in the view. + let item = this.mView.getItemAtIndex(index); + if (!item.container) { + continue; + } + + if (item.url == folderURI) { + if ( + item.children.length && + ((!item.open && openIt) || (item.open && !openIt)) + ) { + this.mView.toggleOpenState(index); + } + + this.mView.selection.select(index); + found = true; + break; + } + } + + // Ensure tree position does not jump unnecessarily. + curFirstVisRow = this.mView.tree.getFirstVisibleRow(); + curLastVisRow = this.mView.tree.getLastVisibleRow(); + if ( + firstVisRow >= 0 && + this.mView.rowCount - curLastVisRow > firstVisRow - curFirstVisRow + ) { + this.mView.tree.scrollToRow(firstVisRow); + } else { + this.mView.tree.ensureRowIsVisible(this.mView.rowCount - 1); + } + + FeedUtils.log.debug( + "selectFolder: curIndex:firstVisRow:" + + "curFirstVisRow:curLastVisRow:rowCount - " + + this.mView.selection.currentIndex + + ":" + + firstVisRow + + ":" + + curFirstVisRow + + ":" + + curLastVisRow + + ":" + + this.mView.rowCount + ); + return found; + }, + + /** + * Find the feed in the tree. The search first gets the feed's folder, + * then selects the child feed. + * + * @param {Feed} aFeed - The feed to find. + * @param {Integer} aParentIndex - Index to start the folder search. + * + * @returns {Boolean} found - true if found, false if not. + */ + selectFeed(aFeed, aParentIndex) { + let folder = aFeed.folder; + let server = aFeed.server || aFeed.folder.server; + let found = false; + + if (aFeed.folder.isServer) { + // If passed the root folder, the caller wants to get the feed's folder + // from the db (for cases of an ancestor folder rename/move). + let destFolder = FeedUtils.getSubscriptionAttr( + aFeed.url, + server, + "destFolder" + ); + folder = server.rootFolder.getChildWithURI(destFolder, true, false); + } + + if (this.selectFolder(folder, { parentIndex: aParentIndex })) { + let seln = this.mView.selection; + let item = this.mView.currentItem; + if (item) { + for (let i = seln.currentIndex + 1; i < this.mView.rowCount; i++) { + if (this.mView.getItemAtIndex(i).url == aFeed.url) { + this.mView.selection.select(i); + this.mView.tree.ensureRowIsVisible(i); + found = true; + break; + } + } + } + } + + return found; + }, + + updateFeedData(aItem) { + if (!aItem) { + return; + } + + let nameValue = document.getElementById("nameValue"); + let locationValue = document.getElementById("locationValue"); + let locationValidate = document.getElementById("locationValidate"); + let isServer = aItem.folder && aItem.folder.isServer; + let isFolder = aItem.folder && !aItem.folder.isServer; + let isFeed = !aItem.container; + let server, displayFolder; + + if (isFeed) { + // A feed item. Set the feed location and title info. + nameValue.value = aItem.name; + locationValue.value = aItem.url; + locationValidate.removeAttribute("collapsed"); + + // Root the location picker to the news & blogs server. + server = aItem.parentFolder.server; + displayFolder = aItem.parentFolder; + } else { + // A folder/container item. + nameValue.value = ""; + nameValue.disabled = true; + locationValue.value = ""; + locationValidate.setAttribute("collapsed", true); + + server = aItem.folder.server; + displayFolder = aItem.folder; + } + + // Common to both folder and feed items. + nameValue.disabled = aItem.container; + this.setFolderPicker(displayFolder, isFeed); + + // Set quick mode value. + document.getElementById("quickMode").checked = aItem.quickMode; + + if (isServer) { + aItem.options = FeedUtils.getOptionsAcct(server); + } + + // Update items. + let updateEnabled = document.getElementById("updateEnabled"); + let updateValue = document.getElementById("updateValue"); + let biffUnits = document.getElementById("biffUnits"); + let recommendedUnits = document.getElementById("recommendedUnits"); + let recommendedUnitsVal = document.getElementById("recommendedUnitsVal"); + let updates = aItem.options + ? aItem.options.updates + : FeedUtils._optionsDefault.updates; + + updateEnabled.checked = updates.enabled; + updateValue.disabled = !updateEnabled.checked || isFolder; + biffUnits.disabled = !updateEnabled.checked || isFolder; + biffUnits.value = updates.updateUnits; + let minutes = + updates.updateUnits == FeedUtils.kBiffUnitsMinutes + ? updates.updateMinutes + : updates.updateMinutes / (24 * 60); + updateValue.value = Number(minutes); + if (isFeed) { + recommendedUnitsVal.value = this.getUpdateMinutesRec(updates); + } else { + recommendedUnitsVal.value = ""; + } + + let hideRec = recommendedUnitsVal.value == ""; + recommendedUnits.hidden = hideRec; + recommendedUnitsVal.hidden = hideRec; + + // Autotag items. + let autotagEnable = document.getElementById("autotagEnable"); + let autotagUsePrefix = document.getElementById("autotagUsePrefix"); + let autotagPrefix = document.getElementById("autotagPrefix"); + let category = aItem.options ? aItem.options.category : null; + + autotagEnable.checked = category && category.enabled; + autotagUsePrefix.checked = category && category.prefixEnabled; + autotagUsePrefix.disabled = !autotagEnable.checked; + autotagPrefix.disabled = + autotagUsePrefix.disabled || !autotagUsePrefix.checked; + autotagPrefix.value = category && category.prefix ? category.prefix : ""; + }, + + setFolderPicker(aFolder, aIsFeed) { + let folderPrettyPath = FeedUtils.getFolderPrettyPath(aFolder); + if (!folderPrettyPath) { + return; + } + + let selectFolder = document.getElementById("selectFolder"); + let selectFolderPopup = document.getElementById("selectFolderPopup"); + let selectFolderValue = document.getElementById("selectFolderValue"); + + selectFolder.setAttribute("hidden", !aIsFeed); + selectFolder._folder = aFolder; + selectFolderValue.toggleAttribute("hidden", aIsFeed); + selectFolderValue.setAttribute("showfilepath", false); + + if (aIsFeed) { + selectFolderPopup._ensureInitialized(); + selectFolderPopup.selectFolder(aFolder); + selectFolder.setAttribute("label", folderPrettyPath); + selectFolder.setAttribute("uri", aFolder.URI); + } else { + selectFolderValue.value = folderPrettyPath; + selectFolderValue.setAttribute("prettypath", folderPrettyPath); + selectFolderValue.setAttribute("filepath", aFolder.filePath.path); + } + }, + + onClickSelectFolderValue(aEvent) { + let target = aEvent.target; + if ( + ("button" in aEvent && + (aEvent.button != 0 || + aEvent.target.localName != "div" || + target.selectionStart != target.selectionEnd)) || + (aEvent.keyCode && aEvent.keyCode != aEvent.DOM_VK_RETURN) + ) { + return; + } + + // Toggle between showing prettyPath and absolute filePath. + if (target.getAttribute("showfilepath") == "true") { + target.setAttribute("showfilepath", false); + target.value = target.getAttribute("prettypath"); + } else { + target.setAttribute("showfilepath", true); + target.value = target.getAttribute("filepath"); + } + }, + + /** + * The user changed the folder for storing the feed. + * + * @param {Event} aEvent - Event. + * @returns {void} + */ + setNewFolder(aEvent) { + aEvent.stopPropagation(); + this.setFolderPicker(aEvent.target._folder, true); + + let seln = this.mView.selection; + if (seln.count != 1) { + return; + } + + let item = this.mView.getItemAtIndex(seln.currentIndex); + if (!item || item.container || !item.parentFolder) { + return; + } + + let selectFolder = document.getElementById("selectFolder"); + let editFolderURI = selectFolder.getAttribute("uri"); + if (item.parentFolder.URI == editFolderURI) { + return; + } + + let feed = new Feed(item.url, item.parentFolder); + + // Make sure the new folderpicked folder is visible. + this.selectFolder(selectFolder._folder); + // Now go back to the feed item. + this.selectFeed(feed, null); + // We need to find the index of the new parent folder. + let newParentIndex = this.mView.kRowIndexUndefined; + for (let index = 0; index < this.mView.rowCount; index++) { + let item = this.mView.getItemAtIndex(index); + if (item && item.container && item.url == editFolderURI) { + newParentIndex = index; + break; + } + } + + if (newParentIndex != this.mView.kRowIndexUndefined) { + this.moveCopyFeed(seln.currentIndex, newParentIndex, "move"); + } + }, + + setSummary(aChecked) { + let item = this.mView.currentItem; + if (!item || !item.folder) { + // Not a folder. + return; + } + + if (item.folder.isServer) { + if (document.getElementById("locationValue").value) { + // Intent is to add a feed/folder to the account, so return. + return; + } + + // An account folder. If it changes, all non feed containing subfolders + // need to be updated with the new default. + item.folder.server.setBoolValue("quickMode", aChecked); + this.FolderListener.folderAdded(item.folder); + } else if (!FeedUtils.getFeedUrlsInFolder(item.folder)) { + // Not a folder with feeds. + return; + } else { + let feedsInFolder = this.getFeedsInFolder(item.folder); + // Update the feeds database, for each feed in the folder. + feedsInFolder.forEach(function (feed) { + feed.quickMode = aChecked; + }); + // Update the folder's feeds properties in the tree map. + item.children.forEach(function (feed) { + feed.quickMode = aChecked; + }); + } + + // Update the folder in the tree map. + item.quickMode = aChecked; + let message = FeedUtils.strings.GetStringFromName("subscribe-feedUpdated"); + this.updateStatusItem("statusText", message); + }, + + setPrefs(aNode) { + let item = this.mView.currentItem; + if (!item) { + return; + } + + let isServer = item.folder && item.folder.isServer; + let isFolder = item.folder && !item.folder.isServer; + let updateEnabled = document.getElementById("updateEnabled"); + let updateValue = document.getElementById("updateValue"); + let biffUnits = document.getElementById("biffUnits"); + let autotagEnable = document.getElementById("autotagEnable"); + let autotagUsePrefix = document.getElementById("autotagUsePrefix"); + let autotagPrefix = document.getElementById("autotagPrefix"); + if ( + isFolder || + (isServer && document.getElementById("locationValue").value) + ) { + // Intend to subscribe a feed to a folder, a value must be in the url + // field. Update states for addFeed() and return. + updateValue.disabled = !updateEnabled.checked; + biffUnits.disabled = !updateEnabled.checked; + autotagUsePrefix.disabled = !autotagEnable.checked; + autotagPrefix.disabled = + autotagUsePrefix.disabled || !autotagUsePrefix.checked; + return; + } + + switch (aNode.id) { + case "nameValue": + // Check to see if the title value changed, no blank title allowed. + if (!aNode.value) { + aNode.value = item.name; + return; + } + + item.name = aNode.value; + let seln = this.mView.selection; + seln.tree.invalidateRow(seln.currentIndex); + break; + case "locationValue": + let updateFeedButton = document.getElementById("updateFeed"); + // Change label based on whether feed url has beed edited. + updateFeedButton.label = + aNode.value == item.url + ? updateFeedButton.getAttribute("verifylabel") + : updateFeedButton.getAttribute("updatelabel"); + updateFeedButton.setAttribute( + "accesskey", + aNode.value == item.url + ? updateFeedButton.getAttribute("verifyaccesskey") + : updateFeedButton.getAttribute("updateaccesskey") + ); + // Disable the Update button if no feed url value is entered. + updateFeedButton.disabled = !aNode.value; + return; + case "updateEnabled": + case "updateValue": + case "biffUnits": + item.options.updates.enabled = updateEnabled.checked; + let minutes = + biffUnits.value == FeedUtils.kBiffUnitsMinutes + ? updateValue.value + : updateValue.value * 24 * 60; + item.options.updates.updateMinutes = Number(minutes); + item.options.updates.updateUnits = biffUnits.value; + break; + case "autotagEnable": + item.options.category.enabled = aNode.checked; + break; + case "autotagUsePrefix": + item.options.category.prefixEnabled = aNode.checked; + item.options.category.prefix = autotagPrefix.value; + break; + case "autotagPrefix": + item.options.category.prefix = aNode.value; + break; + } + + if (isServer) { + FeedUtils.setOptionsAcct(item.folder.server, item.options); + } else { + let feed = new Feed(item.url, item.parentFolder); + feed.title = item.name; + feed.options = item.options; + + if (aNode.id == "updateEnabled") { + FeedUtils.setStatus( + item.parentFolder, + item.url, + "enabled", + aNode.checked + ); + this.mView.selection.tree.invalidateRow( + this.mView.selection.currentIndex + ); + } + if (aNode.id == "updateValue") { + FeedUtils.setStatus( + item.parentFolder, + item.url, + "updateMinutes", + item.options.updates.updateMinutes + ); + } + } + + this.updateFeedData(item); + let message = FeedUtils.strings.GetStringFromName("subscribe-feedUpdated"); + this.updateStatusItem("statusText", message); + }, + + getUpdateMinutesRec(aUpdates) { + // Assume the parser has stored correct/valid values for the spec. If the + // feed doesn't use any of these tags, updatePeriod will be null. + if (aUpdates.updatePeriod == null) { + return ""; + } + + let biffUnits = document.getElementById("biffUnits").value; + let units = biffUnits == FeedUtils.kBiffUnitsDays ? 1 : 24 * 60; + let frequency = aUpdates.updateFrequency; + let val; + switch (aUpdates.updatePeriod) { + case "hourly": + val = + biffUnits == FeedUtils.kBiffUnitsDays + ? 1 / frequency / 24 + : 60 / frequency; + break; + case "daily": + val = units / frequency; + break; + case "weekly": + val = (7 * units) / frequency; + break; + case "monthly": + val = (30 * units) / frequency; + break; + case "yearly": + val = (365 * units) / frequency; + break; + } + + return val ? Math.round(val * 1000) / 1000 : ""; + }, + + onKeyPress(aEvent) { + if ( + aEvent.keyCode == aEvent.DOM_VK_DELETE && + aEvent.target.id == "rssSubscriptionsList" + ) { + this.removeFeed(true); + } + + this.clearStatusInfo(); + }, + + onSelect() { + let item = this.mView.currentItem; + this.updateFeedData(item); + this.setFocus(); + this.updateButtons(item); + }, + + updateButtons(aSelectedItem) { + let item = aSelectedItem; + let isServer = item && item.folder && item.folder.isServer; + let isFeed = item && !item.container; + document.getElementById("addFeed").hidden = !item || isFeed; + document.getElementById("updateFeed").hidden = !isFeed; + document.getElementById("removeFeed").hidden = !isFeed; + document.getElementById("importOPML").hidden = !isServer; + document.getElementById("exportOPML").hidden = !isServer; + + document.getElementById("importOPML").disabled = document.getElementById( + "exportOPML" + ).disabled = this.mActionMode == this.kImportingOPML; + }, + + onMouseDown(aEvent) { + if ( + aEvent.button != 0 || + aEvent.target.id == "validationText" || + aEvent.target.id == "addCertException" + ) { + return; + } + + this.clearStatusInfo(); + }, + + onFocusChange() { + setTimeout(() => { + this.setFocus(); + }, 0); + }, + + setFocus() { + let item = this.mView.currentItem; + if (!item || this.mActionMode == this.kImportingOPML) { + return; + } + + let locationValue = document.getElementById("locationValue"); + let updateEnabled = document.getElementById("updateEnabled"); + + let quickMode = document.getElementById("quickMode"); + let autotagEnable = document.getElementById("autotagEnable"); + let autotagUsePrefix = document.getElementById("autotagUsePrefix"); + let autotagPrefix = document.getElementById("autotagPrefix"); + + let addFeedButton = document.getElementById("addFeed"); + let updateFeedButton = document.getElementById("updateFeed"); + + let isServer = item.folder && item.folder.isServer; + let isFolder = item.folder && !item.folder.isServer; + + // Enabled by default. + updateEnabled.disabled = + quickMode.disabled = + autotagEnable.disabled = + false; + + updateEnabled.parentNode + .querySelectorAll("input,radio,label") + .forEach(item => { + item.disabled = !updateEnabled.checked; + }); + + autotagUsePrefix.disabled = !autotagEnable.checked; + autotagPrefix.disabled = + autotagUsePrefix.disabled || !autotagUsePrefix.checked; + + let focusedElement = window.document.commandDispatcher.focusedElement; + + if (isServer) { + addFeedButton.disabled = + addFeedButton != focusedElement && + locationValue != document.activeElement && + !locationValue.value; + } else if (isFolder) { + let disable = + locationValue != document.activeElement && !locationValue.value; + // Summary is enabled for a folder with feeds or if adding a feed. + quickMode.disabled = + disable && !FeedUtils.getFeedUrlsInFolder(item.folder); + // All other options disabled unless intent is to add a feed. + updateEnabled.disabled = disable; + updateEnabled.parentNode + .querySelectorAll("input,radio,label") + .forEach(item => { + item.disabled = disable; + }); + + autotagEnable.disabled = disable; + + addFeedButton.disabled = + addFeedButton != focusedElement && + locationValue != document.activeElement && + !locationValue.value; + } else { + // Summary is disabled; applied per folder to apply to all feeds in it. + quickMode.disabled = true; + // Ensure the current feed url is restored if the user did not update. + if ( + locationValue.value != item.url && + locationValue != document.activeElement && + focusedElement != updateFeedButton && + focusedElement.id != "addCertException" + ) { + locationValue.value = item.url; + } + this.setPrefs(locationValue); + // Set button state. + updateFeedButton.disabled = !locationValue.value; + } + }, + + removeFeed(aPrompt) { + let seln = this.mView.selection; + if (seln.count != 1) { + return; + } + + let itemToRemove = this.mView.getItemAtIndex(seln.currentIndex); + + if (!itemToRemove || itemToRemove.container) { + return; + } + + if (aPrompt) { + // Confirm unsubscribe prompt. + let pTitle = FeedUtils.strings.GetStringFromName( + "subscribe-confirmFeedDeletionTitle" + ); + let pMessage = FeedUtils.strings.formatStringFromName( + "subscribe-confirmFeedDeletion", + [itemToRemove.name] + ); + if ( + Services.prompt.confirmEx( + window, + pTitle, + pMessage, + Ci.nsIPromptService.STD_YES_NO_BUTTONS, + null, + null, + null, + null, + {} + ) + ) { + return; + } + } + + let feed = new Feed(itemToRemove.url, itemToRemove.parentFolder); + FeedUtils.deleteFeed(feed); + + // Now that we have removed the feed from the datasource, it is time to + // update our view layer. Update parent folder's quickMode if necessary + // and remove the child from its parent folder object. + let parentIndex = this.mView.getParentIndex(seln.currentIndex); + let parentItem = this.mView.getItemAtIndex(parentIndex); + this.updateFolderQuickModeInView(itemToRemove, parentItem, true); + this.mView.removeItemAtIndex(seln.currentIndex, false); + let message = FeedUtils.strings.GetStringFromName("subscribe-feedRemoved"); + this.updateStatusItem("statusText", message); + }, + + /** + * This addFeed is used by 1) Add button, 1) Update button, 3) Drop of a + * feed url on a folder (which can be an add or move). If Update, the new + * url is added and the old removed; thus aParse is false and no new messages + * are downloaded, the feed is only validated and stored in the db. If dnd, + * the drop folder is selected and the url is prefilled, so proceed just as + * though the url were entered manually. This allows a user to see the dnd + * url better in case of errors. + * + * @param {String} aFeedLocation - the feed url; get the url from the + * input field if null. + * @param {nsIMsgFolder} aFolder - folder to subscribe, current selected + * folder if null. + * @param {Boolean} aParse - if true (default) parse and download + * the feed's articles. + * @param {Object} aParams - additional params. + * @param {Integer} aMode - action mode (default is kSubscribeMode) + * of the add. + * + * @returns {Boolean} success - true if edit checks passed and an + * async download has been initiated. + */ + addFeed(aFeedLocation, aFolder, aParse, aParams, aMode) { + let message; + let parse = aParse == null ? true : aParse; + let mode = aMode == null ? this.kSubscribeMode : aMode; + let locationValue = document.getElementById("locationValue"); + let quickMode = + aParams && "quickMode" in aParams + ? aParams.quickMode + : document.getElementById("quickMode").checked; + let name = + aParams && "name" in aParams + ? aParams.name + : document.getElementById("nameValue").value; + let options = aParams && "options" in aParams ? aParams.options : null; + + if (aFeedLocation) { + locationValue.value = aFeedLocation; + } + let feedLocation = locationValue.value.trim(); + + if (!feedLocation) { + locationValue.focus(); + message = locationValue.getAttribute("placeholder"); + this.updateStatusItem("statusText", message); + return false; + } + + if (!FeedUtils.isValidScheme(feedLocation)) { + locationValue.focus(); + message = FeedUtils.strings.GetStringFromName("subscribe-feedNotValid"); + this.updateStatusItem("statusText", message); + return false; + } + + let addFolder; + if (aFolder) { + // For Update or if passed a folder. + if (aFolder instanceof Ci.nsIMsgFolder) { + addFolder = aFolder; + } + } else { + // A folder must be selected for Add and Drop. + let index = this.mView.selection.currentIndex; + let item = this.mView.getItemAtIndex(index); + if (item && item.container) { + addFolder = item.folder; + } + } + + // Shouldn't happen. Or else not passed an nsIMsgFolder. + if (!addFolder) { + return false; + } + + // Before we go any further, make sure the user is not already subscribed + // to this feed. + if (FeedUtils.feedAlreadyExists(feedLocation, addFolder.server)) { + locationValue.focus(); + message = FeedUtils.strings.GetStringFromName( + "subscribe-feedAlreadySubscribed" + ); + this.updateStatusItem("statusText", message); + return false; + } + + if (!options) { + // Not passed a param, get values from the ui. + options = FeedUtils.optionsTemplate; + options.updates.enabled = + document.getElementById("updateEnabled").checked; + let biffUnits = document.getElementById("biffUnits").value; + let units = document.getElementById("updateValue").value; + let minutes = + biffUnits == FeedUtils.kBiffUnitsMinutes ? units : units * 24 * 60; + options.updates.updateUnits = biffUnits; + options.updates.updateMinutes = Number(minutes); + options.category.enabled = + document.getElementById("autotagEnable").checked; + options.category.prefixEnabled = + document.getElementById("autotagUsePrefix").checked; + options.category.prefix = document.getElementById("autotagPrefix").value; + } + + let feedProperties = { + feedName: name, + feedLocation, + feedFolder: addFolder, + quickMode, + options, + }; + + let feed = this.storeFeed(feedProperties); + if (!feed) { + return false; + } + + // Now validate and start downloading the feed. + message = FeedUtils.strings.GetStringFromName("subscribe-validating-feed"); + this.updateStatusItem("statusText", message); + this.updateStatusItem("progressMeter", "?"); + document.getElementById("addFeed").disabled = true; + this.mActionMode = mode; + feed.download(parse, this.mFeedDownloadCallback); + return true; + }, + + // Helper routine used by addFeed and importOPMLFile. + storeFeed(feedProperties) { + let feed = new Feed(feedProperties.feedLocation, feedProperties.feedFolder); + feed.title = feedProperties.feedName; + feed.quickMode = feedProperties.quickMode; + feed.options = feedProperties.options; + return feed; + }, + + /** + * When a feed item is selected, the Update button is used to verify the + * existing feed url, or to verify and update the feed url if the field + * has been edited. This is the only use of the Update button. + * + * @returns {void} + */ + updateFeed() { + let seln = this.mView.selection; + if (seln.count != 1) { + return; + } + + let item = this.mView.getItemAtIndex(seln.currentIndex); + if (!item || item.container || !item.parentFolder) { + return; + } + + let feed = new Feed(item.url, item.parentFolder); + + // Disable the button. + document.getElementById("updateFeed").disabled = true; + + let feedLocation = document.getElementById("locationValue").value.trim(); + if (feed.url != feedLocation) { + // Updating a url. We need to add the new url and delete the old, to + // ensure everything is cleaned up correctly. + this.addFeed(null, item.parentFolder, false, null, this.kUpdateMode); + return; + } + + // Now we want to verify if the stored feed url still works. If it + // doesn't, show the error. + let message = FeedUtils.strings.GetStringFromName( + "subscribe-validating-feed" + ); + this.mActionMode = this.kVerifyUrlMode; + this.updateStatusItem("statusText", message); + this.updateStatusItem("progressMeter", "?"); + feed.download(false, this.mFeedDownloadCallback); + }, + + /** + * Moves or copies a feed to another folder or account. + * + * @param {Integer} aOldFeedIndex - Index in tree of target feed item. + * @param {Integer} aNewParentIndex - Index in tree of target parent folder item. + * @param {String} aMoveCopy - Either "move" or "copy". + * + * @returns {void} + */ + moveCopyFeed(aOldFeedIndex, aNewParentIndex, aMoveCopy) { + let moveFeed = aMoveCopy == "move"; + let currentItem = this.mView.getItemAtIndex(aOldFeedIndex); + if ( + !currentItem || + this.mView.getParentIndex(aOldFeedIndex) == aNewParentIndex + ) { + // If the new parent is the same as the current parent, then do nothing. + return; + } + + let currentParentIndex = this.mView.getParentIndex(aOldFeedIndex); + let currentParentItem = this.mView.getItemAtIndex(currentParentIndex); + let currentFolder = currentParentItem.folder; + + let newParentItem = this.mView.getItemAtIndex(aNewParentIndex); + let newFolder = newParentItem.folder; + + let accountMoveCopy = false; + if (currentFolder.rootFolder.URI == newFolder.rootFolder.URI) { + // Moving within the same account/feeds db. + if (newFolder.isServer || !moveFeed) { + // No moving to account folder if already in the account; can only move, + // not copy, to folder in the same account. + return; + } + + // Update the destFolder for this feed's subscription. + FeedUtils.setSubscriptionAttr( + currentItem.url, + currentItem.parentFolder.server, + "destFolder", + newFolder.URI + ); + + // Update folderpane favicons. + Services.obs.notifyObservers(currentFolder, "folder-properties-changed"); + Services.obs.notifyObservers(newFolder, "folder-properties-changed"); + } else { + // Moving/copying to a new account. If dropping on the account folder, + // a new subfolder is created if necessary. + accountMoveCopy = true; + let mode = moveFeed ? this.kMoveMode : this.kCopyMode; + let params = { + quickMode: currentItem.quickMode, + name: currentItem.name, + options: currentItem.options, + }; + // Subscribe to the new folder first. If it already exists in the + // account or on error, return. + if (!this.addFeed(currentItem.url, newFolder, false, params, mode)) { + return; + } + // Unsubscribe the feed from the old folder, if add to the new folder + // is successful, and doing a move. + if (moveFeed) { + let feed = new Feed(currentItem.url, currentItem.parentFolder); + FeedUtils.deleteFeed(feed); + } + } + + // Update local favicons. + currentParentItem.favicon = newParentItem.favicon = null; + + // Finally, update our view layer. Update old parent folder's quickMode + // and remove the old row, if move. Otherwise no change to the view. + if (moveFeed) { + this.updateFolderQuickModeInView(currentItem, currentParentItem, true); + this.mView.removeItemAtIndex(aOldFeedIndex, true); + if (aNewParentIndex > aOldFeedIndex) { + aNewParentIndex--; + } + } + + if (accountMoveCopy) { + // If a cross account move/copy, download callback will update the view + // with the new location. Preselect folder/mode for callback. + this.selectFolder(newFolder, { parentIndex: aNewParentIndex }); + return; + } + + // Add the new row location to the view. + currentItem.level = newParentItem.level + 1; + currentItem.parentFolder = newFolder; + this.updateFolderQuickModeInView(currentItem, newParentItem, false); + newParentItem.children.push(currentItem); + + if (newParentItem.open) { + // Close the container, selecting the feed will rebuild the view rows. + this.mView.toggle(aNewParentIndex); + } + + this.selectFeed( + { folder: newParentItem.folder, url: currentItem.url }, + aNewParentIndex + ); + + let message = FeedUtils.strings.GetStringFromName("subscribe-feedMoved"); + this.updateStatusItem("statusText", message); + }, + + updateFolderQuickModeInView(aFeedItem, aParentItem, aRemove) { + let feedItem = aFeedItem; + let parentItem = aParentItem; + let feedUrlArray = FeedUtils.getFeedUrlsInFolder(feedItem.parentFolder); + let feedsInFolder = feedUrlArray ? feedUrlArray.length : 0; + + if (aRemove && feedsInFolder < 1) { + // Removed only feed in folder; set quickMode to server default. + parentItem.quickMode = parentItem.folder.server.getBoolValue("quickMode"); + } + + if (!aRemove) { + // Just added a feed to a folder. If there are already feeds in the + // folder, the feed must reflect the parent's quickMode. If it is the + // only feed, update the parent folder to the feed's quickMode. + if (feedsInFolder > 1) { + let feed = new Feed(feedItem.url, feedItem.parentFolder); + feed.quickMode = parentItem.quickMode; + feedItem.quickMode = parentItem.quickMode; + } else { + parentItem.quickMode = feedItem.quickMode; + } + } + }, + + onDragStart(aEvent) { + // Get the selected feed article (if there is one). + let seln = this.mView.selection; + if (seln.count != 1) { + return; + } + + // Only initiate a drag if the item is a feed (ignore folders/containers). + let item = this.mView.getItemAtIndex(seln.currentIndex); + if (!item || item.container) { + return; + } + + aEvent.dataTransfer.setData("text/x-moz-feed-index", seln.currentIndex); + aEvent.dataTransfer.effectAllowed = "copyMove"; + }, + + onDragOver(aEvent) { + this.mView._currentDataTransfer = aEvent.dataTransfer; + }, + + mFeedDownloadCallback: { + mSubscribeMode: true, + async downloaded(feed, aErrorCode) { + // Offline check is done in the context of 3pane, return to the subscribe + // window once the modal prompt is dispatched. + window.focus(); + // Feed is null if our attempt to parse the feed failed. + let message = ""; + let win = FeedSubscriptions; + if ( + aErrorCode == FeedUtils.kNewsBlogSuccess || + aErrorCode == FeedUtils.kNewsBlogNoNewItems + ) { + win.updateStatusItem("progressMeter", 100); + + if (win.mActionMode == win.kVerifyUrlMode) { + // Just checking for errors, if none bye. The (non error) code + // kNewsBlogNoNewItems can only happen in verify mode. + win.mActionMode = null; + win.clearStatusInfo(); + if (Services.io.offline) { + return; + } + + message = FeedUtils.strings.GetStringFromName( + "subscribe-feedVerified" + ); + win.updateStatusItem("statusText", message); + return; + } + + // Update lastUpdateTime if successful. + let options = feed.options; + options.updates.lastUpdateTime = Date.now(); + feed.options = options; + + // Add the feed to the databases. + FeedUtils.addFeed(feed); + + // Set isBusy status, and clear it after getting favicon. This makes + // sure the folder icon is redrawn to reflect what we got. + FeedUtils.setStatus( + feed.folder, + feed.url, + "code", + FeedUtils.kNewsBlogFeedIsBusy + ); + await FeedUtils.getFavicon(feed.folder, feed.url); + FeedUtils.setStatus(feed.folder, feed.url, "code", aErrorCode); + + // Now add the feed to our view. If adding, the current selection will + // be a folder; if updating it will be a feed. No need to rebuild the + // entire view, that is too jarring. + let curIndex = win.mView.selection.currentIndex; + let curItem = win.mView.getItemAtIndex(curIndex); + if (curItem) { + let parentIndex, parentItem, newItem, level; + if (curItem.container) { + // Open the container, if it exists. + let folderExists = win.selectFolder(feed.folder, { + parentIndex: curIndex, + }); + if (!folderExists) { + // This means a new folder was created. + parentIndex = curIndex; + parentItem = curItem; + level = curItem.level + 1; + newItem = win.makeFolderObject(feed.folder, level); + } else { + // If a folder happens to exist which matches one that would + // have been created, the feed system reuses it. Get the + // current item again if reusing a previously unselected folder. + curIndex = win.mView.selection.currentIndex; + curItem = win.mView.getItemAtIndex(curIndex); + parentIndex = curIndex; + parentItem = curItem; + level = curItem.level + 1; + newItem = win.makeFeedObject(feed, feed.folder, level); + } + } else { + // Adding a feed. + parentIndex = win.mView.getParentIndex(curIndex); + parentItem = win.mView.getItemAtIndex(parentIndex); + level = curItem.level; + newItem = win.makeFeedObject(feed, feed.folder, level); + } + + if (!newItem.container) { + win.updateFolderQuickModeInView(newItem, parentItem, false); + } + + parentItem.children.push(newItem); + parentItem.children = win.folderItemSorter(parentItem.children); + parentItem.favicon = null; + + if (win.mActionMode == win.kSubscribeMode) { + message = FeedUtils.strings.GetStringFromName( + "subscribe-feedAdded" + ); + } + if (win.mActionMode == win.kUpdateMode) { + win.removeFeed(false); + message = FeedUtils.strings.GetStringFromName( + "subscribe-feedUpdated" + ); + } + if (win.mActionMode == win.kMoveMode) { + message = FeedUtils.strings.GetStringFromName( + "subscribe-feedMoved" + ); + } + if (win.mActionMode == win.kCopyMode) { + message = FeedUtils.strings.GetStringFromName( + "subscribe-feedCopied" + ); + } + + win.selectFeed(feed, parentIndex); + } + } else { + // Non success. Remove intermediate traces from the feeds database. + // But only if we're not in verify mode. + if ( + win.mActionMode != win.kVerifyUrlMode && + feed && + feed.url && + feed.server + ) { + FeedUtils.deleteFeed(feed); + } + + if (aErrorCode == FeedUtils.kNewsBlogInvalidFeed) { + message = FeedUtils.strings.GetStringFromName( + "subscribe-feedNotValid" + ); + } + if (aErrorCode == FeedUtils.kNewsBlogRequestFailure) { + message = FeedUtils.strings.GetStringFromName( + "subscribe-networkError" + ); + } + if (aErrorCode == FeedUtils.kNewsBlogFileError) { + message = FeedUtils.strings.GetStringFromName( + "subscribe-errorOpeningFile" + ); + } + if (aErrorCode == FeedUtils.kNewsBlogBadCertError) { + let host = Services.io.newURI(feed.url).host; + message = FeedUtils.strings.formatStringFromName( + "newsblog-badCertError", + [host] + ); + } + if (aErrorCode == FeedUtils.kNewsBlogNoAuthError) { + message = FeedUtils.strings.GetStringFromName( + "subscribe-noAuthError" + ); + } + + // Focus the url if verify/update failed. + if ( + win.mActionMode == win.kUpdateMode || + win.mActionMode == win.kVerifyUrlMode + ) { + document.getElementById("locationValue").focus(); + } + } + + win.mActionMode = null; + win.clearStatusInfo(); + let code = feed.url.startsWith("http") ? aErrorCode : null; + win.updateStatusItem("statusText", message, code); + }, + + // This gets called after the RSS parser finishes storing a feed item to + // disk. aCurrentFeedItems is an integer corresponding to how many feed + // items have been downloaded so far. aMaxFeedItems is an integer + // corresponding to the total number of feed items to download. + onFeedItemStored(feed, aCurrentFeedItems, aMaxFeedItems) { + window.focus(); + let message = FeedUtils.strings.formatStringFromName( + "subscribe-gettingFeedItems", + [aCurrentFeedItems, aMaxFeedItems] + ); + FeedSubscriptions.updateStatusItem("statusText", message); + this.onProgress(feed, aCurrentFeedItems, aMaxFeedItems); + }, + + onProgress(feed, aProgress, aProgressMax, aLengthComputable) { + FeedSubscriptions.updateStatusItem( + "progressMeter", + (aProgress * 100) / (aProgressMax || 100) + ); + }, + }, + + // Status routines. + updateStatusItem(aID, aValue, aErrorCode) { + let el = document.getElementById(aID); + if (el.getAttribute("collapsed")) { + el.removeAttribute("collapsed"); + } + if (el.hidden) { + el.hidden = false; + } + + if (aID == "progressMeter") { + if (aValue == "?") { + el.removeAttribute("value"); + } else { + el.value = aValue; + } + } else if (aID == "statusText") { + el.textContent = aValue; + } + + el = document.getElementById("validationText"); + if (aErrorCode == FeedUtils.kNewsBlogInvalidFeed) { + el.removeAttribute("collapsed"); + } else { + el.setAttribute("collapsed", true); + } + + el = document.getElementById("addCertException"); + if (aErrorCode == FeedUtils.kNewsBlogBadCertError) { + el.removeAttribute("collapsed"); + } else { + el.setAttribute("collapsed", true); + } + }, + + clearStatusInfo() { + document.getElementById("statusText").textContent = ""; + document.getElementById("progressMeter").hidden = true; + document.getElementById("validationText").collapsed = true; + document.getElementById("addCertException").collapsed = true; + }, + + checkValidation(aEvent) { + if (aEvent.button != 0) { + return; + } + + let validationQuery = "http://validator.w3.org/feed/check.cgi?url="; + + if (this.mMainWin) { + let tabmail = this.mMainWin.document.getElementById("tabmail"); + if (tabmail) { + let feedLocation = document.getElementById("locationValue").value; + let url = validationQuery + encodeURIComponent(feedLocation); + + this.mMainWin.focus(); + this.mMainWin.openContentTab(url); + FeedUtils.log.debug("checkValidation: query url - " + url); + } + } + aEvent.stopPropagation(); + }, + + addCertExceptionDialog() { + let locationValue = document.getElementById("locationValue"); + let feedURL = locationValue.value.trim(); + let params = { + exceptionAdded: false, + location: feedURL, + prefetchCert: true, + }; + window.openDialog( + "chrome://pippki/content/exceptionDialog.xhtml", + "", + "chrome,centerscreen,modal", + params + ); + if (params.exceptionAdded) { + this.clearStatusInfo(); + } + + locationValue.focus(); + }, + + // Listener for folder pane changes. + FolderListener: { + get feedWindow() { + let subscriptionsWindow = Services.wm.getMostRecentWindow( + "Mail:News-BlogSubscriptions" + ); + return subscriptionsWindow ? subscriptionsWindow.FeedSubscriptions : null; + }, + + get currentSelectedIndex() { + return this.feedWindow + ? this.feedWindow.mView.selection.currentIndex + : -1; + }, + + get currentSelectedItem() { + return this.feedWindow ? this.feedWindow.mView.currentItem : null; + }, + + folderAdded(aFolder) { + if (aFolder.server.type != "rss" || FeedUtils.isInTrash(aFolder)) { + return; + } + + let parentFolder = aFolder.isServer ? aFolder : aFolder.parent; + FeedUtils.log.debug( + "folderAdded: folder:parent - " + + aFolder.name + + ":" + + (parentFolder ? parentFolder.filePath.path : "(null)") + ); + + if (!parentFolder || !this.feedWindow) { + return; + } + + let feedWindow = this.feedWindow; + let curSelItem = this.currentSelectedItem; + let firstVisRow = feedWindow.mView.tree.getFirstVisibleRow(); + let indexInView = feedWindow.mView.getItemInViewIndex(parentFolder); + let open = indexInView != null; + + if (aFolder.isServer) { + if (indexInView != null) { + // Existing account root folder in the view. + open = feedWindow.mView.getItemAtIndex(indexInView).open; + } else { + // Add the account root folder to the view. + feedWindow.mFeedContainers.push( + feedWindow.makeFolderObject(parentFolder, 0) + ); + feedWindow.mView.mRowCount++; + feedWindow.mTree.view = feedWindow.mView; + feedWindow.mView.tree.scrollToRow(firstVisRow); + return; + } + } + + // Rebuild the added folder's parent item in the tree row cache. + feedWindow.selectFolder(parentFolder, { + select: false, + open, + newFolder: parentFolder, + }); + + if (indexInView == null || !curSelItem) { + // Folder isn't in the tree view, no need to update the view. + return; + } + + let parentIndex = feedWindow.mView.getParentIndex(indexInView); + if (parentIndex == feedWindow.mView.kRowIndexUndefined) { + // Root folder is its own parent. + parentIndex = indexInView; + } + + if (open) { + // Close an open parent (or root) folder. + feedWindow.mView.toggle(parentIndex); + feedWindow.mView.toggleOpenState(parentIndex); + } + + feedWindow.mView.tree.scrollToRow(firstVisRow); + + if (curSelItem.container) { + feedWindow.selectFolder(curSelItem.folder, { open: curSelItem.open }); + } else { + feedWindow.selectFeed( + { folder: curSelItem.parentFolder, url: curSelItem.url }, + parentIndex + ); + } + }, + + folderDeleted(aFolder) { + if (aFolder.server.type != "rss" || FeedUtils.isInTrash(aFolder)) { + return; + } + + FeedUtils.log.debug("folderDeleted: folder - " + aFolder.name); + if (!this.feedWindow) { + return; + } + + let feedWindow = this.feedWindow; + let curSelIndex = this.currentSelectedIndex; + let indexInView = feedWindow.mView.getItemInViewIndex(aFolder); + let open = indexInView != null; + + // Delete the folder from the tree row cache. + feedWindow.selectFolder(aFolder, { + select: false, + open: false, + remove: true, + }); + + if (!open || curSelIndex < 0) { + // Folder isn't in the tree view, no need to update the view. + return; + } + + let select = + indexInView == curSelIndex || + feedWindow.mView.isIndexChildOfParentIndex(indexInView, curSelIndex); + + feedWindow.mView.removeItemAtIndex(indexInView, !select); + }, + + folderRenamed(aOrigFolder, aNewFolder) { + if (aNewFolder.server.type != "rss" || FeedUtils.isInTrash(aNewFolder)) { + return; + } + + FeedUtils.log.debug( + "folderRenamed: old:new - " + aOrigFolder.name + ":" + aNewFolder.name + ); + if (!this.feedWindow) { + return; + } + + let feedWindow = this.feedWindow; + let curSelIndex = this.currentSelectedIndex; + let curSelItem = this.currentSelectedItem; + let firstVisRow = feedWindow.mView.tree.getFirstVisibleRow(); + let indexInView = feedWindow.mView.getItemInViewIndex(aOrigFolder); + let open = indexInView != null; + + // Rebuild the renamed folder's item in the tree row cache. + feedWindow.selectFolder(aOrigFolder, { + select: false, + open, + newFolder: aNewFolder, + }); + + if (!open || !curSelItem) { + // Folder isn't in the tree view, no need to update the view. + return; + } + + let select = + indexInView == curSelIndex || + feedWindow.mView.isIndexChildOfParentIndex(indexInView, curSelIndex); + + let parentIndex = feedWindow.mView.getParentIndex(indexInView); + if (parentIndex == feedWindow.mView.kRowIndexUndefined) { + // Root folder is its own parent. + parentIndex = indexInView; + } + + feedWindow.mView.toggle(parentIndex); + feedWindow.mView.toggleOpenState(parentIndex); + feedWindow.mView.tree.scrollToRow(firstVisRow); + + if (curSelItem.container) { + if (curSelItem.folder == aOrigFolder) { + feedWindow.selectFolder(aNewFolder, { open: curSelItem.open }); + } else if (select) { + feedWindow.mView.selection.select(indexInView); + } else { + feedWindow.selectFolder(curSelItem.folder, { open: curSelItem.open }); + } + } else { + feedWindow.selectFeed( + { folder: curSelItem.parentFolder.rootFolder, url: curSelItem.url }, + parentIndex + ); + } + }, + + folderMoveCopyCompleted(aMove, aSrcFolder, aDestFolder) { + if (aDestFolder.server.type != "rss") { + return; + } + + FeedUtils.log.debug( + "folderMoveCopyCompleted: move:src:dest - " + + aMove + + ":" + + aSrcFolder.name + + ":" + + aDestFolder.name + ); + if (!this.feedWindow) { + return; + } + + let feedWindow = this.feedWindow; + let curSelIndex = this.currentSelectedIndex; + let curSelItem = this.currentSelectedItem; + let firstVisRow = feedWindow.mView.tree.getFirstVisibleRow(); + let indexInView = feedWindow.mView.getItemInViewIndex(aSrcFolder); + let destIndexInView = feedWindow.mView.getItemInViewIndex(aDestFolder); + let open = indexInView != null || destIndexInView != null; + let parentIndex = feedWindow.mView.getItemInViewIndex( + aDestFolder.parent || aDestFolder + ); + let select = + indexInView == curSelIndex || + feedWindow.mView.isIndexChildOfParentIndex(indexInView, curSelIndex); + + if (aMove) { + this.folderDeleted(aSrcFolder); + if (aDestFolder.getFlag(Ci.nsMsgFolderFlags.Trash)) { + return; + } + } + + setTimeout(() => { + // State on disk needs to settle before a folder object can be rebuilt. + feedWindow.selectFolder(aDestFolder, { + select: false, + open: open || select, + newFolder: aDestFolder, + }); + + if (!open || !curSelItem) { + // Folder isn't in the tree view, no need to update the view. + return; + } + + feedWindow.mView.toggle(parentIndex); + feedWindow.mView.toggleOpenState(parentIndex); + feedWindow.mView.tree.scrollToRow(firstVisRow); + + if (curSelItem.container) { + if (curSelItem.folder == aSrcFolder || select) { + feedWindow.selectFolder(aDestFolder, { open: true }); + } else { + feedWindow.selectFolder(curSelItem.folder, { + open: curSelItem.open, + }); + } + } else { + feedWindow.selectFeed( + { folder: curSelItem.parentFolder.rootFolder, url: curSelItem.url }, + null + ); + } + }, 50); + }, + }, + + /* *************************************************************** */ + /* OPML Functions */ + /* *************************************************************** */ + + get brandShortName() { + let brandBundle = document.getElementById("bundle_brand"); + return brandBundle ? brandBundle.getString("brandShortName") : ""; + }, + + /** + * Export feeds as opml file Save As filepicker function. + * + * @param {Boolean} aList - If true, exporting as list; if false (default) + * exporting feeds in folder structure - used for title. + * @returns {Promise} nsIFile or null. + */ + opmlPickSaveAsFile(aList) { + let accountName = this.mRSSServer.rootFolder.prettyName; + let fileName = FeedUtils.strings.formatStringFromName( + "subscribe-OPMLExportDefaultFileName", + [this.brandShortName, accountName] + ); + let title = aList + ? FeedUtils.strings.formatStringFromName( + "subscribe-OPMLExportTitleList", + [accountName] + ) + : FeedUtils.strings.formatStringFromName( + "subscribe-OPMLExportTitleStruct", + [accountName] + ); + let fp = Cc["@mozilla.org/filepicker;1"].createInstance(Ci.nsIFilePicker); + + fp.defaultString = fileName; + fp.defaultExtension = "opml"; + if ( + this.opmlLastSaveAsDir && + this.opmlLastSaveAsDir instanceof Ci.nsIFile + ) { + fp.displayDirectory = this.opmlLastSaveAsDir; + } + + let opmlFilterText = FeedUtils.strings.GetStringFromName( + "subscribe-OPMLExportOPMLFilesFilterText" + ); + fp.appendFilter(opmlFilterText, "*.opml"); + fp.appendFilters(Ci.nsIFilePicker.filterAll); + fp.filterIndex = 0; + fp.init(window, title, Ci.nsIFilePicker.modeSave); + + return new Promise(resolve => { + fp.open(rv => { + if ( + (rv != Ci.nsIFilePicker.returnOK && + rv != Ci.nsIFilePicker.returnReplace) || + !fp.file + ) { + resolve(null); + return; + } + + this.opmlLastSaveAsDir = fp.file.parent; + resolve(fp.file); + }); + }); + }, + + /** + * Import feeds opml file Open filepicker function. + * + * @returns {Promise} [{nsIFile} file, {String} fileUrl] or null. + */ + opmlPickOpenFile() { + let title = FeedUtils.strings.GetStringFromName( + "subscribe-OPMLImportTitle" + ); + let fp = Cc["@mozilla.org/filepicker;1"].createInstance(Ci.nsIFilePicker); + + fp.defaultString = ""; + if (this.opmlLastOpenDir && this.opmlLastOpenDir instanceof Ci.nsIFile) { + fp.displayDirectory = this.opmlLastOpenDir; + } + + let opmlFilterText = FeedUtils.strings.GetStringFromName( + "subscribe-OPMLExportOPMLFilesFilterText" + ); + fp.appendFilter(opmlFilterText, "*.opml"); + fp.appendFilters(Ci.nsIFilePicker.filterXML); + fp.appendFilters(Ci.nsIFilePicker.filterAll); + fp.init(window, title, Ci.nsIFilePicker.modeOpen); + + return new Promise(resolve => { + fp.open(rv => { + if (rv != Ci.nsIFilePicker.returnOK || !fp.file) { + resolve(null); + return; + } + + this.opmlLastOpenDir = fp.file.parent; + resolve([fp.file, fp.fileURL.spec]); + }); + }); + }, + + async exportOPML(aEvent) { + // Account folder must be selected. + let item = this.mView.currentItem; + if (!item || !item.folder || !item.folder.isServer) { + return; + } + + this.mRSSServer = item.folder.server; + let rootFolder = this.mRSSServer.rootFolder; + let exportAsList = aEvent.ctrlKey; + let SPACES2 = " "; + let SPACES4 = " "; + + if (this.mRSSServer.rootFolder.hasSubFolders) { + let opmlDoc = document.implementation.createDocument("", "opml", null); + let opmlRoot = opmlDoc.documentElement; + opmlRoot.setAttribute("version", "1.0"); + opmlRoot.setAttribute("xmlns:fz", "urn:forumzilla:"); + + this.generatePPSpace(opmlRoot, SPACES2); + + // Make the <head> element. + let head = opmlDoc.createElement("head"); + this.generatePPSpace(head, SPACES4); + let titleText = FeedUtils.strings.formatStringFromName( + "subscribe-OPMLExportFileDialogTitle", + [this.brandShortName, rootFolder.prettyName] + ); + let title = opmlDoc.createElement("title"); + title.appendChild(opmlDoc.createTextNode(titleText)); + head.appendChild(title); + this.generatePPSpace(head, SPACES4); + let dt = opmlDoc.createElement("dateCreated"); + dt.appendChild(opmlDoc.createTextNode(new Date().toUTCString())); + head.appendChild(dt); + this.generatePPSpace(head, SPACES2); + opmlRoot.appendChild(head); + + this.generatePPSpace(opmlRoot, SPACES2); + + // Add <outline>s to the <body>. + let body = opmlDoc.createElement("body"); + if (exportAsList) { + this.generateOutlineList(rootFolder, body, SPACES4.length + 2); + } else { + this.generateOutlineStruct(rootFolder, body, SPACES4.length); + } + + this.generatePPSpace(body, SPACES2); + + if (!body.childElementCount) { + // No folders/feeds. + return; + } + + opmlRoot.appendChild(body); + this.generatePPSpace(opmlRoot, ""); + + // Get file to save from filepicker. + let saveAsFile = await this.opmlPickSaveAsFile(exportAsList); + if (!saveAsFile) { + return; + } + + let fos = FileUtils.openSafeFileOutputStream(saveAsFile); + let serializer = new XMLSerializer(); + serializer.serializeToStream(opmlDoc, fos, "utf-8"); + FileUtils.closeSafeFileOutputStream(fos); + + let statusReport = FeedUtils.strings.formatStringFromName( + "subscribe-OPMLExportDone", + [saveAsFile.path] + ); + this.updateStatusItem("statusText", statusReport); + FeedUtils.log.info("exportOPML: " + statusReport); + } + }, + + generatePPSpace(aNode, indentString) { + aNode.appendChild(aNode.ownerDocument.createTextNode("\n")); + aNode.appendChild(aNode.ownerDocument.createTextNode(indentString)); + }, + + generateOutlineList(baseFolder, parent, indentLevel) { + // Pretty printing. + let indentString = " ".repeat(indentLevel - 2); + + let feedOutline; + for (let folder of baseFolder.subFolders) { + FeedUtils.log.debug( + "generateOutlineList: folder - " + folder.filePath.path + ); + if ( + !(folder instanceof Ci.nsIMsgFolder) || + folder.getFlag(Ci.nsMsgFolderFlags.Trash) || + folder.getFlag(Ci.nsMsgFolderFlags.Virtual) + ) { + continue; + } + + FeedUtils.log.debug( + "generateOutlineList: CONTINUE folderName - " + folder.name + ); + + if (folder.hasSubFolders) { + FeedUtils.log.debug( + "generateOutlineList: has subfolders - " + folder.name + ); + // Recurse. + this.generateOutlineList(folder, parent, indentLevel); + } + + // Add outline elements with xmlUrls. + let feeds = this.getFeedsInFolder(folder); + for (let feed of feeds) { + FeedUtils.log.debug( + "generateOutlineList: folder has FEED url - " + + folder.name + + " : " + + feed.url + ); + feedOutline = this.exportOPMLOutline(feed, parent.ownerDocument); + this.generatePPSpace(parent, indentString); + parent.appendChild(feedOutline); + } + } + }, + + generateOutlineStruct(baseFolder, parent, indentLevel) { + // Pretty printing. + function indentString(len) { + return " ".repeat(len - 2); + } + + let folderOutline, feedOutline; + for (let folder of baseFolder.subFolders) { + FeedUtils.log.debug( + "generateOutlineStruct: folder - " + folder.filePath.path + ); + if ( + !(folder instanceof Ci.nsIMsgFolder) || + folder.getFlag(Ci.nsMsgFolderFlags.Trash) || + folder.getFlag(Ci.nsMsgFolderFlags.Virtual) + ) { + continue; + } + + FeedUtils.log.debug( + "generateOutlineStruct: CONTINUE folderName - " + folder.name + ); + + // Make a folder outline element. + folderOutline = parent.ownerDocument.createElement("outline"); + folderOutline.setAttribute("title", folder.prettyName); + this.generatePPSpace(parent, indentString(indentLevel + 2)); + + if (folder.hasSubFolders) { + FeedUtils.log.debug( + "generateOutlineStruct: has subfolders - " + folder.name + ); + // Recurse. + this.generateOutlineStruct(folder, folderOutline, indentLevel + 2); + } + + let feeds = this.getFeedsInFolder(folder); + for (let feed of feeds) { + // Add feed outline elements with xmlUrls. + FeedUtils.log.debug( + "generateOutlineStruct: folder has FEED url - " + + folder.name + + " : " + + feed.url + ); + feedOutline = this.exportOPMLOutline(feed, parent.ownerDocument); + this.generatePPSpace(folderOutline, indentString(indentLevel + 4)); + folderOutline.appendChild(feedOutline); + } + + parent.appendChild(folderOutline); + } + }, + + exportOPMLOutline(aFeed, aDoc) { + let outRv = aDoc.createElement("outline"); + outRv.setAttribute("type", "rss"); + outRv.setAttribute("title", aFeed.title); + outRv.setAttribute("text", aFeed.title); + outRv.setAttribute("version", "RSS"); + outRv.setAttribute("fz:quickMode", aFeed.quickMode); + outRv.setAttribute("fz:options", JSON.stringify(aFeed.options)); + outRv.setAttribute("xmlUrl", aFeed.url); + outRv.setAttribute("htmlUrl", aFeed.link); + return outRv; + }, + + async importOPML() { + // Account folder must be selected in subscribe dialog. + let item = this.mView ? this.mView.currentItem : null; + if (!item || !item.folder || !item.folder.isServer) { + return; + } + + let server = item.folder.server; + // Get file and file url to open from filepicker. + let [openFile, openFileUrl] = await this.opmlPickOpenFile(); + + this.mActionMode = this.kImportingOPML; + this.updateButtons(null); + this.selectFolder(item.folder, { select: false, open: true }); + let statusReport = FeedUtils.strings.GetStringFromName("subscribe-loading"); + this.updateStatusItem("statusText", statusReport); + // If there were a getElementsByAttribute in html, we could go determined... + this.updateStatusItem("progressMeter", "?"); + + if ( + !(await this.importOPMLFile( + openFile, + openFileUrl, + server, + this.importOPMLFinished + )) + ) { + this.mActionMode = null; + this.updateButtons(item); + this.clearStatusInfo(); + } + }, + + /** + * Import opml file into a feed account. Used by the Subscribe dialog and + * the Import wizard. + * + * @param {nsIFile} aFile - The opml file. + * @param {string} aFileUrl - The opml file url. + * @param {nsIMsgIncomingServer} aServer - The account server. + * @param {Function} aCallback - Callback function. + * + * @returns {Boolean} - false if error. + */ + async importOPMLFile(aFile, aFileUrl, aServer, aCallback) { + if (aServer && aServer instanceof Ci.nsIMsgIncomingServer) { + this.mRSSServer = aServer; + } + + if (!aFile || !aFileUrl || !this.mRSSServer) { + return false; + } + + let opmlDom, statusReport; + FeedUtils.log.debug( + "importOPMLFile: fileName:fileUrl - " + aFile.leafName + ":" + aFileUrl + ); + let request = new Request(aFileUrl); + await fetch(request) + .then(function (response) { + if (!response.ok) { + // If the OPML file is not readable/accessible. + statusReport = FeedUtils.strings.GetStringFromName( + "subscribe-errorOpeningFile" + ); + return null; + } + + return response.text(); + }) + .then(function (responseText) { + if (responseText != null) { + opmlDom = new DOMParser().parseFromString( + responseText, + "application/xml" + ); + if ( + !XMLDocument.isInstance(opmlDom) || + opmlDom.documentElement.namespaceURI == + FeedUtils.MOZ_PARSERERROR_NS || + opmlDom.documentElement.tagName != "opml" || + !( + opmlDom.querySelector("body") && + opmlDom.querySelector("body").childElementCount + ) + ) { + // If the OPML file is invalid or empty. + statusReport = FeedUtils.strings.formatStringFromName( + "subscribe-OPMLImportInvalidFile", + [aFile.leafName] + ); + } + } + }) + .catch(function (error) { + statusReport = FeedUtils.strings.GetStringFromName( + "subscribe-errorOpeningFile" + ); + FeedUtils.log.error("importOPMLFile: error - " + error.message); + }); + + if (statusReport) { + FeedUtils.log.error("importOPMLFile: status - " + statusReport); + Services.prompt.alert(window, null, statusReport); + return false; + } + + let body = opmlDom.querySelector("body"); + this.importOPMLOutlines(body, this.mRSSServer, aCallback); + return true; + }, + + importOPMLOutlines(aBody, aRSSServer, aCallback) { + let win = this; + let rssServer = aRSSServer; + let callback = aCallback; + let outline, feedFolder; + let badTag = false; + let firstFeedInFolderQuickMode = null; + let lastFolder; + let feedsAdded = 0; + let rssOutlines = 0; + + function processor(aParentNode, aParentFolder) { + FeedUtils.log.trace( + "importOPMLOutlines: PROCESSOR tag:name:children - " + + aParentNode.tagName + + ":" + + aParentNode.getAttribute("text") + + ":" + + aParentNode.childElementCount + ); + while (true) { + if (aParentNode.tagName == "body" && !aParentNode.childElementCount) { + // Finished. + let statusReport = win.importOPMLStatus(feedsAdded, rssOutlines); + callback(statusReport, lastFolder, win); + return; + } + + outline = aParentNode.firstElementChild; + if (outline.tagName != "outline") { + FeedUtils.log.info( + "importOPMLOutlines: skipping, node is not an " + + "<outline> - <" + + outline.tagName + + ">" + ); + badTag = true; + break; + } + + let outlineName = + outline.getAttribute("text") || + outline.getAttribute("title") || + outline.getAttribute("xmlUrl"); + let feedUrl, folder; + + if (outline.getAttribute("type") == "rss") { + // A feed outline. + feedUrl = + outline.getAttribute("xmlUrl") || outline.getAttribute("url"); + if (!feedUrl) { + FeedUtils.log.info( + "importOPMLOutlines: skipping, type=rss <outline> " + + "has no url - " + + outlineName + ); + break; + } + + rssOutlines++; + feedFolder = aParentFolder; + + if (FeedUtils.feedAlreadyExists(feedUrl, rssServer)) { + FeedUtils.log.info( + "importOPMLOutlines: feed already subscribed in account " + + rssServer.prettyName + + ", url - " + + feedUrl + ); + break; + } + + if ( + aParentNode.tagName == "outline" && + aParentNode.getAttribute("type") != "rss" + ) { + // Parent is a folder, already created. + folder = feedFolder; + } else { + // Parent is not a folder outline, likely the <body> in a flat list. + // Create feed's folder with feed's name and account rootFolder as + // parent of feed's folder. + // NOTE: Assume a type=rss outline must be a leaf and is not a + // direct parent of another type=rss outline; such a structure + // may lead to unintended nesting and inaccurate counts. + folder = rssServer.rootFolder; + } + + // Create the feed. + let quickMode = outline.hasAttribute("fz:quickMode") + ? outline.getAttribute("fz:quickMode") == "true" + : rssServer.getBoolValue("quickMode"); + let options = outline.getAttribute("fz:options"); + options = options ? JSON.parse(options) : null; + + if (firstFeedInFolderQuickMode === null) { + // The summary/web page pref applies to all feeds in a folder, + // though it is a property of an individual feed. This can be + // set (and is obvious) in the subscribe dialog; ensure import + // doesn't leave mismatches if mismatched in the opml file. + firstFeedInFolderQuickMode = quickMode; + } else { + quickMode = firstFeedInFolderQuickMode; + } + + let feedProperties = { + feedName: outlineName, + feedLocation: feedUrl, + feedFolder: folder, + quickMode, + options, + }; + + FeedUtils.log.info( + "importOPMLOutlines: importing feed: name, url - " + + outlineName + + ", " + + feedUrl + ); + + let feed = win.storeFeed(feedProperties); + if (outline.hasAttribute("htmlUrl")) { + feed.link = outline.getAttribute("htmlUrl"); + } + + feed.createFolder(); + if (!feed.folder) { + // Non success. Remove intermediate traces from the feeds database. + if (feed && feed.url && feed.server) { + FeedUtils.deleteFeed(feed); + } + + FeedUtils.log.info( + "importOPMLOutlines: skipping, error creating folder - '" + + feed.folderName + + "' from outlineName - '" + + outlineName + + "' in parent folder " + + aParentFolder.filePath.path + ); + badTag = true; + break; + } + + // Add the feed to the databases. + FeedUtils.addFeed(feed); + // Feed correctly added. + feedsAdded++; + lastFolder = feed.folder; + } else { + // A folder outline. If a folder exists in the account structure at + // the same level as in the opml structure, feeds are placed into the + // existing folder. + let folderName = outlineName; + try { + feedFolder = aParentFolder.getChildNamed(folderName); + } catch (ex) { + // Folder not found, create it. + FeedUtils.log.info( + "importOPMLOutlines: creating folder - '" + + folderName + + "' from outlineName - '" + + outlineName + + "' in parent folder " + + aParentFolder.filePath.path + ); + firstFeedInFolderQuickMode = null; + try { + feedFolder = aParentFolder + .QueryInterface(Ci.nsIMsgLocalMailFolder) + .createLocalSubfolder(folderName); + } catch (ex) { + // An error creating. Skip it. + FeedUtils.log.info( + "importOPMLOutlines: skipping, error creating folder - '" + + folderName + + "' from outlineName - '" + + outlineName + + "' in parent folder " + + aParentFolder.filePath.path + ); + let xfolder = aParentFolder.getChildNamed(folderName); + aParentFolder.propagateDelete(xfolder, true); + badTag = true; + break; + } + } + } + + break; + } + + if (!outline.childElementCount || badTag) { + // Remove leaf nodes that are processed or bad tags from the opml dom, + // and go back to reparse. This method lets us use setTimeout to + // prevent UI hang, in situations of both deep and shallow trees. + // A yield/generator.next() method is fine for shallow trees, but not + // the true recursion required for deeper trees; both the shallow loop + // and the recurse should give it up. + outline.remove(); + badTag = false; + outline = aBody; + feedFolder = rssServer.rootFolder; + } + + setTimeout(() => { + processor(outline, feedFolder); + }, 0); + } + + processor(aBody, rssServer.rootFolder); + }, + + importOPMLStatus(aFeedsAdded, aRssOutlines, aFolderOutlines) { + let statusReport; + if (aRssOutlines > aFeedsAdded) { + statusReport = FeedUtils.strings.formatStringFromName( + "subscribe-OPMLImportStatus", + [ + PluralForm.get( + aFeedsAdded, + FeedUtils.strings.GetStringFromName( + "subscribe-OPMLImportUniqueFeeds" + ) + ).replace("#1", aFeedsAdded), + PluralForm.get( + aRssOutlines, + FeedUtils.strings.GetStringFromName( + "subscribe-OPMLImportFoundFeeds" + ) + ).replace("#1", aRssOutlines), + ], + 2 + ); + } else { + statusReport = PluralForm.get( + aFeedsAdded, + FeedUtils.strings.GetStringFromName("subscribe-OPMLImportFeedCount") + ).replace("#1", aFeedsAdded); + } + + return statusReport; + }, + + importOPMLFinished(aStatusReport, aLastFolder, aWin) { + if (aLastFolder) { + aWin.selectFolder(aLastFolder, { select: false, newFolder: aLastFolder }); + aWin.selectFolder(aLastFolder.parent); + } + aWin.mActionMode = null; + aWin.updateButtons(aWin.mView.currentItem); + aWin.clearStatusInfo(); + aWin.updateStatusItem("statusText", aStatusReport); + }, +}; diff --git a/comm/mailnews/extensions/newsblog/feed-subscriptions.xhtml b/comm/mailnews/extensions/newsblog/feed-subscriptions.xhtml new file mode 100644 index 0000000000..d2f8cba2bb --- /dev/null +++ b/comm/mailnews/extensions/newsblog/feed-subscriptions.xhtml @@ -0,0 +1,373 @@ +<?xml version="1.0"?> +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> + +<?xml-stylesheet href="chrome://messenger/skin/messenger.css" type="text/css"?> +<?xml-stylesheet href="chrome://messenger/skin/folderPane.css" type="text/css"?> +<?xml-stylesheet href="chrome://messenger/skin/icons.css" type="text/css"?> +<?xml-stylesheet href="chrome://messenger/skin/folderMenus.css" type="text/css"?> +<?xml-stylesheet href="chrome://messenger-newsblog/skin/feed-subscriptions.css" type="text/css"?> +<?xml-stylesheet href="chrome://messenger/skin/input-fields.css" type="text/css"?> +<?xml-stylesheet href="chrome://messenger/skin/colors.css" type="text/css"?> +<?xml-stylesheet href="chrome://messenger/skin/themeableDialog.css" type="text/css"?> +<?xml-stylesheet href="chrome://messenger/skin/contextMenu.css" type="text/css"?> + +<!DOCTYPE html [ <!ENTITY % feedDTD SYSTEM "chrome://messenger-newsblog/locale/feed-subscriptions.dtd"> +%feedDTD; +<!ENTITY % newsblogDTD SYSTEM "chrome://messenger-newsblog/locale/am-newsblog.dtd"> +%newsblogDTD; ]> + +<html + id="feedSubscriptions" + xmlns="http://www.w3.org/1999/xhtml" + xmlns:xul="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul" + xmlns:html="http://www.w3.org/1999/xhtml" + scrolling="false" + windowtype="Mail:News-BlogSubscriptions" + persist="width height screenX screenY sizemode" + lightweightthemes="true" +> + <head> + <title>&feedSubscriptions.label;</title> + <link rel="localization" href="security/certificates/certManager.ftl" /> + <script + defer="defer" + src="chrome://messenger/content/globalOverlay.js" + ></script> + <script + defer="defer" + src="chrome://global/content/editMenuOverlay.js" + ></script> + <script + defer="defer" + src="chrome://messenger/content/specialTabs.js" + ></script> + <script + defer="defer" + src="chrome://messenger/content/dialogShadowDom.js" + ></script> + <script + defer="defer" + src="chrome://messenger-newsblog/content/feed-subscriptions.js" + ></script> + <script> + window.addEventListener("load", event => { + FeedSubscriptions.onLoad(); + }); + window.addEventListener("keypress", event => { + FeedSubscriptions.onKeyPress(event); + }); + window.addEventListener("mousedown", event => { + FeedSubscriptions.onMouseDown(event); + }); + </script> + </head> + <html:body + xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul" + > + <dialog + id="subscriptionsDialog" + buttons="accept" + buttonlabelaccept="&button.close.label;" + > + <keyset id="extensionsKeys"> + <key + id="key_close" + key="&cmd.close.commandKey;" + modifiers="accel" + oncommand="window.close();" + /> + <key id="key_close2" keycode="VK_ESCAPE" oncommand="window.close();" /> + </keyset> + + <stringbundle + id="bundle_newsblog" + src="chrome://messenger-newsblog/locale/newsblog.properties" + /> + <stringbundle + id="bundle_brand" + src="chrome://branding/locale/brand.properties" + /> + + <vbox flex="1" id="contentPane"> + <hbox pack="end"> + <label + is="text-link" + id="learnMore" + crop="end" + value="&learnMore.label;" + href="https://support.mozilla.org/kb/how-subscribe-news-feeds-and-blogs" + /> + </hbox> + + <tree + id="rssSubscriptionsList" + treelines="true" + flex="1" + hidecolumnpicker="true" + onselect="FeedSubscriptions.onSelect();" + seltype="single" + > + <treecols> + <treecol id="folderNameCol" primary="true" hideheader="true" /> + </treecols> + <treechildren + id="subscriptionChildren" + ondragstart="FeedSubscriptions.onDragStart(event);" + ondragover="FeedSubscriptions.onDragOver(event);" + /> + </tree> + + <hbox id="rssFeedInfoBox"> + <vbox flex="1"> + <hbox flex="1"> + <vbox pack="end"> + <hbox flex="1" align="center"> + <label + id="nameLabel" + accesskey="&feedTitle.accesskey;" + control="nameValue" + value="&feedTitle.label;" + /> + </hbox> + <hbox flex="1" align="center"> + <label + id="locationLabel" + accesskey="&feedLocation.accesskey;" + control="locationValue" + value="&feedLocation.label;" + /> + </hbox> + <hbox flex="1" align="center"> + <label + id="feedFolderLabel" + value="&feedFolder.label;" + accesskey="&feedFolder.accesskey;" + control="selectFolder" + /> + </hbox> + </vbox> + <vbox flex="1"> + <html:input + id="nameValue" + type="text" + class="input-inline" + aria-labelledby="nameLabel" + onchange="FeedSubscriptions.setPrefs(this);" + /> + <hbox class="input-container"> + <html:input + id="locationValue" + type="url" + class="uri-element input-inline" + aria-labelledby="locationLabel" + placeholder="&feedLocation2.placeholder;" + onchange="FeedSubscriptions.setPrefs(this);" + onfocus="FeedSubscriptions.onFocusChange();" + onblur="FeedSubscriptions.onFocusChange();" + /> + <hbox align="center"> + <label + is="text-link" + id="locationValidate" + collapsed="true" + crop="end" + value="&locationValidate.label;" + onclick="FeedSubscriptions.checkValidation(event);" + /> + </hbox> + </hbox> + <hbox class="input-container"> + <menulist + id="selectFolder" + flex="1" + class="folderMenuItem" + hidden="true" + > + <menupopup + is="folder-menupopup" + id="selectFolderPopup" + class="menulist-menupopup" + mode="feeds" + showFileHereLabel="true" + showAccountsFileHere="true" + oncommand="FeedSubscriptions.setNewFolder(event);" + /> + </menulist> + <html:input + id="selectFolderValue" + class="input-inline" + readonly="readonly" + aria-labelledby="feedFolderLabel" + onkeypress="FeedSubscriptions.onClickSelectFolderValue(event);" + onclick="FeedSubscriptions.onClickSelectFolderValue(event);" + /> + </hbox> + </vbox> + </hbox> + + <hbox align="center"> + <checkbox + id="updateEnabled" + label="&biffStart.label;" + accesskey="&biffStart.accesskey;" + oncommand="FeedSubscriptions.setPrefs(this);" + /> + <html:input + id="updateValue" + type="number" + class="size3 input-inline" + min="1" + aria-labelledby="updateEnabled updateValue biffMinutes biffDays recommendedUnits recommendedUnitsVal" + oninput="FeedSubscriptions.setPrefs(this);" + onchange="FeedSubscriptions.setPrefs(this);" + /> + <radiogroup + id="biffUnits" + orient="horizontal" + oncommand="FeedSubscriptions.setPrefs(this);" + > + <radio + id="biffMinutes" + value="min" + label="&biffMinutes.label;" + accesskey="&biffMinutes.accesskey;" + /> + <radio + id="biffDays" + value="d" + label="&biffDays.label;" + accesskey="&biffDays.accesskey;" + /> + </radiogroup> + <hbox id="recommendedBox"> + <label + id="recommendedUnits" + value="&recommendedUnits.label;" + hidden="true" + control="updateMinutes" + /> + <label + id="recommendedUnitsVal" + value="" + hidden="true" + control="updateMinutes" + /> + </hbox> + </hbox> + <checkbox + id="quickMode" + accesskey="&quickMode.accesskey;" + label="&quickMode.label;" + oncommand="FeedSubscriptions.setSummary(this.checked);" + /> + <checkbox + id="autotagEnable" + accesskey="&autotagEnable.accesskey;" + label="&autotagEnable.label;" + oncommand="FeedSubscriptions.setPrefs(this);" + /> + <hbox class="input-container"> + <checkbox + id="autotagUsePrefix" + class="indent" + accesskey="&autotagUsePrefix.accesskey;" + label="&autotagUsePrefix.label;" + oncommand="FeedSubscriptions.setPrefs(this);" + /> + <html:input + id="autotagPrefix" + type="text" + class="input-inline" + placeholder="&autoTagPrefix.placeholder;" + onchange="FeedSubscriptions.setPrefs(this);" + /> + </hbox> + <separator class="thin" /> + </vbox> + </hbox> + + <hbox id="statusContainerBox" align="center"> + <vbox flex="1"> + <description id="statusText" /> + </vbox> + <spacer flex="1" /> + <label + id="validationText" + collapsed="true" + class="text-link" + crop="end" + value="&validateText.label;" + onclick="FeedSubscriptions.checkValidation(event);" + /> + <button + id="addCertException" + collapsed="true" + data-l10n-id="certmgr-add-exception" + oncommand="FeedSubscriptions.addCertExceptionDialog();" + /> + <html:progress + id="progressMeter" + hidden="hidden" + value="0" + max="100" + /> + </hbox> + + <hbox align="end"> + <hbox class="actionButtons" flex="1"> + <button + id="addFeed" + hidden="true" + disabled="true" + label="&button.addFeed.label;" + accesskey="&button.addFeed.accesskey;" + oncommand="FeedSubscriptions.addFeed();" + /> + + <button + id="updateFeed" + hidden="true" + disabled="true" + label="&button.verifyFeed.label;" + accesskey="&button.verifyFeed.accesskey;" + verifylabel="&button.verifyFeed.label;" + verifyaccesskey="&button.verifyFeed.accesskey;" + updatelabel="&button.updateFeed.label;" + updateaccesskey="&button.updateFeed.accesskey;" + oncommand="FeedSubscriptions.updateFeed();" + /> + + <button + id="removeFeed" + hidden="true" + label="&button.removeFeed.label;" + accesskey="&button.removeFeed.accesskey;" + oncommand="FeedSubscriptions.removeFeed(true);" + /> + + <spacer flex="1" /> + + <button + id="importOPML" + hidden="true" + label="&button.importOPML.label;" + accesskey="&button.importOPML.accesskey;" + oncommand="FeedSubscriptions.importOPML();" + /> + + <button + id="exportOPML" + hidden="true" + label="&button.exportOPML.label;" + accesskey="&button.exportOPML.accesskey;" + tooltiptext="&button.exportOPML.tooltip;" + oncommand="FeedSubscriptions.exportOPML(event);" + /> + </hbox> + </hbox> + </vbox> + </dialog> + </html:body> +</html> diff --git a/comm/mailnews/extensions/newsblog/feedAccountWizard.js b/comm/mailnews/extensions/newsblog/feedAccountWizard.js new file mode 100644 index 0000000000..686caff3a3 --- /dev/null +++ b/comm/mailnews/extensions/newsblog/feedAccountWizard.js @@ -0,0 +1,56 @@ +/* -*- Mode: JavaScript; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +var { FeedUtils } = ChromeUtils.import("resource:///modules/FeedUtils.jsm"); + +window.addEventListener("DOMContentLoaded", () => { + FeedAccountWizard.onLoad(); +}); + +/** Feed account standalone wizard functions. */ +var FeedAccountWizard = { + accountName: "", + + onLoad() { + document + .querySelector("wizard") + .addEventListener("wizardfinish", this.onFinish.bind(this)); + let accountSetupPage = document.getElementById("accountsetuppage"); + accountSetupPage.addEventListener( + "pageshow", + this.accountSetupPageValidate.bind(this) + ); + accountSetupPage.addEventListener( + "pagehide", + this.accountSetupPageValidate.bind(this) + ); + let donePage = document.getElementById("done"); + donePage.addEventListener("pageshow", this.donePageInit.bind(this)); + }, + + accountSetupPageValidate() { + this.accountName = document.getElementById("prettyName").value.trim(); + document.querySelector("wizard").canAdvance = this.accountName; + }, + + donePageInit() { + document.getElementById("account.name.text").value = this.accountName; + }, + + onFinish() { + let account = FeedUtils.createRssAccount(this.accountName); + let openerWindow = window.opener.top; + // The following block is the same as in AccountWizard.js. + if ("selectServer" in openerWindow) { + // Opened from Account Settings. + openerWindow.selectServer(account.incomingServer); + } + + // Post a message to the main window on successful account setup. + openerWindow.postMessage("account-created", "*"); + + window.close(); + }, +}; diff --git a/comm/mailnews/extensions/newsblog/feedAccountWizard.xhtml b/comm/mailnews/extensions/newsblog/feedAccountWizard.xhtml new file mode 100644 index 0000000000..ef1be03dd7 --- /dev/null +++ b/comm/mailnews/extensions/newsblog/feedAccountWizard.xhtml @@ -0,0 +1,95 @@ +<?xml version="1.0"?> +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> + +<?xml-stylesheet href="chrome://messenger/skin/accountWizard.css" type="text/css"?> +<?xml-stylesheet type="text/css" href="chrome://messenger/skin/input-fields.css"?> +<?xml-stylesheet href="chrome://messenger/skin/colors.css" type="text/css"?> +<?xml-stylesheet href="chrome://messenger/skin/themeableDialog.css" type="text/css"?> + +<!DOCTYPE html [ <!ENTITY % accountDTD SYSTEM "chrome://messenger/locale/AccountWizard.dtd"> +%accountDTD; +<!ENTITY % newsblogDTD SYSTEM "chrome://messenger-newsblog/locale/am-newsblog.dtd" > +%newsblogDTD; +<!ENTITY % imDTD SYSTEM "chrome://messenger/locale/imAccountWizard.dtd" > +%imDTD; ]> + +<html + id="FeedAccountWizard" + xmlns="http://www.w3.org/1999/xhtml" + xmlns:xul="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul" + xmlns:html="http://www.w3.org/1999/xhtml" + lightweightthemes="true" + scrolling="false" +> + <head> + <title>&feedWindowTitle.label;</title> + <link rel="localization" href="toolkit/global/wizard.ftl" /> + <script + defer="defer" + src="chrome://messenger/content/globalOverlay.js" + ></script> + <script + defer="defer" + src="chrome://global/content/editMenuOverlay.js" + ></script> + <script + defer="defer" + src="chrome://messenger-newsblog/content/feedAccountWizard.js" + ></script> + </head> + <html:body + xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul" + > + <wizard> + <!-- Account setup page : User gets a choice to enter a name for the account --> + <!-- Defaults : Feed account name -> default string --> + <wizardpage + id="accountsetuppage" + pageid="accountsetuppage" + label="&accnameTitle.label;" + > + <vbox flex="1"> + <description>&accnameDesc.label;</description> + <separator class="thin" /> + <hbox align="center" class="input-container"> + <label + id="prettyNameLabel" + class="label" + value="&accnameLabel.label;" + accesskey="&accnameLabel.accesskey;" + control="prettyName" + /> + <html:input + id="prettyName" + type="text" + class="input-inline" + value="&feeds.accountName;" + aria-labelledby="prettyNameLabel" + oninput="FeedAccountWizard.accountSetupPageValidate();" + /> + </hbox> + </vbox> + </wizardpage> + + <!-- Done page : Summarizes information collected to create a feed account --> + <wizardpage id="done" pageid="done" label="&accountSummaryTitle.label;"> + <vbox flex="1"> + <description>&accountSummaryInfo.label;</description> + <separator class="thin" /> + <hbox id="account.name" align="center"> + <label + id="account.name.label" + class="label" + value="&accnameLabel.label;" + /> + <label id="account.name.text" class="label" /> + </hbox> + <separator /> + <spacer flex="1" /> + </vbox> + </wizardpage> + </wizard> + </html:body> +</html> diff --git a/comm/mailnews/extensions/newsblog/jar.mn b/comm/mailnews/extensions/newsblog/jar.mn new file mode 100644 index 0000000000..46d1a700cd --- /dev/null +++ b/comm/mailnews/extensions/newsblog/jar.mn @@ -0,0 +1,13 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +newsblog.jar: +% content messenger-newsblog %content/messenger-newsblog/ + content/messenger-newsblog/newsblogOverlay.js (newsblogOverlay.js) + content/messenger-newsblog/feed-subscriptions.js (feed-subscriptions.js) + content/messenger-newsblog/feed-subscriptions.xhtml (feed-subscriptions.xhtml) + content/messenger-newsblog/am-newsblog.js (am-newsblog.js) + content/messenger-newsblog/am-newsblog.xhtml (am-newsblog.xhtml) + content/messenger-newsblog/feedAccountWizard.js (feedAccountWizard.js) + content/messenger-newsblog/feedAccountWizard.xhtml (feedAccountWizard.xhtml) diff --git a/comm/mailnews/extensions/newsblog/moz.build b/comm/mailnews/extensions/newsblog/moz.build new file mode 100644 index 0000000000..f4a9d0dd9f --- /dev/null +++ b/comm/mailnews/extensions/newsblog/moz.build @@ -0,0 +1,25 @@ +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +EXTRA_JS_MODULES += [ + "Feed.jsm", + "FeedItem.jsm", + "FeedParser.jsm", + "FeedUtils.jsm", + "NewsBlog.jsm", +] + +XPCOM_MANIFESTS += [ + "components.conf", +] + +JAR_MANIFESTS += ["jar.mn"] + +BROWSER_CHROME_MANIFESTS += [ + "test/browser/browser.ini", +] +XPCSHELL_TESTS_MANIFESTS += [ + "test/unit/xpcshell.ini", +] diff --git a/comm/mailnews/extensions/newsblog/newsblogOverlay.js b/comm/mailnews/extensions/newsblog/newsblogOverlay.js new file mode 100644 index 0000000000..9145ad0dab --- /dev/null +++ b/comm/mailnews/extensions/newsblog/newsblogOverlay.js @@ -0,0 +1,416 @@ +/* -*- Mode: JavaScript; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* globals ReloadMessage, getMessagePaneBrowser, openContentTab, + GetNumSelectedMessages, gMessageNotificationBar */ + +var { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); +var { MailServices } = ChromeUtils.import( + "resource:///modules/MailServices.jsm" +); +var { FeedUtils } = ChromeUtils.import("resource:///modules/FeedUtils.jsm"); +var { MailE10SUtils } = ChromeUtils.import( + "resource:///modules/MailE10SUtils.jsm" +); + +XPCOMUtils.defineLazyModuleGetters(this, { + MsgHdrToMimeMessage: "resource:///modules/gloda/MimeMessage.jsm", +}); + +// This global is for SeaMonkey compatibility. +var gShowFeedSummary; + +var FeedMessageHandler = { + gShowSummary: true, + gToggle: false, + kSelectOverrideWebPage: 0, + kSelectOverrideSummary: 1, + kSelectFeedDefault: 2, + kOpenWebPage: 0, + kOpenSummary: 1, + kOpenToggleInMessagePane: 2, + kOpenLoadInBrowser: 3, + + FeedAccountTypes: ["rss"], + + /** + * How to load message on threadpane select. + */ + get onSelectPref() { + return Services.prefs.getIntPref("rss.show.summary"); + }, + + set onSelectPref(val) { + Services.prefs.setIntPref("rss.show.summary", val); + ReloadMessage(); + }, + + /** + * Load web page on threadpane select. + */ + get loadWebPageOnSelectPref() { + return Services.prefs.getIntPref("rss.message.loadWebPageOnSelect"); + }, + + /** + * How to load message on open (enter/dbl click in threadpane, contextmenu). + */ + get onOpenPref() { + return Services.prefs.getIntPref("rss.show.content-base"); + }, + + set onOpenPref(val) { + Services.prefs.setIntPref("rss.show.content-base", val); + }, + + /** + * Determine whether to show a feed message summary or load a web page in the + * message pane. + * + * @param {nsIMsgDBHdr} aMsgHdr - The message. + * @param {boolean} aToggle - true if in toggle mode, false otherwise. + * + * @returns {Boolean} - true if summary is to be displayed, false if web page. + */ + shouldShowSummary(aMsgHdr, aToggle) { + // Not a feed message, always show summary (the message). + if (!FeedUtils.isFeedMessage(aMsgHdr)) { + return true; + } + + // Notified of a summary reload when toggling, reset toggle and return. + if (!aToggle && this.gToggle) { + return !(this.gToggle = false); + } + + let showSummary = true; + this.gToggle = aToggle; + + // Thunderbird 2 rss messages with 'Show article summary' not selected, + // ie message body constructed to show web page in an iframe, can't show + // a summary - notify user. + let browser = getMessagePaneBrowser(); + let contentDoc = browser ? browser.contentDocument : null; + let rssIframe = contentDoc + ? contentDoc.getElementById("_mailrssiframe") + : null; + if (rssIframe) { + if (this.gToggle || this.onSelectPref == this.kSelectOverrideSummary) { + this.gToggle = false; + } + + return false; + } + + if (aToggle) { + // Toggle mode, flip value. + return (gShowFeedSummary = this.gShowSummary = !this.gShowSummary); + } + + let wintype = document.documentElement.getAttribute("windowtype"); + let tabMail = document.getElementById("tabmail"); + let messageTab = tabMail && tabMail.currentTabInfo.mode.type == "message"; + let messageWindow = wintype == "mail:messageWindow"; + + switch (this.onSelectPref) { + case this.kSelectOverrideWebPage: + showSummary = false; + break; + case this.kSelectOverrideSummary: + showSummary = true; + break; + case this.kSelectFeedDefault: + // Get quickmode per feed folder pref from feed subscriptions. If the feed + // message is not in a feed account folder (hence the folder is not in + // the feeds database), err on the side of showing the summary. + // For the former, toggle or global override is necessary; for the + // latter, a show summary checkbox toggle in Subscribe dialog will set + // one on the path to bliss. + let folder = aMsgHdr.folder; + showSummary = true; + const ds = FeedUtils.getSubscriptionsDS(folder.server); + for (let sub of ds.data) { + if (sub.destFolder == folder.URI) { + showSummary = sub.quickMode; + break; + } + } + break; + } + + gShowFeedSummary = this.gShowSummary = showSummary; + + if (messageWindow || messageTab) { + // Message opened in either standalone window or tab, due to either + // message open pref (we are here only if the pref is 0 or 1) or + // contextmenu open. + switch (this.onOpenPref) { + case this.kOpenToggleInMessagePane: + // Opened by contextmenu, use the value derived above. + // XXX: allow a toggle via crtl? + break; + case this.kOpenWebPage: + showSummary = false; + break; + case this.kOpenSummary: + showSummary = true; + break; + } + } + + // Auto load web page in browser on select, per pref; shouldShowSummary() is + // always called first to 1)test if feed, 2)get summary pref, so do it here. + if (this.loadWebPageOnSelectPref) { + setTimeout(FeedMessageHandler.loadWebPage, 20, aMsgHdr, { + browser: true, + }); + } + + return showSummary; + }, + + /** + * Load a web page for feed messages. Use MsgHdrToMimeMessage() to get + * the content-base url from the message headers. We cannot rely on + * currentHeaderData; it has not yet been streamed at our entry point in + * displayMessageChanged(), and in the case of a collapsed message pane it + * is not streamed. + * + * @param {nsIMsgDBHdr} aMessageHdr - The message. + * @param {Object} aWhere - name value=true pair, where name is in: + * 'messagepane', 'browser', 'tab', 'window'. + * @returns {void} + */ + loadWebPage(aMessageHdr, aWhere) { + MsgHdrToMimeMessage(aMessageHdr, null, function (aMsgHdr, aMimeMsg) { + if ( + aMimeMsg && + aMimeMsg.headers["content-base"] && + aMimeMsg.headers["content-base"][0] + ) { + let url = aMimeMsg.headers["content-base"], + uri; + try { + // The message and headers are stored as a string of UTF-8 bytes + // and we need to convert that cpp |string| to js UTF-16 explicitly + // for idn and non-ascii urls with this api. + url = decodeURIComponent(escape(url)); + uri = Services.io.newURI(url); + } catch (ex) { + FeedUtils.log.info( + "FeedMessageHandler.loadWebPage: " + + "invalid Content-Base header url - " + + url + ); + return; + } + if (aWhere.browser) { + Cc["@mozilla.org/uriloader/external-protocol-service;1"] + .getService(Ci.nsIExternalProtocolService) + .loadURI(uri); + } else if (aWhere.messagepane) { + let browser = getMessagePaneBrowser(); + // Load about:blank in the browser before (potentially) switching + // to a remote process. This prevents sandbox flags being carried + // over to the web document. + MailE10SUtils.loadAboutBlank(browser); + MailE10SUtils.loadURI(browser, url); + } else if (aWhere.tab) { + openContentTab(url, "tab", null); + } else if (aWhere.window) { + openContentTab(url, "window", null); + } + } else { + FeedUtils.log.info( + "FeedMessageHandler.loadWebPage: could not get " + + "Content-Base header url for this message" + ); + } + }); + }, + + /** + * Display summary or load web page for feed messages. Caller should already + * know if the message is a feed message. + * + * @param {nsIMsgDBHdr} aMsgHdr - The message. + * @param {Boolean} aShowSummary - true if summary is to be displayed, + * false if web page. + * @returns {void} + */ + setContent(aMsgHdr, aShowSummary) { + if (aShowSummary) { + // Only here if toggling to summary in 3pane. + if (this.gToggle && window.gDBView && GetNumSelectedMessages() == 1) { + ReloadMessage(); + } + } else { + let browser = getMessagePaneBrowser(); + if (browser && browser.contentDocument && browser.contentDocument.body) { + browser.contentDocument.body.hidden = true; + } + // If in a non rss folder, hide possible remote content bar on a web + // page load, as it doesn't apply. + gMessageNotificationBar.clearMsgNotifications(); + + this.loadWebPage(aMsgHdr, { messagepane: true }); + this.gToggle = false; + } + }, +}; + +function openSubscriptionsDialog(aFolder) { + // Check for an existing feed subscriptions window and focus it. + let subscriptionsWindow = Services.wm.getMostRecentWindow( + "Mail:News-BlogSubscriptions" + ); + + if (subscriptionsWindow) { + if (aFolder) { + subscriptionsWindow.FeedSubscriptions.selectFolder(aFolder); + subscriptionsWindow.FeedSubscriptions.mView.tree.ensureRowIsVisible( + subscriptionsWindow.FeedSubscriptions.mView.selection.currentIndex + ); + } + + subscriptionsWindow.focus(); + } else { + window.browsingContext.topChromeWindow.openDialog( + "chrome://messenger-newsblog/content/feed-subscriptions.xhtml", + "", + "centerscreen,chrome,dialog=no,resizable", + { folder: aFolder } + ); + } +} + +// Special case attempts to reply/forward/edit as new RSS articles. For +// messages stored prior to Tb15, we are here only if the message's folder's +// account server is rss and feed messages moved to other types will have their +// summaries loaded, as viewing web pages only happened in an rss account. +// The user may choose whether to load a summary or web page link by ensuring +// the current feed message is being viewed as either a summary or web page. +function openComposeWindowForRSSArticle( + aMsgComposeWindow, + aMsgHdr, + aMessageUri, + aType, + aFormat, + aIdentity, + aMsgWindow +) { + // Ensure right content is handled for web pages in window/tab. + let tabmail = document.getElementById("tabmail"); + let is3pane = + tabmail && tabmail.selectedTab && tabmail.selectedTab.mode + ? tabmail.selectedTab.mode.type == "folder" + : false; + let showingwebpage = + "FeedMessageHandler" in window && + !is3pane && + FeedMessageHandler.onOpenPref == FeedMessageHandler.kOpenWebPage; + + if (gShowFeedSummary && !showingwebpage) { + // The user is viewing the summary. + MailServices.compose.OpenComposeWindow( + aMsgComposeWindow, + aMsgHdr, + aMessageUri, + aType, + aFormat, + aIdentity, + null, + aMsgWindow + ); + } else { + // Set up the compose message and get the feed message's web page link. + let msgHdr = aMsgHdr; + let type = aType; + let msgComposeType = Ci.nsIMsgCompType; + let subject = msgHdr.mime2DecodedSubject; + let fwdPrefix = Services.prefs.getCharPref("mail.forward_subject_prefix"); + fwdPrefix = fwdPrefix ? fwdPrefix + ": " : ""; + + let params = Cc[ + "@mozilla.org/messengercompose/composeparams;1" + ].createInstance(Ci.nsIMsgComposeParams); + + let composeFields = Cc[ + "@mozilla.org/messengercompose/composefields;1" + ].createInstance(Ci.nsIMsgCompFields); + + if ( + type == msgComposeType.Reply || + type == msgComposeType.ReplyAll || + type == msgComposeType.ReplyToSender || + type == msgComposeType.ReplyToGroup || + type == msgComposeType.ReplyToSenderAndGroup || + type == msgComposeType.ReplyToList + ) { + subject = "Re: " + subject; + } else if ( + type == msgComposeType.ForwardInline || + type == msgComposeType.ForwardAsAttachment + ) { + subject = fwdPrefix + subject; + } + + params.composeFields = composeFields; + params.composeFields.subject = subject; + params.composeFields.body = ""; + params.bodyIsLink = false; + params.identity = aIdentity; + + try { + // The feed's web page url is stored in the Content-Base header. + MsgHdrToMimeMessage( + msgHdr, + null, + function (aMsgHdr, aMimeMsg) { + if ( + aMimeMsg && + aMimeMsg.headers["content-base"] && + aMimeMsg.headers["content-base"][0] + ) { + let url = decodeURIComponent( + escape(aMimeMsg.headers["content-base"]) + ); + params.composeFields.body = url; + params.bodyIsLink = true; + MailServices.compose.OpenComposeWindowWithParams(null, params); + } else { + // No content-base url, use the summary. + MailServices.compose.OpenComposeWindow( + aMsgComposeWindow, + aMsgHdr, + aMessageUri, + aType, + aFormat, + aIdentity, + null, + aMsgWindow + ); + } + }, + false, + { saneBodySize: true } + ); + } catch (ex) { + // Error getting header, use the summary. + MailServices.compose.OpenComposeWindow( + aMsgComposeWindow, + aMsgHdr, + aMessageUri, + aType, + aFormat, + aIdentity, + null, + aMsgWindow + ); + } + } +} diff --git a/comm/mailnews/extensions/newsblog/test/browser/browser.ini b/comm/mailnews/extensions/newsblog/test/browser/browser.ini new file mode 100644 index 0000000000..1abed389c7 --- /dev/null +++ b/comm/mailnews/extensions/newsblog/test/browser/browser.ini @@ -0,0 +1,20 @@ +[DEFAULT] +prefs = + datareporting.policy.dataSubmissionPolicyBypassNotification=true + mail.account.account1.server=server1 + mail.accountmanager.accounts=account1 + mail.provider.suppress_dialog_on_startup=true + mail.server.server1.hostname=Feeds + mail.server.server1.name=Feeds + mail.server.server1.type=rss + mail.server.server1.userName=nobody + mail.spellcheck.inline=false + mail.spotlight.firstRunDone=true + mail.winsearch.firstRunDone=true + mailnews.database.global.indexer.enabled=false + mailnews.start_page.override_url=about:blank + mailnews.start_page.url=about:blank +subsuite = thunderbird +support-files = data/** + +[browser_feedDisplay.js] diff --git a/comm/mailnews/extensions/newsblog/test/browser/browser_feedDisplay.js b/comm/mailnews/extensions/newsblog/test/browser/browser_feedDisplay.js new file mode 100644 index 0000000000..98fa755c46 --- /dev/null +++ b/comm/mailnews/extensions/newsblog/test/browser/browser_feedDisplay.js @@ -0,0 +1,228 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, you can obtain one at http://mozilla.org/MPL/2.0/. */ + +var { MailServices } = ChromeUtils.import( + "resource:///modules/MailServices.jsm" +); +var { mailTestUtils } = ChromeUtils.import( + "resource://testing-common/mailnews/MailTestUtils.jsm" +); +var { MockRegistrar } = ChromeUtils.importESModule( + "resource://testing-common/MockRegistrar.sys.mjs" +); + +add_task(async () => { + function folderTreeClick(row, event = {}) { + EventUtils.synthesizeMouseAtCenter( + folderTree.rows[row].querySelector(".name"), + event, + about3Pane + ); + } + function threadTreeClick(row, event = {}) { + EventUtils.synthesizeMouseAtCenter( + threadTree.getRowAtIndex(row), + event, + about3Pane + ); + } + + /** @implements {nsIExternalProtocolService} */ + let mockExternalProtocolService = { + QueryInterface: ChromeUtils.generateQI(["nsIExternalProtocolService"]), + _loadedURLs: [], + loadURI(uri, windowContext) { + this._loadedURLs.push(uri.spec); + }, + isExposedProtocol(scheme) { + return true; + }, + urlLoaded(url) { + return this._loadedURLs.includes(url); + }, + }; + + let mockExternalProtocolServiceCID = MockRegistrar.register( + "@mozilla.org/uriloader/external-protocol-service;1", + mockExternalProtocolService + ); + + registerCleanupFunction(() => { + MockRegistrar.unregister(mockExternalProtocolServiceCID); + + // Some tests that open new windows don't return focus to the main window + // in a way that satisfies mochitest, and the test times out. + Services.focus.focusedWindow = about3Pane; + }); + + let tabmail = document.getElementById("tabmail"); + let about3Pane = tabmail.currentAbout3Pane; + let { folderTree, threadTree, messageBrowser } = about3Pane; + let menu = about3Pane.document.getElementById("folderPaneContext"); + let menuItem = about3Pane.document.getElementById( + "folderPaneContext-subscribe" + ); + // Not `currentAboutMessage` as that's null right now. + let aboutMessage = messageBrowser.contentWindow; + let messagePane = aboutMessage.getMessagePaneBrowser(); + + let account = MailServices.accounts.getAccount("account1"); + let rootFolder = account.incomingServer.rootFolder; + about3Pane.displayFolder(rootFolder.URI); + let index = about3Pane.folderTree.selectedIndex; + Assert.equal(index, 0); + + let shownPromise = BrowserTestUtils.waitForEvent(menu, "popupshown"); + folderTreeClick(index, { type: "contextmenu" }); + await shownPromise; + + let hiddenPromise = BrowserTestUtils.waitForEvent(menu, "popuphidden"); + let dialogPromise = BrowserTestUtils.promiseAlertDialog( + null, + "chrome://messenger-newsblog/content/feed-subscriptions.xhtml", + { + async callback(dialogWindow) { + let dialogDocument = dialogWindow.document; + + let list = dialogDocument.getElementById("rssSubscriptionsList"); + let locationInput = dialogDocument.getElementById("locationValue"); + let addFeedButton = dialogDocument.getElementById("addFeed"); + + await BrowserTestUtils.waitForEvent(list, "select"); + + EventUtils.synthesizeMouseAtCenter(locationInput, {}, dialogWindow); + await TestUtils.waitForCondition(() => !addFeedButton.disabled); + EventUtils.sendString( + "https://example.org/browser/comm/mailnews/extensions/newsblog/test/browser/data/rss.xml", + dialogWindow + ); + EventUtils.synthesizeKey("VK_TAB", {}, dialogWindow); + + // There's no good way to know if we're ready to continue. + await new Promise(r => dialogWindow.setTimeout(r, 250)); + + let hiddenPromise = BrowserTestUtils.waitForAttribute( + "hidden", + addFeedButton, + "true" + ); + EventUtils.synthesizeMouseAtCenter(addFeedButton, {}, dialogWindow); + await hiddenPromise; + + EventUtils.synthesizeMouseAtCenter( + dialogDocument.querySelector("dialog").getButton("accept"), + {}, + dialogWindow + ); + }, + } + ); + menu.activateItem(menuItem); + await Promise.all([hiddenPromise, dialogPromise]); + + let folder = rootFolder.subFolders.find(f => f.name == "Test Feed"); + Assert.ok(folder); + + about3Pane.displayFolder(folder.URI); + index = folderTree.selectedIndex; + Assert.equal(about3Pane.threadTree.view.rowCount, 1); + + // Description mode. + + let loadedPromise = BrowserTestUtils.browserLoaded(messagePane); + threadTreeClick(0); + await loadedPromise; + + Assert.notEqual(messagePane.currentURI.spec, "about:blank"); + await SpecialPowers.spawn(messagePane, [], () => { + let doc = content.document; + + let p = doc.querySelector("p"); + Assert.equal(p.textContent, "This is the description."); + + let style = content.getComputedStyle(doc.body); + Assert.equal(style.backgroundColor, "rgba(0, 0, 0, 0)"); + + let noscript = doc.querySelector("noscript"); + style = content.getComputedStyle(noscript); + Assert.equal(style.display, "inline"); + }); + + Assert.ok( + aboutMessage.document.getElementById("expandedtoRow").hidden, + "The To field is not visible" + ); + Assert.equal( + aboutMessage.document.getElementById("dateLabel").textContent, + aboutMessage.document.getElementById("dateLabelSubject").textContent, + "The regular date label and the subject date have the same value" + ); + Assert.ok( + BrowserTestUtils.is_hidden( + aboutMessage.document.getElementById("dateLabel"), + "The regular date label is not visible" + ) + ); + Assert.ok( + BrowserTestUtils.is_visible( + aboutMessage.document.getElementById("dateLabelSubject") + ), + "The date label on the subject line is visible" + ); + + await BrowserTestUtils.synthesizeMouseAtCenter("a", {}, messagePane); + Assert.deepEqual(mockExternalProtocolService._loadedURLs, [ + "https://example.org/link/from/description", + ]); + mockExternalProtocolService._loadedURLs.length = 0; + + // Web mode. + + loadedPromise = BrowserTestUtils.browserLoaded( + messagePane, + false, + "https://example.org/browser/comm/mailnews/extensions/newsblog/test/browser/data/article.html" + ); + window.FeedMessageHandler.onSelectPref = 0; + await loadedPromise; + + await SpecialPowers.spawn(messagePane, [], () => { + let doc = content.document; + + let p = doc.querySelector("p"); + Assert.equal(p.textContent, "This is the article."); + + let style = content.getComputedStyle(doc.body); + Assert.equal(style.backgroundColor, "rgb(0, 128, 0)"); + + let noscript = doc.querySelector("noscript"); + style = content.getComputedStyle(noscript); + Assert.equal(style.display, "none"); + }); + await BrowserTestUtils.synthesizeMouseAtCenter("a", {}, messagePane); + Assert.deepEqual(mockExternalProtocolService._loadedURLs, [ + "https://example.org/link/from/article", + ]); + mockExternalProtocolService._loadedURLs.length = 0; + + // Clean up. + + shownPromise = BrowserTestUtils.waitForEvent(menu, "popupshown"); + EventUtils.synthesizeMouseAtCenter( + about3Pane.folderTree.selectedRow, + { type: "contextmenu" }, + about3Pane + ); + await shownPromise; + + hiddenPromise = BrowserTestUtils.waitForEvent(menu, "popuphidden"); + let promptPromise = BrowserTestUtils.promiseAlertDialog("accept"); + menuItem = about3Pane.document.getElementById("folderPaneContext-remove"); + menu.activateItem(menuItem); + await Promise.all([hiddenPromise, promptPromise]); + + window.FeedMessageHandler.onSelectPref = 1; + + folderTree.selectedIndex = 0; +}); diff --git a/comm/mailnews/extensions/newsblog/test/browser/data/article.html b/comm/mailnews/extensions/newsblog/test/browser/data/article.html new file mode 100644 index 0000000000..4c6b780c41 --- /dev/null +++ b/comm/mailnews/extensions/newsblog/test/browser/data/article.html @@ -0,0 +1,17 @@ +<!DOCTYPE html> +<html> + <head> + <meta charset="utf-8"/> + <title></title> + </head> + <body> + <p>This is the article.</p> + <p><a href="https://example.org/link/from/article">Here's a link.</a></p> + <script> + document.body.style.backgroundColor = "green"; + </script> + <noscript> + <p>This noscript should not display.</p> + </noscript> + </body> +</html> diff --git a/comm/mailnews/extensions/newsblog/test/browser/data/rss.xml b/comm/mailnews/extensions/newsblog/test/browser/data/rss.xml new file mode 100644 index 0000000000..ec6aebccc1 --- /dev/null +++ b/comm/mailnews/extensions/newsblog/test/browser/data/rss.xml @@ -0,0 +1,27 @@ +<?xml version="1.0" encoding="UTF-8"?> +<rss version="2.0"> + <channel> + <title>Test Feed</title> + <link>https://example.org/</link> + <description></description> + <lastBuildDate>Thu, 21 Jan 2021 17:57:54 +0000</lastBuildDate> + <language>en-US</language> + + <item> + <title>Test Article</title> + <link>https://example.org/browser/comm/mailnews/extensions/newsblog/test/browser/data/article.html</link> + <pubDate>Wed, 20 Jan 2021 17:00:39 +0000</pubDate> + + <description><![CDATA[ + <p>This is the description.</p> + <p><a href="https://example.org/link/from/description">Here's a link.</a></p> + <script> + document.body.style.backgroundColor = "red"; + </script> + <noscript> + <p>This noscript should display.</p> + </noscript> + ]]></description> + </item> + </channel> +</rss> diff --git a/comm/mailnews/extensions/newsblog/test/unit/head_feeds.js b/comm/mailnews/extensions/newsblog/test/unit/head_feeds.js new file mode 100644 index 0000000000..004e1acf68 --- /dev/null +++ b/comm/mailnews/extensions/newsblog/test/unit/head_feeds.js @@ -0,0 +1,35 @@ +var { MailServices } = ChromeUtils.import( + "resource:///modules/MailServices.jsm" +); +var { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); +var { mailTestUtils } = ChromeUtils.import( + "resource://testing-common/mailnews/MailTestUtils.jsm" +); +var { localAccountUtils } = ChromeUtils.import( + "resource://testing-common/mailnews/LocalAccountUtils.jsm" +); + +let { FeedParser } = ChromeUtils.import("resource:///modules/FeedParser.jsm"); +let { Feed } = ChromeUtils.import("resource:///modules/Feed.jsm"); +let { FeedUtils } = ChromeUtils.import("resource:///modules/FeedUtils.jsm"); +let { HttpServer } = ChromeUtils.import("resource://testing-common/httpd.js"); + +// Set up local web server to serve up test files. +// We run it on a random port so that other tests can run concurrently +// even if they also run a web server. +let httpServer = new HttpServer(); +httpServer.registerDirectory("/", do_get_file("resources")); +httpServer.start(-1); +const SERVER_PORT = httpServer.identity.primaryPort; + +// Ensure the profile directory is set up +do_get_profile(); + +var gDEPTH = "../../../../../"; + +registerCleanupFunction(async () => { + await httpServer.stop(); + load(gDEPTH + "mailnews/resources/mailShutdown.js"); +}); diff --git a/comm/mailnews/extensions/newsblog/test/unit/resources/README.md b/comm/mailnews/extensions/newsblog/test/unit/resources/README.md new file mode 100644 index 0000000000..df655769b0 --- /dev/null +++ b/comm/mailnews/extensions/newsblog/test/unit/resources/README.md @@ -0,0 +1,24 @@ +Data files for unit testing the feeds code. + +- `rss_7_1.rdf` + Simple RSS1.0 feed example, from: + https://www.w3.org/2000/10/rdf-tests/RSS_1.0/rss_7_1.rdf + +- `rss_7_1_BORKED.rdf` + Sabotaged version of `rss_7_1.rdf` with a bad + <items> list, pointing to all sorts of URLs not + represented as <item>s in the feed (see Bug 476641). + +- `rss2_example.xml` + RSS2.0 example from wikipedia, but with + Japanese text in the title, with leading/trailing + whitespace. + +- `rss2_guid.xml` + RSS2.0 feed where two items have the same link but different guid. + (they should both appear in the feed). + +- `feeds-*/` + Test cases for migrating legacy .rdf config files to + the new json ones. The .rdf files are the old data, + and the .json files are the expected results. diff --git a/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-bad/feeditems.json b/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-bad/feeditems.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-bad/feeditems.json @@ -0,0 +1 @@ +{} diff --git a/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-bad/feeditems.rdf b/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-bad/feeditems.rdf new file mode 100644 index 0000000000..81a5a62ec5 --- /dev/null +++ b/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-bad/feeditems.rdf @@ -0,0 +1,6 @@ +<?xml version="1.0"?> +<RDF:RDF xmlns:dc="http://purl.org/dc/elements/1.1/" + xmlns:fz="urn:forumzilla:" + xmlns:NC="http://home.netscape.com/NC-rdf#" + xmlns:RDF="http://www.w3.org/1999/02/22-rdf-syntax-ns#"> +</RDF:RDF> diff --git a/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-bad/feeds.json b/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-bad/feeds.json new file mode 100644 index 0000000000..fe51488c70 --- /dev/null +++ b/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-bad/feeds.json @@ -0,0 +1 @@ +[] diff --git a/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-bad/feeds.rdf b/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-bad/feeds.rdf new file mode 100644 index 0000000000..f7e6b400e2 --- /dev/null +++ b/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-bad/feeds.rdf @@ -0,0 +1,17 @@ +<?xml version="1.0"?> +<RDF:RDF xmlns:NS1="http://purl.org/rss/1.0/" + xmlns:dc="http://purl.org/dc/elements/1.1/" + xmlns:fz="urn:forumzilla:" + xmlns:NC="http://home.netscape.com/NC-rdf#" + xmlns:RDF="http://www.w3.org/1999/02/22-rdf-syntax-ns#"> + <RDF:Description RDF:about="urn:forumzilla:root"> + <fz:feeds RDF:resource="rdf:#$cvA6q"/> + </RDF:Description> + <fz:feed RDF:about="https://example.com/feed/" + fz:quickMode="false" + dc:title="A feed with no dc:identifier, and thus no url. Should be ditched."> + </fz:feed> + <RDF:Seq RDF:about="rdf:#$cvA6q"> + <RDF:li RDF:resource="https://example.com/feed/"/> + </RDF:Seq> +</RDF:RDF> diff --git a/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-empty/feeditems.json b/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-empty/feeditems.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-empty/feeditems.json @@ -0,0 +1 @@ +{} diff --git a/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-empty/feeditems.rdf b/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-empty/feeditems.rdf new file mode 100644 index 0000000000..81a5a62ec5 --- /dev/null +++ b/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-empty/feeditems.rdf @@ -0,0 +1,6 @@ +<?xml version="1.0"?> +<RDF:RDF xmlns:dc="http://purl.org/dc/elements/1.1/" + xmlns:fz="urn:forumzilla:" + xmlns:NC="http://home.netscape.com/NC-rdf#" + xmlns:RDF="http://www.w3.org/1999/02/22-rdf-syntax-ns#"> +</RDF:RDF> diff --git a/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-empty/feeds.json b/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-empty/feeds.json new file mode 100644 index 0000000000..fe51488c70 --- /dev/null +++ b/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-empty/feeds.json @@ -0,0 +1 @@ +[] diff --git a/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-empty/feeds.rdf b/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-empty/feeds.rdf new file mode 100644 index 0000000000..a5f9c997e8 --- /dev/null +++ b/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-empty/feeds.rdf @@ -0,0 +1,12 @@ +<?xml version="1.0"?> +<RDF:RDF xmlns:NS1="http://purl.org/rss/1.0/" + xmlns:dc="http://purl.org/dc/elements/1.1/" + xmlns:fz="urn:forumzilla:" + xmlns:NC="http://home.netscape.com/NC-rdf#" + xmlns:RDF="http://www.w3.org/1999/02/22-rdf-syntax-ns#"> + <RDF:Description RDF:about="urn:forumzilla:root"> + <fz:feeds RDF:resource="rdf:#$cvA6q"/> + </RDF:Description> + <RDF:Seq RDF:about="rdf:#$cvA6q"> + </RDF:Seq> +</RDF:RDF> diff --git a/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-missing-timestamp/feeditems.json b/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-missing-timestamp/feeditems.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-missing-timestamp/feeditems.json @@ -0,0 +1 @@ +{} diff --git a/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-missing-timestamp/feeditems.rdf b/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-missing-timestamp/feeditems.rdf new file mode 100644 index 0000000000..81a5a62ec5 --- /dev/null +++ b/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-missing-timestamp/feeditems.rdf @@ -0,0 +1,6 @@ +<?xml version="1.0"?> +<RDF:RDF xmlns:dc="http://purl.org/dc/elements/1.1/" + xmlns:fz="urn:forumzilla:" + xmlns:NC="http://home.netscape.com/NC-rdf#" + xmlns:RDF="http://www.w3.org/1999/02/22-rdf-syntax-ns#"> +</RDF:RDF> diff --git a/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-missing-timestamp/feeds.json b/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-missing-timestamp/feeds.json new file mode 100644 index 0000000000..e20f7a2f00 --- /dev/null +++ b/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-missing-timestamp/feeds.json @@ -0,0 +1,23 @@ +[ + { + "title": "Government Digital Service", + "url": "https://gds.blog.gov.uk/feed/", + "quickMode": false, + "options": { + "version": 2, + "updates": { + "enabled": true, + "updateMinutes": 100, + "updateUnits": "min", + "lastUpdateTime": 1568784489107, + "lastDownloadTime": null, + "updatePeriod": "", + "updateFrequency": "", + "updateBase": "" + }, + "category": { "enabled": false, "prefixEnabled": false, "prefix": "" } + }, + "destFolder": "mailbox://nobody@Feeds/Government%20Digital%20Service", + "link": "https://gds.blog.gov.uk/feed/" + } +] diff --git a/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-missing-timestamp/feeds.rdf b/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-missing-timestamp/feeds.rdf new file mode 100644 index 0000000000..3b6b6b2df8 --- /dev/null +++ b/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-missing-timestamp/feeds.rdf @@ -0,0 +1,21 @@ +<?xml version="1.0"?> +<RDF:RDF xmlns:NS1="http://purl.org/rss/1.0/" + xmlns:dc="http://purl.org/dc/elements/1.1/" + xmlns:fz="urn:forumzilla:" + xmlns:NC="http://home.netscape.com/NC-rdf#" + xmlns:RDF="http://www.w3.org/1999/02/22-rdf-syntax-ns#"> + <RDF:Description RDF:about="urn:forumzilla:root"> + <fz:feeds RDF:resource="rdf:#$cvA6q"/> + </RDF:Description> + <fz:feed RDF:about="https://gds.blog.gov.uk/feed/" + fz:quickMode="false" + dc:title="Government Digital Service" + NS1:link="https://gds.blog.gov.uk/feed/" + fz:options="{"version":2,"updates":{"enabled":true,"updateMinutes":100,"updateUnits":"min","lastUpdateTime":1568784489107,"lastDownloadTime":null,"updatePeriod":"","updateFrequency":"","updateBase":""},"category":{"enabled":false,"prefixEnabled":false,"prefix":""}}" + dc:identifier="https://gds.blog.gov.uk/feed/"> + <fz:destFolder RDF:resource="mailbox://nobody@Feeds/Government%20Digital%20Service"/> + </fz:feed> + <RDF:Seq RDF:about="rdf:#$cvA6q"> + <RDF:li RDF:resource="https://gds.blog.gov.uk/feed/"/> + </RDF:Seq> +</RDF:RDF> diff --git a/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-simple/feeditems.json b/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-simple/feeditems.json new file mode 100644 index 0000000000..a16c68f0c0 --- /dev/null +++ b/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-simple/feeditems.json @@ -0,0 +1,122 @@ +{ + "https://gds.blog.gov.uk/?p=32978": { + "stored": true, + "valid": true, + "lastSeenTime": 1568784488792, + "feedURLs": ["https://gds.blog.gov.uk/feed/"] + }, + "https://gds.blog.gov.uk/?p=32944": { + "stored": true, + "valid": true, + "lastSeenTime": 1568784488971, + "feedURLs": ["https://gds.blog.gov.uk/feed/"] + }, + "https://gds.blog.gov.uk/?p=33011": { + "stored": true, + "valid": true, + "lastSeenTime": 1568784488610, + "feedURLs": ["https://gds.blog.gov.uk/feed/"] + }, + "https://gds.blog.gov.uk/?p=33020": { + "stored": true, + "valid": true, + "lastSeenTime": 1568784488551, + "feedURLs": ["https://gds.blog.gov.uk/feed/"] + }, + "https://civilservice.blog.gov.uk/?p=16464": { + "stored": true, + "valid": true, + "lastSeenTime": 1568784520041, + "feedURLs": ["https://civilservice.blog.gov.uk/feed/"] + }, + "https://gds.blog.gov.uk/?p=32951": { + "stored": true, + "valid": true, + "lastSeenTime": 1568784488909, + "feedURLs": ["https://gds.blog.gov.uk/feed/"] + }, + "https://gds.blog.gov.uk/?p=32963": { + "stored": true, + "valid": true, + "lastSeenTime": 1568784488851, + "feedURLs": ["https://gds.blog.gov.uk/feed/"] + }, + "https://civilservice.blog.gov.uk/?p=16431": { + "stored": true, + "valid": true, + "lastSeenTime": 1568784520152, + "feedURLs": ["https://civilservice.blog.gov.uk/feed/"] + }, + "https://civilservice.blog.gov.uk/?p=16477": { + "stored": true, + "valid": true, + "lastSeenTime": 1568784519983, + "feedURLs": ["https://civilservice.blog.gov.uk/feed/"] + }, + "https://gds.blog.gov.uk/?p=32939": { + "stored": true, + "valid": true, + "lastSeenTime": 1568784489030, + "feedURLs": ["https://gds.blog.gov.uk/feed/"] + }, + "https://civilservice.blog.gov.uk/?p=16453": { + "stored": true, + "valid": true, + "lastSeenTime": 1568784520096, + "feedURLs": ["https://civilservice.blog.gov.uk/feed/"] + }, + "https://civilservice.blog.gov.uk/?p=16418": { + "stored": true, + "valid": true, + "lastSeenTime": 1568784520209, + "feedURLs": ["https://civilservice.blog.gov.uk/feed/"] + }, + "https://civilservice.blog.gov.uk/?p=16507": { + "stored": true, + "valid": true, + "lastSeenTime": 1568784519869, + "feedURLs": ["https://civilservice.blog.gov.uk/feed/"] + }, + "https://civilservice.blog.gov.uk/?p=16490": { + "stored": true, + "valid": true, + "lastSeenTime": 1568784519926, + "feedURLs": ["https://civilservice.blog.gov.uk/feed/"] + }, + "https://civilservice.blog.gov.uk/?p=16378": { + "stored": true, + "valid": true, + "lastSeenTime": 1568784520323, + "feedURLs": ["https://civilservice.blog.gov.uk/feed/"] + }, + "https://gds.blog.gov.uk/?p=32927": { + "stored": true, + "valid": true, + "lastSeenTime": 1568784489089, + "feedURLs": ["https://gds.blog.gov.uk/feed/"] + }, + "https://gds.blog.gov.uk/?p=33001": { + "stored": true, + "valid": true, + "lastSeenTime": 1568784488670, + "feedURLs": ["https://gds.blog.gov.uk/feed/"] + }, + "https://civilservice.blog.gov.uk/?p=16393": { + "stored": true, + "valid": true, + "lastSeenTime": 1568784520265, + "feedURLs": ["https://civilservice.blog.gov.uk/feed/"] + }, + "https://civilservice.blog.gov.uk/?p=16514": { + "stored": true, + "valid": true, + "lastSeenTime": 1568784519812, + "feedURLs": ["https://civilservice.blog.gov.uk/feed/"] + }, + "https://gds.blog.gov.uk/?p=32988": { + "stored": true, + "valid": true, + "lastSeenTime": 1568784488731, + "feedURLs": ["https://gds.blog.gov.uk/feed/"] + } +} diff --git a/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-simple/feeditems.rdf b/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-simple/feeditems.rdf new file mode 100644 index 0000000000..f0941b5f6b --- /dev/null +++ b/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-simple/feeditems.rdf @@ -0,0 +1,126 @@ +<?xml version="1.0"?> +<RDF:RDF xmlns:dc="http://purl.org/dc/elements/1.1/" + xmlns:fz="urn:forumzilla:" + xmlns:NC="http://home.netscape.com/NC-rdf#" + xmlns:RDF="http://www.w3.org/1999/02/22-rdf-syntax-ns#"> + <RDF:Description RDF:about="urn:feeditem:https:%2f%2fgds.blog.gov.uk%2f%3fp=32978" + fz:stored="true" + fz:last-seen-timestamp="1568784488792" + fz:valid="true"> + <fz:feed RDF:resource="https://gds.blog.gov.uk/feed/"/> + </RDF:Description> + <RDF:Description RDF:about="urn:feeditem:https:%2f%2fgds.blog.gov.uk%2f%3fp=32944" + fz:stored="true" + fz:last-seen-timestamp="1568784488971" + fz:valid="true"> + <fz:feed RDF:resource="https://gds.blog.gov.uk/feed/"/> + </RDF:Description> + <RDF:Description RDF:about="urn:feeditem:https:%2f%2fgds.blog.gov.uk%2f%3fp=33011" + fz:stored="true" + fz:last-seen-timestamp="1568784488610" + fz:valid="true"> + <fz:feed RDF:resource="https://gds.blog.gov.uk/feed/"/> + </RDF:Description> + <RDF:Description RDF:about="urn:feeditem:https:%2f%2fgds.blog.gov.uk%2f%3fp=33020" + fz:stored="true" + fz:last-seen-timestamp="1568784488551" + fz:valid="true"> + <fz:feed RDF:resource="https://gds.blog.gov.uk/feed/"/> + </RDF:Description> + <RDF:Description RDF:about="urn:feeditem:https:%2f%2fcivilservice.blog.gov.uk%2f%3fp=16464" + fz:stored="true" + fz:last-seen-timestamp="1568784520041" + fz:valid="true"> + <fz:feed RDF:resource="https://civilservice.blog.gov.uk/feed/"/> + </RDF:Description> + <RDF:Description RDF:about="urn:feeditem:https:%2f%2fgds.blog.gov.uk%2f%3fp=32951" + fz:stored="true" + fz:last-seen-timestamp="1568784488909" + fz:valid="true"> + <fz:feed RDF:resource="https://gds.blog.gov.uk/feed/"/> + </RDF:Description> + <RDF:Description RDF:about="urn:feeditem:https:%2f%2fgds.blog.gov.uk%2f%3fp=32963" + fz:stored="true" + fz:last-seen-timestamp="1568784488851" + fz:valid="true"> + <fz:feed RDF:resource="https://gds.blog.gov.uk/feed/"/> + </RDF:Description> + <RDF:Description RDF:about="urn:feeditem:https:%2f%2fcivilservice.blog.gov.uk%2f%3fp=16431" + fz:stored="true" + fz:last-seen-timestamp="1568784520152" + fz:valid="true"> + <fz:feed RDF:resource="https://civilservice.blog.gov.uk/feed/"/> + </RDF:Description> + <RDF:Description RDF:about="urn:feeditem:https:%2f%2fcivilservice.blog.gov.uk%2f%3fp=16477" + fz:stored="true" + fz:last-seen-timestamp="1568784519983" + fz:valid="true"> + <fz:feed RDF:resource="https://civilservice.blog.gov.uk/feed/"/> + </RDF:Description> + <RDF:Description RDF:about="urn:feeditem:https:%2f%2fgds.blog.gov.uk%2f%3fp=32939" + fz:stored="true" + fz:last-seen-timestamp="1568784489030" + fz:valid="true"> + <fz:feed RDF:resource="https://gds.blog.gov.uk/feed/"/> + </RDF:Description> + <RDF:Description RDF:about="urn:feeditem:https:%2f%2fcivilservice.blog.gov.uk%2f%3fp=16453" + fz:stored="true" + fz:last-seen-timestamp="1568784520096" + fz:valid="true"> + <fz:feed RDF:resource="https://civilservice.blog.gov.uk/feed/"/> + </RDF:Description> + <RDF:Description RDF:about="urn:feeditem:https:%2f%2fcivilservice.blog.gov.uk%2f%3fp=16418" + fz:stored="true" + fz:last-seen-timestamp="1568784520209" + fz:valid="true"> + <fz:feed RDF:resource="https://civilservice.blog.gov.uk/feed/"/> + </RDF:Description> + <RDF:Description RDF:about="urn:feeditem:https:%2f%2fcivilservice.blog.gov.uk%2f%3fp=16507" + fz:stored="true" + fz:last-seen-timestamp="1568784519869" + fz:valid="true"> + <fz:feed RDF:resource="https://civilservice.blog.gov.uk/feed/"/> + </RDF:Description> + <RDF:Description RDF:about="urn:feeditem:https:%2f%2fcivilservice.blog.gov.uk%2f%3fp=16490" + fz:stored="true" + fz:last-seen-timestamp="1568784519926" + fz:valid="true"> + <fz:feed RDF:resource="https://civilservice.blog.gov.uk/feed/"/> + </RDF:Description> + <RDF:Description RDF:about="urn:feeditem:https:%2f%2fcivilservice.blog.gov.uk%2f%3fp=16378" + fz:stored="true" + fz:last-seen-timestamp="1568784520323" + fz:valid="true"> + <fz:feed RDF:resource="https://civilservice.blog.gov.uk/feed/"/> + </RDF:Description> + <RDF:Description RDF:about="urn:feeditem:https:%2f%2fgds.blog.gov.uk%2f%3fp=32927" + fz:stored="true" + fz:last-seen-timestamp="1568784489089" + fz:valid="true"> + <fz:feed RDF:resource="https://gds.blog.gov.uk/feed/"/> + </RDF:Description> + <RDF:Description RDF:about="urn:feeditem:https:%2f%2fgds.blog.gov.uk%2f%3fp=33001" + fz:stored="true" + fz:last-seen-timestamp="1568784488670" + fz:valid="true"> + <fz:feed RDF:resource="https://gds.blog.gov.uk/feed/"/> + </RDF:Description> + <RDF:Description RDF:about="urn:feeditem:https:%2f%2fcivilservice.blog.gov.uk%2f%3fp=16393" + fz:stored="true" + fz:last-seen-timestamp="1568784520265" + fz:valid="true"> + <fz:feed RDF:resource="https://civilservice.blog.gov.uk/feed/"/> + </RDF:Description> + <RDF:Description RDF:about="urn:feeditem:https:%2f%2fcivilservice.blog.gov.uk%2f%3fp=16514" + fz:stored="true" + fz:last-seen-timestamp="1568784519812" + fz:valid="true"> + <fz:feed RDF:resource="https://civilservice.blog.gov.uk/feed/"/> + </RDF:Description> + <RDF:Description RDF:about="urn:feeditem:https:%2f%2fgds.blog.gov.uk%2f%3fp=32988" + fz:stored="true" + fz:last-seen-timestamp="1568784488731" + fz:valid="true"> + <fz:feed RDF:resource="https://gds.blog.gov.uk/feed/"/> + </RDF:Description> +</RDF:RDF> diff --git a/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-simple/feeds.json b/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-simple/feeds.json new file mode 100644 index 0000000000..2ab7d4f780 --- /dev/null +++ b/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-simple/feeds.json @@ -0,0 +1,46 @@ +[ + { + "title": "Government Digital Service", + "lastModified": "Wed, 11 Sep 2019 15:47:49 GMT", + "url": "https://gds.blog.gov.uk/feed/", + "quickMode": false, + "options": { + "version": 2, + "updates": { + "enabled": true, + "updateMinutes": 100, + "updateUnits": "min", + "lastUpdateTime": 1568784489107, + "lastDownloadTime": null, + "updatePeriod": "", + "updateFrequency": "", + "updateBase": "" + }, + "category": { "enabled": false, "prefixEnabled": false, "prefix": "" } + }, + "destFolder": "mailbox://nobody@Feeds/Government%20Digital%20Service", + "link": "https://gds.blog.gov.uk/feed/" + }, + { + "title": "Civil Service", + "lastModified": "Tue, 17 Sep 2019 16:21:00 GMT", + "url": "https://civilservice.blog.gov.uk/feed/", + "quickMode": false, + "options": { + "version": 2, + "updates": { + "enabled": true, + "updateMinutes": 100, + "updateUnits": "min", + "lastUpdateTime": 1568784520338, + "lastDownloadTime": null, + "updatePeriod": "", + "updateFrequency": "", + "updateBase": "" + }, + "category": { "enabled": false, "prefixEnabled": false, "prefix": "" } + }, + "destFolder": "mailbox://nobody@Feeds/Civil%20Service", + "link": "https://civilservice.blog.gov.uk/feed/" + } +] diff --git a/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-simple/feeds.rdf b/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-simple/feeds.rdf new file mode 100644 index 0000000000..5c2fb72c74 --- /dev/null +++ b/comm/mailnews/extensions/newsblog/test/unit/resources/feeds-simple/feeds.rdf @@ -0,0 +1,32 @@ +<?xml version="1.0"?> +<RDF:RDF xmlns:NS1="http://purl.org/rss/1.0/" + xmlns:dc="http://purl.org/dc/elements/1.1/" + xmlns:fz="urn:forumzilla:" + xmlns:NC="http://home.netscape.com/NC-rdf#" + xmlns:RDF="http://www.w3.org/1999/02/22-rdf-syntax-ns#"> + <RDF:Description RDF:about="urn:forumzilla:root"> + <fz:feeds RDF:resource="rdf:#$cvA6q"/> + </RDF:Description> + <fz:feed RDF:about="https://gds.blog.gov.uk/feed/" + fz:quickMode="false" + dc:title="Government Digital Service" + NS1:link="https://gds.blog.gov.uk/feed/" + dc:lastModified="Wed, 11 Sep 2019 15:47:49 GMT" + fz:options="{"version":2,"updates":{"enabled":true,"updateMinutes":100,"updateUnits":"min","lastUpdateTime":1568784489107,"lastDownloadTime":null,"updatePeriod":"","updateFrequency":"","updateBase":""},"category":{"enabled":false,"prefixEnabled":false,"prefix":""}}" + dc:identifier="https://gds.blog.gov.uk/feed/"> + <fz:destFolder RDF:resource="mailbox://nobody@Feeds/Government%20Digital%20Service"/> + </fz:feed> + <fz:feed RDF:about="https://civilservice.blog.gov.uk/feed/" + fz:quickMode="false" + dc:title="Civil Service" + NS1:link="https://civilservice.blog.gov.uk/feed/" + dc:lastModified="Tue, 17 Sep 2019 16:21:00 GMT" + fz:options="{"version":2,"updates":{"enabled":true,"updateMinutes":100,"updateUnits":"min","lastUpdateTime":1568784520338,"lastDownloadTime":null,"updatePeriod":"","updateFrequency":"","updateBase":""},"category":{"enabled":false,"prefixEnabled":false,"prefix":""}}" + dc:identifier="https://civilservice.blog.gov.uk/feed/"> + <fz:destFolder RDF:resource="mailbox://nobody@Feeds/Civil%20Service"/> + </fz:feed> + <RDF:Seq RDF:about="rdf:#$cvA6q"> + <RDF:li RDF:resource="https://gds.blog.gov.uk/feed/"/> + <RDF:li RDF:resource="https://civilservice.blog.gov.uk/feed/"/> + </RDF:Seq> +</RDF:RDF> diff --git a/comm/mailnews/extensions/newsblog/test/unit/resources/rss2_example.xml b/comm/mailnews/extensions/newsblog/test/unit/resources/rss2_example.xml new file mode 100644 index 0000000000..0fe6268ec1 --- /dev/null +++ b/comm/mailnews/extensions/newsblog/test/unit/resources/rss2_example.xml @@ -0,0 +1,25 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<rss version="2.0"> +<channel> + <title> + + 本当に簡単なシンジケーションの例 + + </title> + <description>This is an example of an RSS feed</description> + <link>http://www.example.com/main.html</link> + <lastBuildDate>Mon, 06 Sep 2010 00:01:00 +0000 </lastBuildDate> + <pubDate>Sun, 06 Sep 2009 16:20:00 +0000</pubDate> + <ttl>1800</ttl> + + <item> + <title>Example entry</title> + <description>Here is some text containing an interesting description.</description> + <link>http://www.example.com/blog/post/1</link> + <guid isPermaLink="false">7bd204c6-1655-4c27-aeee-53f933c5395f</guid> + <pubDate>Sun, 06 Sep 2009 16:20:00 +0000</pubDate> + </item> + +</channel> +</rss> + diff --git a/comm/mailnews/extensions/newsblog/test/unit/resources/rss2_guid.xml b/comm/mailnews/extensions/newsblog/test/unit/resources/rss2_guid.xml new file mode 100644 index 0000000000..9f1efea9cc --- /dev/null +++ b/comm/mailnews/extensions/newsblog/test/unit/resources/rss2_guid.xml @@ -0,0 +1,42 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<rss version="2.0"> +<channel> + <title>GUID test</title> + <description>This RSS feed has multiple items with the same link, but with different guids. They should be treated as separate items (see Bug 1656090)</description> + <link>http://www.example.com/main.html</link> + <lastBuildDate>Mon, 06 Sep 2020 00:01:00 +0000 </lastBuildDate> + <pubDate>Sun, 06 Sep 2019 16:20:00 +0000</pubDate> + <ttl>1800</ttl> + + <item> + <title>Entry One</title> + <description>Blah blah blah.</description> + <link>http://www.example.com/blog/post/1</link> + <guid isPermaLink="false">0524a046-df56-11ea-8bc9-47d63411283f</guid> + <pubDate>Sun, 06 Sep 2019 16:20:00 +0000</pubDate> + </item> + <item> + <title>Entry One Again(with same link but different guid!)</title> + <description>Blah blah blah.</description> + <link>http://www.example.com/blog/post/1</link> + <guid isPermaLink="false">0524a35c-df56-11ea-8bca-43f820f3bd93</guid> + <pubDate>Sun, 06 Sep 2019 16:20:00 +0000</pubDate> + </item> + <item> + <title>Entry Two</title> + <description>Blah blah blah.</description> + <link>http://www.example.com/blog/post/2</link> + <guid isPermaLink="false">0524a442-df56-11ea-8bcb-035a71f5f71a</guid> + <pubDate>Sun, 06 Sep 2019 16:20:00 +0000</pubDate> + </item> + <item> + <title>Entry Three</title> + <description>Blah blah blah.</description> + <link>http://www.example.com/blog/post/3</link> + <guid isPermaLink="false">0524a53c-df56-11ea-8bcc-8f0977d58351</guid> + <pubDate>Sun, 06 Sep 2019 16:20:00 +0000</pubDate> + </item> + +</channel> +</rss> + diff --git a/comm/mailnews/extensions/newsblog/test/unit/resources/rss_7_1.rdf b/comm/mailnews/extensions/newsblog/test/unit/resources/rss_7_1.rdf new file mode 100644 index 0000000000..179965e7de --- /dev/null +++ b/comm/mailnews/extensions/newsblog/test/unit/resources/rss_7_1.rdf @@ -0,0 +1,66 @@ +<?xml version="1.0"?> + +<!-- RDF Site Summary (RSS) 1.0 + http://groups.yahoo.com/group/rss-dev/files/specification.html + Section 7 + --> + +<rdf:RDF + xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns="http://purl.org/rss/1.0/"> + + <channel rdf:about="http://www.xml.com/xml/news.rss"> + <title>XML.com</title> + <link>http://xml.com/pub</link> + <description> + XML.com features a rich mix of information and services + for the XML community. + </description> + + <image rdf:resource="http://xml.com/universal/images/xml_tiny.gif" /> + + <items> + <rdf:Seq> + <rdf:li resource="http://xml.com/pub/2000/08/09/xslt/xslt.html" /> + <rdf:li resource="http://xml.com/pub/2000/08/09/rdfdb/index.html" /> + </rdf:Seq> + </items> + + <textinput rdf:resource="http://search.xml.com" /> + + </channel> + + <image rdf:about="http://xml.com/universal/images/xml_tiny.gif"> + <title>XML.com</title> + <link>http://www.xml.com</link> + <url>http://xml.com/universal/images/xml_tiny.gif</url> + </image> + + <item rdf:about="http://xml.com/pub/2000/08/09/xslt/xslt.html"> + <title>Processing Inclusions with XSLT</title> + <link>http://xml.com/pub/2000/08/09/xslt/xslt.html</link> + <description> + Processing document inclusions with general XML tools can be + problematic. This article proposes a way of preserving inclusion + information through SAX-based processing. + </description> + </item> + + <item rdf:about="http://xml.com/pub/2000/08/09/rdfdb/index.html"> + <title>Putting RDF to Work</title> + <link>http://xml.com/pub/2000/08/09/rdfdb/index.html</link> + <description> + Tool and API support for the Resource Description Framework + is slowly coming of age. Edd Dumbill takes a look at RDFDB, + one of the most exciting new RDF toolkits. + </description> + </item> + + <textinput rdf:about="http://search.xml.com"> + <title>Search XML.com</title> + <description>Search XML.com's XML collection</description> + <name>s</name> + <link>http://search.xml.com</link> + </textinput> + +</rdf:RDF> diff --git a/comm/mailnews/extensions/newsblog/test/unit/resources/rss_7_1_BORKED.rdf b/comm/mailnews/extensions/newsblog/test/unit/resources/rss_7_1_BORKED.rdf new file mode 100644 index 0000000000..e2c6e0b109 --- /dev/null +++ b/comm/mailnews/extensions/newsblog/test/unit/resources/rss_7_1_BORKED.rdf @@ -0,0 +1,66 @@ +<?xml version="1.0"?> + +<!-- RDF Site Summary (RSS) 1.0 + http://groups.yahoo.com/group/rss-dev/files/specification.html + Section 7 + --> + +<rdf:RDF + xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns="http://purl.org/rss/1.0/"> + + <channel rdf:about="http://www.xml.com/xml/news.rss"> + <title>XML.com</title> + <link>http://xml.com/pub</link> + <description> + XML.com features a rich mix of information and services + for the XML community. + </description> + + <image rdf:resource="http://xml.com/universal/images/xml_tiny.gif" /> + + <items> + <rdf:Seq> + <rdf:li resource="http://OBVIOUSLY_WRONG_URL.com/blah/blah.html" /> + <rdf:li resource="http://ANOTHER_OBVIOUSLY_WRONG_URL.com/foo/bar/wibble.html" /> + </rdf:Seq> + </items> + + <textinput rdf:resource="http://search.xml.com" /> + + </channel> + + <image rdf:about="http://xml.com/universal/images/xml_tiny.gif"> + <title>XML.com</title> + <link>http://www.xml.com</link> + <url>http://xml.com/universal/images/xml_tiny.gif</url> + </image> + + <item rdf:about="http://xml.com/pub/2000/08/09/xslt/xslt.html"> + <title>Processing Inclusions with XSLT</title> + <link>http://xml.com/pub/2000/08/09/xslt/xslt.html</link> + <description> + Processing document inclusions with general XML tools can be + problematic. This article proposes a way of preserving inclusion + information through SAX-based processing. + </description> + </item> + + <item rdf:about="http://xml.com/pub/2000/08/09/rdfdb/index.html"> + <title>Putting RDF to Work</title> + <link>http://xml.com/pub/2000/08/09/rdfdb/index.html</link> + <description> + Tool and API support for the Resource Description Framework + is slowly coming of age. Edd Dumbill takes a look at RDFDB, + one of the most exciting new RDF toolkits. + </description> + </item> + + <textinput rdf:about="http://search.xml.com"> + <title>Search XML.com</title> + <description>Search XML.com's XML collection</description> + <name>s</name> + <link>http://search.xml.com</link> + </textinput> + +</rdf:RDF> diff --git a/comm/mailnews/extensions/newsblog/test/unit/test_feedparser.js b/comm/mailnews/extensions/newsblog/test/unit/test_feedparser.js new file mode 100644 index 0000000000..6577280356 --- /dev/null +++ b/comm/mailnews/extensions/newsblog/test/unit/test_feedparser.js @@ -0,0 +1,146 @@ +// see https://www.w3.org/2000/10/rdf-tests/ for test files (including rss1) +// - test examples for all feed types +// - test items in test_download() +// - test rss1 feed with itunes `new-feed-url` redirect +// - test rss1 feed with RSS syndication extension tags (updatePeriod et al) +// - test multiple/missing authors (with fallback to feed title) +// - test missing dates +// - test content formatting + +// Some RSS1 feeds in the wild: +// https://www.livejournal.com/stats/latest-rss.bml +// https://journals.sagepub.com/action/showFeed?ui=0&mi=ehikzz&ai=2b4&jc=acrc&type=etoc&feed=rss +// https://www.revolutionspodcast.com/index.rdf +// https://www.tandfonline.com/feed/rss/uasa20 +// http://export.arxiv.org/rss/astro-ph +// - uses html formatting in <dc:creator> + +// Helper to compare feeditems. +function assertItemsEqual(got, expected) { + Assert.equal(got.length, expected.length); + for (let i = 0; i < expected.length; i++) { + // Only check fields in expected. Means testdata can exclude "description" and other bulky fields. + for (let k of Object.keys(expected[i])) { + Assert.equal(got[i][k], expected[i][k]); + } + } +} + +// Test the rss1 feed parser +add_task(async function test_rss1() { + // Boilerplate. + let account = FeedUtils.createRssAccount("test_rss1"); + let rootFolder = account.incomingServer.rootMsgFolder.QueryInterface( + Ci.nsIMsgLocalMailFolder + ); + let folder = rootFolder.createLocalSubfolder("folderofeeds"); + + // These two files yield the same feed, but the second one has a sabotaged + // <items> to simulate badly-encoded feeds seen in the wild. + for (let testFile of [ + "resources/rss_7_1.rdf", + "resources/rss_7_1_BORKED.rdf", + ]) { + dump(`checking ${testFile}\n`); + // Would be nicer to use the test http server to fetch the file, but that + // would involve XMLHTTPRequest. This is more concise. + let doc = await do_parse_document(testFile, "application/xml"); + let feed = new Feed( + "https://www.w3.org/2000/10/rdf-tests/RSS_1.0/rss_7_1.rdf", + folder + ); + feed.parseItems = true; // We want items too, not just the feed details. + feed.onParseError = function (f) { + throw new Error("PARSE ERROR"); + }; + let parser = new FeedParser(); + let items = parser.parseAsRSS1(feed, doc); + + // Check some channel details. + Assert.equal(feed.title, "XML.com"); + Assert.equal(feed.link, "http://xml.com/pub"); + + // Check the items (the titles and links at least!). + assertItemsEqual(items, [ + { + url: "http://xml.com/pub/2000/08/09/xslt/xslt.html", + title: "Processing Inclusions with XSLT", + }, + { + url: "http://xml.com/pub/2000/08/09/rdfdb/index.html", + title: "Putting RDF to Work", + }, + ]); + } +}); + +// Test feed downloading. +// Mainly checking that it doesn't crash and that the right feed parser is used. +add_task(async function test_download() { + // Boilerplate + let account = FeedUtils.createRssAccount("test_feed_download"); + let rootFolder = account.incomingServer.rootMsgFolder.QueryInterface( + Ci.nsIMsgLocalMailFolder + ); + + // load & parse example rss feed + // Feed object rejects anything other than http and https, so we're + // running a local http server for testing (see head_feeds.js for it). + let feedTests = [ + { + url: "http://localhost:" + SERVER_PORT + "/rss_7_1.rdf", + feedType: "RSS_1.xRDF", + title: "XML.com", + expectedItems: 2, + }, + { + // Has Japanese title with leading/trailing whitespace. + url: "http://localhost:" + SERVER_PORT + "/rss2_example.xml", + feedType: "RSS_2.0", + title: "本当に簡単なシンジケーションの例", + expectedItems: 1, + }, + { + // Has two items with same link but different guid (Bug 1656090). + url: "http://localhost:" + SERVER_PORT + "/rss2_guid.xml", + feedType: "RSS_2.0", + title: "GUID test", + expectedItems: 4, + }, + // TODO: examples for the other feed types! + ]; + + let n = 1; + for (let test of feedTests) { + let folder = rootFolder.createLocalSubfolder("feed" + n); + n++; + let feed = new Feed(test.url, folder); + + let dl = new Promise(function (resolve, reject) { + let cb = { + downloaded(f, error, disable) { + if (error != FeedUtils.kNewsBlogSuccess) { + reject( + new Error(`download failed (url=${feed.url} error=${error})`) + ); + return; + } + // Feed has downloaded - make sure the right type was detected. + Assert.equal(feed.mFeedType, test.feedType, "feed type matching"); + Assert.equal(feed.title, test.title, "title matching"); + // Make sure we're got the expected number of messages in the folder. + let cnt = [...folder.messages].length; + Assert.equal(cnt, test.expectedItems, "itemcount matching"); + + resolve(); + }, + onProgress(f, loaded, total, lengthComputable) {}, + }; + + feed.download(true, cb); + }); + + // Wait for this feed to complete downloading. + await dl; + } +}); diff --git a/comm/mailnews/extensions/newsblog/test/unit/test_rdfmigration.js b/comm/mailnews/extensions/newsblog/test/unit/test_rdfmigration.js new file mode 100644 index 0000000000..a2bc2cf702 --- /dev/null +++ b/comm/mailnews/extensions/newsblog/test/unit/test_rdfmigration.js @@ -0,0 +1,61 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const { MailMigrator } = ChromeUtils.import( + "resource:///modules/MailMigrator.jsm" +); + +/** + * Tests migration of old .rdf feed config files to the new .json files. + * + * @param {String} testDataDir - A directory containing legacy feeds.rdf and + * feeditems.rdf files, along with coressponding + * .json files containing the expected results + * of the migration. + * @returns {void} + */ +async function migrationTest(testDataDir) { + // Set up an RSS account/server. + let account = FeedUtils.createRssAccount("rss_migration_test"); + let rootFolder = account.incomingServer.rootMsgFolder.QueryInterface( + Ci.nsIMsgLocalMailFolder + ); + // Note, we don't create any folders to hold downloaded feed items, + // that's OK here, because we're only migrating the config files, not + // downloading feeds. The migration doesn't check destFolder existence. + let rootDir = rootFolder.filePath.path; + + // Install legacy feeds.rdf/feeditems.rdf + for (let f of ["feeds.rdf", "feeditems.rdf"]) { + await IOUtils.copy( + PathUtils.join(testDataDir, f), + PathUtils.join(rootDir, f) + ); + } + + // Perform the migration + await MailMigrator._migrateRSSServer(account.incomingServer); + + // Check actual results against expectations. + for (let f of ["feeds.json", "feeditems.json"]) { + let got = await IOUtils.readJSON(PathUtils.join(rootDir, f)); + let expected = await IOUtils.readJSON(PathUtils.join(testDataDir, f)); + Assert.deepEqual(got, expected, `match ${testDataDir}/${f}`); + } + + // Delete the account and all it's files. + MailServices.accounts.removeAccount(account, true); +} + +add_task(async function test_rdfmigration() { + let testDataDirs = [ + "feeds-simple", + "feeds-empty", + "feeds-missing-timestamp", + "feeds-bad", + ]; + for (let d of testDataDirs) { + await migrationTest(do_get_file("resources/" + d).path); + } +}); diff --git a/comm/mailnews/extensions/newsblog/test/unit/xpcshell.ini b/comm/mailnews/extensions/newsblog/test/unit/xpcshell.ini new file mode 100644 index 0000000000..3b7fc2869c --- /dev/null +++ b/comm/mailnews/extensions/newsblog/test/unit/xpcshell.ini @@ -0,0 +1,8 @@ +[DEFAULT] +head = head_feeds.js +tail = +support-files = resources/* + +[test_feedparser.js] +[test_rdfmigration.js] + diff --git a/comm/mailnews/extensions/offline-startup/OfflineStartup.jsm b/comm/mailnews/extensions/offline-startup/OfflineStartup.jsm new file mode 100644 index 0000000000..a710b7ef32 --- /dev/null +++ b/comm/mailnews/extensions/offline-startup/OfflineStartup.jsm @@ -0,0 +1,126 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +var EXPORTED_SYMBOLS = ["OfflineStartup"]; + +var kDebug = false; +var kOfflineStartupPref = "offline.startup_state"; +var kRememberLastState = 0; +var kAskForOnlineState = 1; +var kAlwaysOnline = 2; +var kAlwaysOffline = 3; +var kAutomatic = 4; +var gStartingUp = true; +var gOfflineStartupMode; // 0 = remember last state, 1 = ask me, 2 == online, 3 == offline, 4 = automatic +var gDebugLog; + +// Debug helper + +if (!kDebug) { + gDebugLog = function (m) {}; +} else { + gDebugLog = function (m) { + dump("\t *** nsOfflineStartup: " + m + "\n"); + }; +} + +// nsOfflineStartup : nsIObserver +// +// Check if the user has set the pref to be prompted for +// online/offline startup mode. If so, prompt the user. Also, +// check if the user wants to remember their offline state +// the next time they start up. +// If the user shutdown offline, and is now starting up in online +// mode, we will set the boolean pref "mailnews.playback_offline" to true. + +function OfflineStartup() {} +OfflineStartup.prototype = { + onProfileStartup() { + gDebugLog("onProfileStartup"); + + if (gStartingUp) { + gStartingUp = false; + // if checked, the "work offline" checkbox overrides + if (Services.io.offline && !Services.io.manageOfflineStatus) { + gDebugLog("already offline!"); + return; + } + } + + var manageOfflineStatus = Services.prefs.getBoolPref("offline.autoDetect"); + gOfflineStartupMode = Services.prefs.getIntPref(kOfflineStartupPref); + let wasOffline = !Services.prefs.getBoolPref("network.online"); + + if (gOfflineStartupMode == kAutomatic) { + // Offline state should be managed automatically + // so do nothing specific at startup. + } else if (gOfflineStartupMode == kAlwaysOffline) { + Services.io.manageOfflineStatus = false; + Services.io.offline = true; + } else if (gOfflineStartupMode == kAlwaysOnline) { + Services.io.manageOfflineStatus = manageOfflineStatus; + if (wasOffline) { + Services.prefs.setBoolPref("mailnews.playback_offline", true); + } + // If we're managing the offline status, don't force online here... it may + // be the network really is offline. + if (!manageOfflineStatus) { + Services.io.offline = false; + } + } else if (gOfflineStartupMode == kRememberLastState) { + Services.io.manageOfflineStatus = manageOfflineStatus && !wasOffline; + // If we are meant to be online, and managing the offline status + // then don't force it - it may be the network really is offline. + if (!manageOfflineStatus || wasOffline) { + Services.io.offline = wasOffline; + } + } else if (gOfflineStartupMode == kAskForOnlineState) { + var bundle = Services.strings.createBundle( + "chrome://messenger/locale/offlineStartup.properties" + ); + var title = bundle.GetStringFromName("title"); + var desc = bundle.GetStringFromName("desc"); + var button0Text = bundle.GetStringFromName("workOnline"); + var button1Text = bundle.GetStringFromName("workOffline"); + var checkVal = { value: 0 }; + + // Set Offline to true by default to prevent new mail checking at startup before + // the user answers the following question. + Services.io.manageOfflineStatus = false; + Services.io.offline = true; + var result = Services.prompt.confirmEx( + null, + title, + desc, + Services.prompt.BUTTON_POS_0 * Services.prompt.BUTTON_TITLE_IS_STRING + + Services.prompt.BUTTON_POS_1 * Services.prompt.BUTTON_TITLE_IS_STRING, + button0Text, + button1Text, + null, + null, + checkVal + ); + gDebugLog("result = " + result + "\n"); + Services.io.manageOfflineStatus = manageOfflineStatus && result != 1; + Services.io.offline = result == 1; + if (result != 1 && wasOffline) { + Services.prefs.setBoolPref("mailnews.playback_offline", true); + } + } + }, + + observe(aSubject, aTopic, aData) { + gDebugLog("observe: " + aTopic); + + if (aTopic == "profile-change-net-teardown") { + gDebugLog("remembering offline state"); + Services.prefs.setBoolPref("network.online", !Services.io.offline); + } else if (aTopic == "profile-after-change") { + Services.obs.addObserver(this, "profile-change-net-teardown"); + this.onProfileStartup(); + } + }, + + QueryInterface: ChromeUtils.generateQI(["nsIObserver"]), +}; diff --git a/comm/mailnews/extensions/offline-startup/components.conf b/comm/mailnews/extensions/offline-startup/components.conf new file mode 100644 index 0000000000..8ca8a2cf2c --- /dev/null +++ b/comm/mailnews/extensions/offline-startup/components.conf @@ -0,0 +1,15 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +Classes = [ + { + "cid": "{3028a3c8-2165-42a4-b878-398da5d32736}", + "contract_ids": ["@mozilla.org/offline-startup;1"], + "jsm": "resource:///modules/OfflineStartup.jsm", + "constructor": "OfflineStartup", + "categories": {"profile-after-change": "Offline-startup"}, + }, +] diff --git a/comm/mailnews/extensions/offline-startup/moz.build b/comm/mailnews/extensions/offline-startup/moz.build new file mode 100644 index 0000000000..e3b8854866 --- /dev/null +++ b/comm/mailnews/extensions/offline-startup/moz.build @@ -0,0 +1,12 @@ +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +EXTRA_JS_MODULES += [ + "OfflineStartup.jsm", +] + +XPCOM_MANIFESTS += [ + "components.conf", +] diff --git a/comm/mailnews/extensions/smime/certFetchingStatus.js b/comm/mailnews/extensions/smime/certFetchingStatus.js new file mode 100644 index 0000000000..ea7cd63226 --- /dev/null +++ b/comm/mailnews/extensions/smime/certFetchingStatus.js @@ -0,0 +1,255 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +let USER_CERT_ATTRIBUTE = "usercertificate;binary"; + +let gEmailAddresses; +let gDirectoryPref; +let gLdapServerURL; +let gLdapConnection; +let gCertDB; +let gLdapOperation; +let gLogin; + +window.addEventListener("DOMContentLoaded", onLoad); +document.addEventListener("dialogcancel", stopFetching); + +/** + * Expects the following arguments: + * - pref name of LDAP directory to fetch from + * - array with email addresses + * + * Display modal dialog with message and stop button. + * In onload, kick off binding to LDAP. + * When bound, kick off the searches. + * On finding certificates, import into permanent cert database. + * When all searches are finished, close the dialog. + */ +function onLoad() { + gDirectoryPref = window.arguments[0]; + gEmailAddresses = window.arguments[1]; + + if (!gEmailAddresses.length) { + window.close(); + return; + } + + setTimeout(search); +} + +function search() { + // Get the login to authenticate as, if there is one. No big deal if we don't + // have one. + gLogin = Services.prefs.getStringPref(gDirectoryPref + ".auth.dn", undefined); + + try { + let url = Services.prefs.getCharPref(gDirectoryPref + ".uri"); + + gLdapServerURL = Services.io.newURI(url).QueryInterface(Ci.nsILDAPURL); + + gLdapConnection = Cc["@mozilla.org/network/ldap-connection;1"] + .createInstance() + .QueryInterface(Ci.nsILDAPConnection); + + gLdapConnection.init( + gLdapServerURL, + gLogin, + new BindListener(), + null, + Ci.nsILDAPConnection.VERSION3 + ); + } catch (ex) { + console.error(ex); + window.close(); + } +} + +function stopFetching() { + if (gLdapOperation) { + try { + gLdapOperation.abandon(); + } catch (e) {} + } +} + +function importCert(ber_value) { + if (!gCertDB) { + gCertDB = Cc["@mozilla.org/security/x509certdb;1"].getService( + Ci.nsIX509CertDB + ); + } + + // ber_value has type nsILDAPBERValue + let cert_bytes = ber_value.get(); + if (cert_bytes) { + gCertDB.importEmailCertificate(cert_bytes, cert_bytes.length, null); + } +} + +function getLDAPOperation() { + gLdapOperation = Cc["@mozilla.org/network/ldap-operation;1"].createInstance( + Ci.nsILDAPOperation + ); + + gLdapOperation.init(gLdapConnection, new LDAPMessageListener(), null); +} + +async function getPassword() { + // we only need a password if we are using credentials + if (!gLogin) { + return null; + } + let authPrompter = Services.ww.getNewAuthPrompter(window); + let strBundle = document.getElementById("bundle_ldap"); + let password = { value: "" }; + + // nsLDAPAutocompleteSession uses asciiHost instead of host for the prompt + // text, I think we should be consistent. + if ( + await authPrompter.asyncPromptPassword( + strBundle.getString("authPromptTitle"), + strBundle.getFormattedString("authPromptText", [ + gLdapServerURL.asciiHost, + ]), + gLdapServerURL.spec, + authPrompter.SAVE_PASSWORD_PERMANENTLY, + password + ) + ) { + return password.value; + } + return null; +} + +/** + * Checks if the LDAP connection can be bound. + * @implements {nsILDAPMessageListener} + */ +class BindListener { + QueryInterface = ChromeUtils.generateQI(["nsILDAPMessageListener"]); + + async onLDAPInit(conn, status) { + // Kick off bind. + getLDAPOperation(); + gLdapOperation.simpleBind(await getPassword()); + } + + onLDAPMessage(message) {} + + onLDAPError(status, secInfo, location) { + if (secInfo) { + console.warn(`LDAP bind connection security error for ${location}`); + } else { + console.warn(`LDAP bind error: ${status}`); + } + window.close(); + } +} + +/** + * LDAPMessageListener. + * @implements {nsILDAPMessageListener} + */ +class LDAPMessageListener { + QueryInterface = ChromeUtils.generateQI(["nsILDAPMessageListener"]); + + onLDAPInit(conn, status) {} + + onLDAPMessage(message) { + if (Ci.nsILDAPMessage.RES_SEARCH_RESULT == message.type) { + window.close(); + return; + } + + if (Ci.nsILDAPMessage.RES_BIND == message.type) { + if (Ci.nsILDAPErrors.SUCCESS != message.errorCode) { + window.close(); + return; + } + // Kick off search. + let prefix1 = ""; + let suffix1 = ""; + + let urlFilter = gLdapServerURL.filter; + if ( + urlFilter != null && + urlFilter.length > 0 && + urlFilter != "(objectclass=*)" + ) { + if (urlFilter.startsWith("(")) { + prefix1 = "(&" + urlFilter; + } else { + prefix1 = "(&(" + urlFilter + ")"; + } + suffix1 = ")"; + } + + let prefix2 = ""; + let suffix2 = ""; + + if (gEmailAddresses.length > 1) { + prefix2 = "(|"; + suffix2 = ")"; + } + + let mailFilter = ""; + + for (let email of gEmailAddresses) { + mailFilter += "(mail=" + email + ")"; + } + + let filter = prefix1 + prefix2 + mailFilter + suffix2 + suffix1; + + // Max search results => + // Double number of email addresses, because each person might have + // multiple certificates listed. We expect at most two certificates, + // one for signing, one for encrypting. + // Maybe that number should be larger, to allow for deployments, + // where even more certs can be stored per user??? + + let maxEntriesWanted = gEmailAddresses.length * 2; + + getLDAPOperation(); + gLdapOperation.searchExt( + gLdapServerURL.dn, + gLdapServerURL.scope, + filter, + USER_CERT_ATTRIBUTE, + 0, + maxEntriesWanted + ); + return; + } + + if (Ci.nsILDAPMessage.RES_SEARCH_ENTRY == message.type) { + let outBinValues = null; + try { + // This call may throw if the result message is empty or doesn't + // contain this attribute. + // It's an allowed condition that the attribute is missing on + // the server, so we silently ignore a failure to obtain it. + outBinValues = message.getBinaryValues(USER_CERT_ATTRIBUTE); + } catch (ex) {} + if (outBinValues) { + for (let i = 0; i < outBinValues.length; ++i) { + importCert(outBinValues[i]); + } + } + } + } + + /** + * @param {nsresult} status + * @param {?nsITransportSecurityInfo} secInfo + * @param {?string} location + */ + onLDAPError(status, secInfo, location) { + if (secInfo) { + console.warn(`LDAP connection security error for ${location}`); + } else { + console.warn(`LDAP error: ${status}`); + } + window.close(); + } +} diff --git a/comm/mailnews/extensions/smime/certFetchingStatus.xhtml b/comm/mailnews/extensions/smime/certFetchingStatus.xhtml new file mode 100644 index 0000000000..b82042edda --- /dev/null +++ b/comm/mailnews/extensions/smime/certFetchingStatus.xhtml @@ -0,0 +1,40 @@ +<?xml version="1.0"?> +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> + +<?xml-stylesheet href="chrome://global/skin/global.css" type="text/css"?> +<?xml-stylesheet href="chrome://messenger/skin/messenger.css" type="text/css"?> + +<!DOCTYPE html SYSTEM "chrome://messenger-smime/locale/certFetchingStatus.dtd"> + +<html + xmlns="http://www.w3.org/1999/xhtml" + xmlns:xul="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul" + xmlns:html="http://www.w3.org/1999/xhtml" + scrolling="false" +> + <head> + <title>&title.label;</title> + <script + defer="defer" + src="chrome://messenger-smime/content/certFetchingStatus.js" + ></script> + </head> + <html:body + xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul" + > + <dialog + id="certFetchingStatus" + buttons="cancel" + buttonlabelcancel="&stop.label;" + > + <stringbundle + id="bundle_ldap" + src="chrome://mozldap/locale/ldap.properties" + /> + + <description>&info.message;</description> + </dialog> + </html:body> +</html> diff --git a/comm/mailnews/extensions/smime/certpicker.js b/comm/mailnews/extensions/smime/certpicker.js new file mode 100644 index 0000000000..31ed9d3ed2 --- /dev/null +++ b/comm/mailnews/extensions/smime/certpicker.js @@ -0,0 +1,69 @@ +/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +"use strict"; + +var dialogParams; +var itemCount = 0; + +window.addEventListener("DOMContentLoaded", onLoad); + +document.addEventListener("dialogaccept", doOK); +document.addEventListener("dialogcancel", doCancel); + +function onLoad() { + dialogParams = window.arguments[0].QueryInterface(Ci.nsIDialogParamBlock); + + var selectElement = document.getElementById("nicknames"); + itemCount = dialogParams.GetInt(0); + + var selIndex = dialogParams.GetInt(1); + if (selIndex < 0) { + selIndex = 0; + } + + for (let i = 0; i < itemCount; i++) { + let menuItemNode = document.createXULElement("menuitem"); + let nick = dialogParams.GetString(i); + menuItemNode.setAttribute("value", i); + menuItemNode.setAttribute("label", nick); // This is displayed. + selectElement.menupopup.appendChild(menuItemNode); + + if (selIndex == i) { + selectElement.selectedItem = menuItemNode; + } + } + + dialogParams.SetInt(0, 0); // Set cancel return value. + setDetails(); +} + +function setDetails() { + let selItem = document.getElementById("nicknames").value; + if (selItem.length == 0) { + return; + } + + let index = parseInt(selItem); + let details = dialogParams.GetString(index + itemCount); + document.getElementById("details").value = details; +} + +function onCertSelected() { + setDetails(); +} + +function doOK() { + // Signal that the user accepted. + dialogParams.SetInt(0, 1); + + // Signal the index of the selected cert in the list of cert nicknames + // provided. + let index = parseInt(document.getElementById("nicknames").value); + dialogParams.SetInt(1, index); +} + +function doCancel() { + dialogParams.SetInt(0, 0); // Signal that the user cancelled. +} diff --git a/comm/mailnews/extensions/smime/certpicker.xhtml b/comm/mailnews/extensions/smime/certpicker.xhtml new file mode 100644 index 0000000000..c0ff513cb6 --- /dev/null +++ b/comm/mailnews/extensions/smime/certpicker.xhtml @@ -0,0 +1,54 @@ +<?xml version="1.0"?> +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> + +<?xml-stylesheet href="chrome://global/skin/global.css" type="text/css"?> + +<!DOCTYPE html [ <!ENTITY % amE2EDTD SYSTEM "chrome://messenger/locale/am-smime.dtd"> +%amE2EDTD; ]> + +<html + xmlns="http://www.w3.org/1999/xhtml" + xmlns:xul="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul" + xmlns:html="http://www.w3.org/1999/xhtml" + scrolling="false" +> + <head> + <title>&certPicker.title;</title> + <script + defer="defer" + src="chrome://messenger/content/globalOverlay.js" + ></script> + <script + defer="defer" + src="chrome://global/content/editMenuOverlay.js" + ></script> + <script + defer="defer" + src="chrome://messenger/content/certpicker.js" + ></script> + </head> + <html:body + xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul" + > + <dialog id="certPicker" buttons="accept,cancel"> + <hbox align="center"> + <label id="pickerInfo" value="&certPicker.info;" /> + <!-- The items in this menulist must never be sorted, + but remain in the order filled by the application + --> + <menulist id="nicknames" oncommand="onCertSelected();"> + <menupopup /> + </menulist> + </hbox> + <separator class="thin" /> + <label value="&certPicker.detailsLabel;" /> + <html:textarea + id="details" + readonly="readonly" + style="height: 12em" + ></html:textarea> + </dialog> + </html:body> +</html> diff --git a/comm/mailnews/extensions/smime/components.conf b/comm/mailnews/extensions/smime/components.conf new file mode 100644 index 0000000000..a5287d608a --- /dev/null +++ b/comm/mailnews/extensions/smime/components.conf @@ -0,0 +1,63 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, you can obtain one at http://mozilla.org/MPL/2.0/. + +Classes = [ + { + "cid": "{54976882-7421-4286-8ecc-46373f15d7b5}", + "contract_ids": ["@mozilla.org/messengercompose/composesecure;1"], + "type": "nsMsgComposeSecure", + "headers": ["/comm/mailnews/extensions/smime/nsMsgComposeSecure.h"], + }, + { + "cid": "{a0134d58-018f-4d40-a099-fa079e5024a6}", + "contract_ids": ["@mozilla.org/messenger-smime/smime-encrypted-uris-service;1"], + "type": "nsEncryptedSMIMEURIsService", + "headers": ["/comm/mailnews/extensions/smime/nsEncryptedSMIMEURIsService.h"], + }, + { + "cid": "{5fb907e0-1dd2-11b2-a7c0-f14c416a62a1}", + "contract_ids": ["@mozilla.org/nsCMSSecureMessage;1"], + "type": "nsCMSSecureMessage", + "init_method": "Init", + "headers": ["/comm/mailnews/extensions/smime/nsCMSSecureMessage.h"], + }, + { + "cid": "{9dcef3a4-a3bc-11d5-ba47-00108303b117}", + "contract_ids": ["@mozilla.org/nsCMSDecoder;1"], + "type": "nsCMSDecoder", + "init_method": "Init", + "headers": ["/comm/mailnews/extensions/smime/nsCMS.h"], + }, + { + "cid": "{fb62c8ed-b875-488a-be35-ab9764bcad25}", + "contract_ids": ["@mozilla.org/nsCMSDecoderJS;1"], + "type": "nsCMSDecoderJS", + "init_method": "Init", + "headers": ["/comm/mailnews/extensions/smime/nsCMS.h"], + }, + { + "cid": "{a15789aa-8903-462b-81e9-4aa2cff4d5cb}", + "contract_ids": ["@mozilla.org/nsCMSEncoder;1"], + "type": "nsCMSEncoder", + "init_method": "Init", + "headers": ["/comm/mailnews/extensions/smime/nsCMS.h"], + }, + { + "cid": "{a4557478-ae16-11d5-ba4b-00108303b117}", + "contract_ids": ["@mozilla.org/nsCMSMessage;1"], + "type": "nsCMSMessage", + "init_method": "Init", + "headers": ["/comm/mailnews/extensions/smime/nsCMS.h"], + }, + { + "cid": "{735959a1-af01-447e-b02d-56e968fa52b4}", + "contract_ids": [ + "@mozilla.org/nsCertPickDialogs;1", + "@mozilla.org/user_cert_picker;1", + ], + "type": "nsCertPicker", + "init_method": "Init", + "headers": ["/comm/mailnews/extensions/smime/nsCertPicker.h"], + }, +] diff --git a/comm/mailnews/extensions/smime/moz.build b/comm/mailnews/extensions/smime/moz.build new file mode 100644 index 0000000000..f33678ecf2 --- /dev/null +++ b/comm/mailnews/extensions/smime/moz.build @@ -0,0 +1,37 @@ +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +XPIDL_SOURCES += [ + "nsICertPickDialogs.idl", + "nsICMSDecoder.idl", + "nsICMSDecoderJS.idl", + "nsICMSEncoder.idl", + "nsICMSMessage.idl", + "nsICMSMessageErrors.idl", + "nsICMSSecureMessage.idl", + "nsIEncryptedSMIMEURIsSrvc.idl", + "nsIMsgSMIMEHeaderSink.idl", + "nsIUserCertPicker.idl", +] + +XPIDL_MODULE = "msgsmime" + +SOURCES += [ + "nsCertPicker.cpp", + "nsCMS.cpp", + "nsCMSSecureMessage.cpp", + "nsEncryptedSMIMEURIsService.cpp", + "nsMsgComposeSecure.cpp", +] + +FINAL_LIBRARY = "mail" + +LOCAL_INCLUDES += [ + "/security/certverifier", + "/security/manager/pki", + "/security/manager/ssl", +] + +XPCOM_MANIFESTS += ["components.conf"] diff --git a/comm/mailnews/extensions/smime/msgCompSecurityInfo.js b/comm/mailnews/extensions/smime/msgCompSecurityInfo.js new file mode 100644 index 0000000000..eb760a9c58 --- /dev/null +++ b/comm/mailnews/extensions/smime/msgCompSecurityInfo.js @@ -0,0 +1,122 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +var gListBox; +var gViewButton; +var gBundle; + +var gCerts = []; + +window.addEventListener("DOMContentLoaded", onLoad); +window.addEventListener("resize", resizeColumns); + +function onLoad() { + let params = window.arguments[0]; + if (!params) { + return; + } + + gListBox = document.getElementById("infolist"); + gViewButton = document.getElementById("viewCertButton"); + gBundle = document.getElementById("bundle_smime_comp_info"); + + let certdb = Cc["@mozilla.org/security/x509certdb;1"].getService( + Ci.nsIX509CertDB + ); + + let missing = []; + for (let i = 0; i < params.recipients.length; i++) { + let email = params.recipients[i]; + let dbKey = params.compFields.composeSecure.getCertDBKeyForEmail(email); + + if (dbKey) { + gCerts.push(certdb.findCertByDBKey(dbKey)); + } else { + gCerts.push(null); + } + + if (!gCerts[i]) { + missing.push(params.recipients[i]); + } + } + + for (let i = 0; i < params.recipients.length; ++i) { + let email = document.createXULElement("label"); + email.setAttribute("value", params.recipients[i]); + email.setAttribute("crop", "end"); + email.setAttribute("style", "width: var(--recipientWidth)"); + + let listitem = document.createXULElement("richlistitem"); + listitem.appendChild(email); + + let cert = gCerts[i]; + let statusItem = document.createXULElement("label"); + statusItem.setAttribute( + "value", + gBundle.getString(cert ? "StatusValid" : "StatusNotFound") + ); + statusItem.setAttribute("style", "width: var(--statusWidth)"); + listitem.appendChild(statusItem); + + gListBox.appendChild(listitem); + } + resizeColumns(); +} + +function resizeColumns() { + let list = document.getElementById("infolist"); + let cols = list.getElementsByTagName("treecol"); + list.style.setProperty( + "--recipientWidth", + cols[0].getBoundingClientRect().width + "px" + ); + list.style.setProperty( + "--statusWidth", + cols[1].getBoundingClientRect().width + "px" + ); + list.style.setProperty( + "--issuedWidth", + cols[2].getBoundingClientRect().width + "px" + ); + list.style.setProperty( + "--expireWidth", + cols[3].getBoundingClientRect().width - 5 + "px" + ); +} + +// --- borrowed from pippki.js --- +const PRErrorCodeSuccess = 0; + +const certificateUsageEmailSigner = 0x0010; +const certificateUsageEmailRecipient = 0x0020; + +// A map from the name of a certificate usage to the value of the usage. +const certificateUsages = { + certificateUsageEmailRecipient, +}; + +function onSelectionChange(event) { + gViewButton.disabled = !( + gListBox.selectedItems.length == 1 && certForRow(gListBox.selectedIndex) + ); +} + +function viewCertHelper(parent, cert) { + let url = `about:certificate?cert=${encodeURIComponent( + cert.getBase64DERString() + )}`; + let mail3PaneWindow = Services.wm.getMostRecentWindow("mail:3pane"); + mail3PaneWindow.switchToTabHavingURI(url, true, {}); + parent.close(); +} + +function certForRow(aRowIndex) { + return gCerts[aRowIndex]; +} + +function viewSelectedCert() { + if (!gViewButton.disabled) { + viewCertHelper(window, certForRow(gListBox.selectedIndex)); + } +} diff --git a/comm/mailnews/extensions/smime/msgCompSecurityInfo.xhtml b/comm/mailnews/extensions/smime/msgCompSecurityInfo.xhtml new file mode 100644 index 0000000000..982d76818e --- /dev/null +++ b/comm/mailnews/extensions/smime/msgCompSecurityInfo.xhtml @@ -0,0 +1,69 @@ +<?xml version="1.0"?> +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> + +<?xml-stylesheet href="chrome://global/skin/global.css" type="text/css"?> +<?xml-stylesheet href="chrome://messenger/skin/smime/msgCompSecurityInfo.css" type="text/css"?> +<?xml-stylesheet href="chrome://messenger/skin/colors.css" type="text/css"?> +<?xml-stylesheet href="chrome://messenger/skin/variables.css" type="text/css"?> +<?xml-stylesheet href="chrome://messenger/skin/themeableDialog.css" type="text/css"?> + +<!DOCTYPE html SYSTEM "chrome://messenger-smime/locale/msgCompSecurityInfo.dtd"> +<html + id="msgCompSecurityInfo" + xmlns="http://www.w3.org/1999/xhtml" + xmlns:xul="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul" + xmlns:html="http://www.w3.org/1999/xhtml" + lightweightthemes="true" + scrolling="false" +> + <head> + <title>&title.label;</title> + <link rel="localization" href="branding/brand.ftl" /> + <script + defer="defer" + src="chrome://messenger/content/dialogShadowDom.js" + ></script> + <script + defer="defer" + src="chrome://messenger-smime/content/msgCompSecurityInfo.js" + ></script> + </head> + <html:body + xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul" + > + <dialog buttons="accept"> + <stringbundle + id="bundle_smime_comp_info" + src="chrome://messenger-smime/locale/msgCompSecurityInfo.properties" + /> + + <separator class="thin" /> + <label value="&status.certificates;" control="infolist" /> + + <richlistbox + id="infolist" + class="theme-listbox" + flex="1" + onselect="onSelectionChange(event);" + > + <treecols> + <treecol id="recipientHeader" label="&tree.recipient;" /> + <treecol id="statusHeader" label="&tree.status;" /> + <treecol id="issuedDateHeader" label="&tree.issuedDate;" /> + <treecol id="expiresDateHeader" label="&tree.expiresDate;" /> + </treecols> + </richlistbox> + <hbox pack="start"> + <button + id="viewCertButton" + disabled="true" + label="&view.label;" + accesskey="&view.accesskey;" + oncommand="viewSelectedCert();" + /> + </hbox> + </dialog> + </html:body> +</html> diff --git a/comm/mailnews/extensions/smime/msgReadSMIMEOverlay.js b/comm/mailnews/extensions/smime/msgReadSMIMEOverlay.js new file mode 100644 index 0000000000..bd7672bbe8 --- /dev/null +++ b/comm/mailnews/extensions/smime/msgReadSMIMEOverlay.js @@ -0,0 +1,251 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* import-globals-from ../../../mail/base/content/aboutMessage.js */ + +var gEncryptionStatus = -1; +var gSignatureStatus = -1; +var gSignerCert = null; +var gEncryptionCert = null; + +function showImapSignatureUnknown() { + let readSmimeBundle = Services.strings.createBundle( + "chrome://messenger-smime/locale/msgReadSMIMEOverlay.properties" + ); + let brandBundle = document.getElementById("bundle_brand"); + if (!readSmimeBundle || !brandBundle) { + return; + } + + if ( + Services.prompt.confirm( + window, + brandBundle.getString("brandShortName"), + readSmimeBundle.GetStringFromName("ImapOnDemand") + ) + ) { + gDBView.reloadMessageWithAllParts(); + } +} + +/** + * Populate the message security popup panel with S/MIME data. + */ +function loadSmimeMessageSecurityInfo() { + let sBundle = Services.strings.createBundle( + "chrome://messenger-smime/locale/msgSecurityInfo.properties" + ); + + let sigInfoLabel = null; + let sigInfoHeader = null; + let sigInfo = null; + let sigInfo_clueless = false; + let sigClass = null; + + switch (gSignatureStatus) { + case -1: + case Ci.nsICMSMessageErrors.VERIFY_NOT_SIGNED: + sigInfoLabel = "SINoneLabel"; + sigInfo = "SINone"; + sigClass = "none"; + break; + + case Ci.nsICMSMessageErrors.SUCCESS: + sigInfoLabel = "SIValidLabel"; + sigInfo = "SIValid"; + sigClass = "ok"; + break; + + case Ci.nsICMSMessageErrors.VERIFY_BAD_SIGNATURE: + case Ci.nsICMSMessageErrors.VERIFY_DIGEST_MISMATCH: + sigInfoLabel = "SIInvalidLabel"; + sigInfoHeader = "SIInvalidHeader"; + sigInfo = "SIContentAltered"; + sigClass = "mismatch"; + break; + + case Ci.nsICMSMessageErrors.VERIFY_UNKNOWN_ALGO: + case Ci.nsICMSMessageErrors.VERIFY_UNSUPPORTED_ALGO: + sigInfoLabel = "SIInvalidLabel"; + sigInfoHeader = "SIInvalidHeader"; + sigInfo = "SIInvalidCipher"; + sigClass = "unknown"; + break; + + case Ci.nsICMSMessageErrors.VERIFY_HEADER_MISMATCH: + sigInfoLabel = "SIPartiallyValidLabel"; + sigInfoHeader = "SIPartiallyValidHeader"; + sigInfo = "SIHeaderMismatch"; + sigClass = "mismatch"; + break; + + case Ci.nsICMSMessageErrors.VERIFY_CERT_WITHOUT_ADDRESS: + sigInfoLabel = "SIPartiallyValidLabel"; + sigInfoHeader = "SIPartiallyValidHeader"; + sigInfo = "SICertWithoutAddress"; + sigClass = "unknown"; + break; + + case Ci.nsICMSMessageErrors.VERIFY_UNTRUSTED: + sigInfoLabel = "SIInvalidLabel"; + sigInfoHeader = "SIInvalidHeader"; + sigInfo = "SIUntrustedCA"; + sigClass = "notok"; + // XXX Need to extend to communicate better errors + // might also be: + // SIExpired SIRevoked SINotYetValid SIUnknownCA SIExpiredCA SIRevokedCA SINotYetValidCA + break; + + case Ci.nsICMSMessageErrors.VERIFY_NOT_YET_ATTEMPTED: + case Ci.nsICMSMessageErrors.GENERAL_ERROR: + case Ci.nsICMSMessageErrors.VERIFY_NO_CONTENT_INFO: + case Ci.nsICMSMessageErrors.VERIFY_BAD_DIGEST: + case Ci.nsICMSMessageErrors.VERIFY_NOCERT: + case Ci.nsICMSMessageErrors.VERIFY_ERROR_UNVERIFIED: + case Ci.nsICMSMessageErrors.VERIFY_ERROR_PROCESSING: + case Ci.nsICMSMessageErrors.VERIFY_MALFORMED_SIGNATURE: + case Ci.nsICMSMessageErrors.VERIFY_TIME_MISMATCH: + sigInfoLabel = "SIInvalidLabel"; + sigInfoHeader = "SIInvalidHeader"; + sigInfo_clueless = true; + sigClass = "unverified"; + break; + default: + console.error("Unexpected gSignatureStatus: " + gSignatureStatus); + } + + document.getElementById("techLabel").textContent = "- S/MIME"; + + let signatureLabel = document.getElementById("signatureLabel"); + signatureLabel.textContent = sBundle.GetStringFromName(sigInfoLabel); + + // Remove the second class to properly update the signature icon. + signatureLabel.classList.remove(signatureLabel.classList.item(1)); + signatureLabel.classList.add(sigClass); + + if (sigInfoHeader) { + let label = document.getElementById("signatureHeader"); + label.collapsed = false; + label.textContent = sBundle.GetStringFromName(sigInfoHeader); + } + + let str; + if (sigInfo) { + str = sBundle.GetStringFromName(sigInfo); + } else if (sigInfo_clueless) { + str = + sBundle.GetStringFromName("SIClueless") + " (" + gSignatureStatus + ")"; + } + document.getElementById("signatureExplanation").textContent = str; + + let encInfoLabel = null; + let encInfoHeader = null; + let encInfo = null; + let encInfo_clueless = false; + let encClass = null; + + switch (gEncryptionStatus) { + case -1: + encInfoLabel = "EINoneLabel2"; + encInfo = "EINone"; + encClass = "none"; + break; + + case Ci.nsICMSMessageErrors.SUCCESS: + encInfoLabel = "EIValidLabel"; + encInfo = "EIValid"; + encClass = "ok"; + break; + + case Ci.nsICMSMessageErrors.ENCRYPT_INCOMPLETE: + encInfoLabel = "EIInvalidLabel"; + encInfo = "EIContentAltered"; + encClass = "notok"; + break; + + case Ci.nsICMSMessageErrors.GENERAL_ERROR: + encInfoLabel = "EIInvalidLabel"; + encInfoHeader = "EIInvalidHeader"; + encInfo_clueless = 1; + encClass = "notok"; + break; + default: + console.error("Unexpected gEncryptionStatus: " + gEncryptionStatus); + } + + let encryptionLabel = document.getElementById("encryptionLabel"); + encryptionLabel.textContent = sBundle.GetStringFromName(encInfoLabel); + + // Remove the second class to properly update the encryption icon. + encryptionLabel.classList.remove(encryptionLabel.classList.item(1)); + encryptionLabel.classList.add(encClass); + + if (encInfoHeader) { + let label = document.getElementById("encryptionHeader"); + label.collapsed = false; + label.textContent = sBundle.GetStringFromName(encInfoHeader); + } + + if (encInfo) { + str = sBundle.GetStringFromName(encInfo); + } else if (encInfo_clueless) { + str = sBundle.GetStringFromName("EIClueless"); + } + document.getElementById("encryptionExplanation").textContent = str; + + if (gSignerCert) { + document.getElementById("signatureCert").collapsed = false; + if (gSignerCert.subjectName) { + document.getElementById("signedBy").textContent = gSignerCert.commonName; + } + if (gSignerCert.emailAddress) { + document.getElementById("signerEmail").textContent = + gSignerCert.emailAddress; + } + if (gSignerCert.issuerName) { + document.getElementById("sigCertIssuedBy").textContent = + gSignerCert.issuerCommonName; + } + } + + if (gEncryptionCert) { + document.getElementById("encryptionCert").collapsed = false; + if (gEncryptionCert.subjectName) { + document.getElementById("encryptedFor").textContent = + gEncryptionCert.commonName; + } + if (gEncryptionCert.emailAddress) { + document.getElementById("recipientEmail").textContent = + gEncryptionCert.emailAddress; + } + if (gEncryptionCert.issuerName) { + document.getElementById("encCertIssuedBy").textContent = + gEncryptionCert.issuerCommonName; + } + } +} + +function viewSignatureCert() { + if (!gSignerCert) { + return; + } + + let url = `about:certificate?cert=${encodeURIComponent( + gSignerCert.getBase64DERString() + )}`; + let mail3PaneWindow = Services.wm.getMostRecentWindow("mail:3pane"); + mail3PaneWindow.switchToTabHavingURI(url, true, {}); +} + +function viewEncryptionCert() { + if (!gEncryptionCert) { + return; + } + + let url = `about:certificate?cert=${encodeURIComponent( + gEncryptionCert.getBase64DERString() + )}`; + let mail3PaneWindow = Services.wm.getMostRecentWindow("mail:3pane"); + mail3PaneWindow.switchToTabHavingURI(url, true, {}); +} diff --git a/comm/mailnews/extensions/smime/nsCMS.cpp b/comm/mailnews/extensions/smime/nsCMS.cpp new file mode 100644 index 0000000000..be743b9663 --- /dev/null +++ b/comm/mailnews/extensions/smime/nsCMS.cpp @@ -0,0 +1,1187 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsCMS.h" + +#include "CertVerifier.h" +#include "CryptoTask.h" +#include "ScopedNSSTypes.h" +#include "cms.h" +#include "mozilla/Logging.h" +#include "mozilla/RefPtr.h" +#include "nsDependentSubstring.h" +#include "nsICryptoHash.h" +#include "nsISupports.h" +#include "nsIX509CertDB.h" +#include "nsNSSCertificate.h" +#include "nsNSSComponent.h" +#include "nsNSSHelper.h" +#include "nsServiceManagerUtils.h" +#include "mozpkix/Result.h" +#include "mozpkix/pkixtypes.h" +#include "sechash.h" +#include "secerr.h" +#include "smime.h" +#include "mozilla/StaticMutex.h" +#include "nsIPrefBranch.h" + +using namespace mozilla; +using namespace mozilla::psm; +using namespace mozilla::pkix; + +static mozilla::LazyLogModule gCMSLog("CMS"); + +NS_IMPL_ISUPPORTS(nsCMSMessage, nsICMSMessage) + +nsCMSMessage::nsCMSMessage() { m_cmsMsg = nullptr; } +nsCMSMessage::nsCMSMessage(NSSCMSMessage* aCMSMsg) { m_cmsMsg = aCMSMsg; } + +nsCMSMessage::~nsCMSMessage() { + if (m_cmsMsg) { + NSS_CMSMessage_Destroy(m_cmsMsg); + } +} + +nsresult nsCMSMessage::Init() { + nsresult rv; + nsCOMPtr<nsISupports> nssInitialized = + do_GetService("@mozilla.org/psm;1", &rv); + return rv; +} + +NS_IMETHODIMP nsCMSMessage::VerifySignature(int32_t verifyFlags) { + return CommonVerifySignature(verifyFlags, {}, 0); +} + +NSSCMSSignerInfo* nsCMSMessage::GetTopLevelSignerInfo() { + if (!m_cmsMsg) return nullptr; + + if (!NSS_CMSMessage_IsSigned(m_cmsMsg)) return nullptr; + + NSSCMSContentInfo* cinfo = NSS_CMSMessage_ContentLevel(m_cmsMsg, 0); + if (!cinfo) return nullptr; + + NSSCMSSignedData* sigd = + (NSSCMSSignedData*)NSS_CMSContentInfo_GetContent(cinfo); + if (!sigd) return nullptr; + + PR_ASSERT(NSS_CMSSignedData_SignerInfoCount(sigd) > 0); + return NSS_CMSSignedData_GetSignerInfo(sigd, 0); +} + +NS_IMETHODIMP nsCMSMessage::GetSignerEmailAddress(char** aEmail) { + MOZ_LOG(gCMSLog, LogLevel::Debug, ("nsCMSMessage::GetSignerEmailAddress")); + NS_ENSURE_ARG(aEmail); + + NSSCMSSignerInfo* si = GetTopLevelSignerInfo(); + if (!si) return NS_ERROR_FAILURE; + + *aEmail = NSS_CMSSignerInfo_GetSignerEmailAddress(si); + return NS_OK; +} + +NS_IMETHODIMP nsCMSMessage::GetSignerCommonName(char** aName) { + MOZ_LOG(gCMSLog, LogLevel::Debug, ("nsCMSMessage::GetSignerCommonName")); + NS_ENSURE_ARG(aName); + + NSSCMSSignerInfo* si = GetTopLevelSignerInfo(); + if (!si) return NS_ERROR_FAILURE; + + *aName = NSS_CMSSignerInfo_GetSignerCommonName(si); + return NS_OK; +} + +NS_IMETHODIMP nsCMSMessage::ContentIsEncrypted(bool* isEncrypted) { + MOZ_LOG(gCMSLog, LogLevel::Debug, ("nsCMSMessage::ContentIsEncrypted")); + NS_ENSURE_ARG(isEncrypted); + + if (!m_cmsMsg) return NS_ERROR_FAILURE; + + *isEncrypted = NSS_CMSMessage_IsEncrypted(m_cmsMsg); + + return NS_OK; +} + +NS_IMETHODIMP nsCMSMessage::ContentIsSigned(bool* isSigned) { + MOZ_LOG(gCMSLog, LogLevel::Debug, ("nsCMSMessage::ContentIsSigned")); + NS_ENSURE_ARG(isSigned); + + if (!m_cmsMsg) return NS_ERROR_FAILURE; + + *isSigned = NSS_CMSMessage_IsSigned(m_cmsMsg); + + return NS_OK; +} + +NS_IMETHODIMP nsCMSMessage::GetSignerCert(nsIX509Cert** scert) { + NSSCMSSignerInfo* si = GetTopLevelSignerInfo(); + if (!si) return NS_ERROR_FAILURE; + + nsCOMPtr<nsIX509Cert> cert; + if (si->cert) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::GetSignerCert got signer cert")); + + nsCOMPtr<nsIX509CertDB> certdb = do_GetService(NS_X509CERTDB_CONTRACTID); + nsTArray<uint8_t> certBytes; + certBytes.AppendElements(si->cert->derCert.data, si->cert->derCert.len); + nsresult rv = certdb->ConstructX509(certBytes, getter_AddRefs(cert)); + NS_ENSURE_SUCCESS(rv, rv); + } else { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::GetSignerCert no signer cert, do we have a cert " + "list? %s", + (si->certList ? "yes" : "no"))); + + *scert = nullptr; + } + + cert.forget(scert); + + return NS_OK; +} + +NS_IMETHODIMP nsCMSMessage::GetEncryptionCert(nsIX509Cert**) { + return NS_ERROR_NOT_IMPLEMENTED; +} + +NS_IMETHODIMP nsCMSMessage::GetSigningTime(PRTime* aTime) { + MOZ_LOG(gCMSLog, LogLevel::Debug, ("nsCMSMessage::GetSigningTime")); + NS_ENSURE_ARG(aTime); + + NSSCMSSignerInfo* si = GetTopLevelSignerInfo(); + if (!si) { + return NS_ERROR_FAILURE; + } + + SECStatus getSigningTimeResult = NSS_CMSSignerInfo_GetSigningTime(si, aTime); + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::GetSigningTime result: %s", + (getSigningTimeResult ? "ok" : "fail"))); + + return getSigningTimeResult == SECSuccess ? NS_OK : NS_ERROR_FAILURE; +} + +NS_IMETHODIMP +nsCMSMessage::VerifyDetachedSignature(int32_t verifyFlags, + const nsTArray<uint8_t>& aDigestData, + int16_t aDigestType) { + if (aDigestData.IsEmpty()) return NS_ERROR_FAILURE; + + return CommonVerifySignature(verifyFlags, aDigestData, aDigestType); +} + +// This is an exact copy of NSS_CMSArray_Count from NSS' cmsarray.c, +// temporarily necessary, see below for for justification. +static int myNSS_CMSArray_Count(void** array) { + int n = 0; + + if (array == NULL) return 0; + + while (*array++ != NULL) n++; + + return n; +} + +// This is an exact copy of NSS_CMSArray_Add from NSS' cmsarray.c, +// temporarily necessary, see below for for justification. +static SECStatus myNSS_CMSArray_Add(PLArenaPool* poolp, void*** array, + void* obj) { + void** p; + int n; + void** dest; + + PORT_Assert(array != NULL); + if (array == NULL) return SECFailure; + + if (*array == NULL) { + dest = (void**)PORT_ArenaAlloc(poolp, 2 * sizeof(void*)); + n = 0; + } else { + n = 0; + p = *array; + while (*p++) n++; + dest = (void**)PORT_ArenaGrow(poolp, *array, (n + 1) * sizeof(void*), + (n + 2) * sizeof(void*)); + } + + if (dest == NULL) return SECFailure; + + dest[n] = obj; + dest[n + 1] = NULL; + *array = dest; + return SECSuccess; +} + +// This is an exact copy of NSS_CMSArray_Add from NSS' cmsarray.c, +// temporarily necessary, see below for for justification. +static SECStatus myNSS_CMSSignedData_AddTempCertificate(NSSCMSSignedData* sigd, + CERTCertificate* cert) { + CERTCertificate* c; + SECStatus rv; + + if (!sigd || !cert) { + PORT_SetError(SEC_ERROR_INVALID_ARGS); + return SECFailure; + } + + c = CERT_DupCertificate(cert); + rv = myNSS_CMSArray_Add(sigd->cmsg->poolp, (void***)&(sigd->tempCerts), + (void*)c); + return rv; +} + +typedef SECStatus (*extraVerificationOnCertFn)(CERTCertificate* cert, + SECCertUsage certusage); + +static SECStatus myExtraVerificationOnCert(CERTCertificate* cert, + SECCertUsage certusage) { + RefPtr<SharedCertVerifier> certVerifier; + certVerifier = GetDefaultCertVerifier(); + if (!certVerifier) { + return SECFailure; + } + + SECCertificateUsage usageForPkix; + + switch (certusage) { + case certUsageEmailSigner: + usageForPkix = certificateUsageEmailSigner; + break; + case certUsageEmailRecipient: + usageForPkix = certificateUsageEmailRecipient; + break; + default: + return SECFailure; + } + + nsTArray<uint8_t> certBytes(cert->derCert.data, cert->derCert.len); + nsTArray<nsTArray<uint8_t>> builtChain; + // This code is used when verifying incoming certificates, including + // a signature certificate. Performing OCSP is necessary. + // Allowing OCSP in blocking mode should be fine, because all our + // callers run this code on a separate thread, using + // SMimeVerificationTask/CryptoTask. + mozilla::pkix::Result result = certVerifier->VerifyCert( + certBytes, usageForPkix, Now(), nullptr /*XXX pinarg*/, + nullptr /*hostname*/, builtChain); + if (result != mozilla::pkix::Success) { + return SECFailure; + } + + return SECSuccess; +} + +// This is a temporary copy of NSS_CMSSignedData_ImportCerts, which +// performs additional verifications prior to import. +// The copy is almost identical to the original. +// +// The ONLY DIFFERENCE is the addition of parameter extraVerifyFn, +// and the call to it - plus a non-null check. +// +// NSS should add this or a similar API in the future, +// and then these temporary functions should be removed, including +// the ones above. Request is tracked in bugzilla 1738592. +static SECStatus myNSS_CMSSignedData_ImportCerts( + NSSCMSSignedData* sigd, CERTCertDBHandle* certdb, SECCertUsage certusage, + PRBool keepcerts, extraVerificationOnCertFn extraVerifyFn) { + int certcount; + CERTCertificate** certArray = NULL; + CERTCertList* certList = NULL; + CERTCertListNode* node; + SECStatus rv; + SECItem** rawArray; + int i; + PRTime now; + + if (!sigd) { + PORT_SetError(SEC_ERROR_INVALID_ARGS); + return SECFailure; + } + + certcount = myNSS_CMSArray_Count((void**)sigd->rawCerts); + + /* get the certs in the temp DB */ + rv = CERT_ImportCerts(certdb, certusage, certcount, sigd->rawCerts, + &certArray, PR_FALSE, PR_FALSE, NULL); + if (rv != SECSuccess) { + goto loser; + } + + /* save the certs so they don't get destroyed */ + for (i = 0; i < certcount; i++) { + CERTCertificate* cert = certArray[i]; + if (cert) myNSS_CMSSignedData_AddTempCertificate(sigd, cert); + } + + if (!keepcerts) { + goto done; + } + + /* build a CertList for filtering */ + certList = CERT_NewCertList(); + if (certList == NULL) { + rv = SECFailure; + goto loser; + } + for (i = 0; i < certcount; i++) { + CERTCertificate* cert = certArray[i]; + if (cert) cert = CERT_DupCertificate(cert); + if (cert) CERT_AddCertToListTail(certList, cert); + } + + /* filter out the certs we don't want */ + rv = CERT_FilterCertListByUsage(certList, certusage, PR_FALSE); + if (rv != SECSuccess) { + goto loser; + } + + /* go down the remaining list of certs and verify that they have + * valid chains, then import them. + */ + now = PR_Now(); + for (node = CERT_LIST_HEAD(certList); !CERT_LIST_END(node, certList); + node = CERT_LIST_NEXT(node)) { + CERTCertificateList* certChain; + + if (!node->cert) { + continue; + } + + if (extraVerifyFn) { + if ((*extraVerifyFn)(node->cert, certusage) != SECSuccess) { + continue; + } + } + + if (CERT_VerifyCert(certdb, node->cert, PR_TRUE, certusage, now, NULL, + NULL) != SECSuccess) { + continue; + } + + certChain = CERT_CertChainFromCert(node->cert, certusage, PR_FALSE); + if (!certChain) { + continue; + } + + /* + * CertChain returns an array of SECItems, import expects an array of + * SECItem pointers. Create the SECItem Pointers from the array of + * SECItems. + */ + rawArray = (SECItem**)PORT_Alloc(certChain->len * sizeof(SECItem*)); + if (!rawArray) { + CERT_DestroyCertificateList(certChain); + continue; + } + for (i = 0; i < certChain->len; i++) { + rawArray[i] = &certChain->certs[i]; + } + (void)CERT_ImportCerts(certdb, certusage, certChain->len, rawArray, NULL, + keepcerts, PR_FALSE, NULL); + PORT_Free(rawArray); + CERT_DestroyCertificateList(certChain); + } + + rv = SECSuccess; + + /* XXX CRL handling */ + +done: + if (sigd->signerInfos != NULL) { + /* fill in all signerinfo's certs */ + for (i = 0; sigd->signerInfos[i] != NULL; i++) + (void)NSS_CMSSignerInfo_GetSigningCertificate(sigd->signerInfos[i], + certdb); + } + +loser: + /* now free everything */ + if (certArray) { + CERT_DestroyCertArray(certArray, certcount); + } + if (certList) { + CERT_DestroyCertList(certList); + } + + return rv; +} + +nsresult nsCMSMessage::CommonVerifySignature( + int32_t verifyFlags, const nsTArray<uint8_t>& aDigestData, + int16_t aDigestType) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CommonVerifySignature, content level count %d", + NSS_CMSMessage_ContentLevelCount(m_cmsMsg))); + NSSCMSContentInfo* cinfo = nullptr; + NSSCMSSignedData* sigd = nullptr; + NSSCMSSignerInfo* si; + int32_t nsigners; + nsresult rv = NS_ERROR_FAILURE; + SECOidTag sigAlgTag; + + if (!NSS_CMSMessage_IsSigned(m_cmsMsg)) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CommonVerifySignature - not signed")); + return NS_ERROR_CMS_VERIFY_NOT_SIGNED; + } + + cinfo = NSS_CMSMessage_ContentLevel(m_cmsMsg, 0); + if (cinfo) { + switch (NSS_CMSContentInfo_GetContentTypeTag(cinfo)) { + case SEC_OID_PKCS7_SIGNED_DATA: + sigd = reinterpret_cast<NSSCMSSignedData*>( + NSS_CMSContentInfo_GetContent(cinfo)); + break; + + case SEC_OID_PKCS7_ENVELOPED_DATA: + case SEC_OID_PKCS7_ENCRYPTED_DATA: + case SEC_OID_PKCS7_DIGESTED_DATA: + default: { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CommonVerifySignature - unexpected " + "ContentTypeTag")); + rv = NS_ERROR_CMS_VERIFY_NO_CONTENT_INFO; + goto loser; + } + } + } + + if (!sigd) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CommonVerifySignature - no content info")); + rv = NS_ERROR_CMS_VERIFY_NO_CONTENT_INFO; + goto loser; + } + + if (!aDigestData.IsEmpty()) { + SECOidTag oidTag; + SECItem digest; + // NSS_CMSSignedData_SetDigestValue() takes a copy and won't mutate our + // data, so we're OK to cast away the const here. + digest.data = const_cast<uint8_t*>(aDigestData.Elements()); + digest.len = aDigestData.Length(); + + if (NSS_CMSSignedData_HasDigests(sigd)) { + SECAlgorithmID** existingAlgs = NSS_CMSSignedData_GetDigestAlgs(sigd); + if (existingAlgs) { + while (*existingAlgs) { + SECAlgorithmID* alg = *existingAlgs; + SECOidTag algOIDTag = SECOID_FindOIDTag(&alg->algorithm); + NSS_CMSSignedData_SetDigestValue(sigd, algOIDTag, NULL); + ++existingAlgs; + } + } + } + + oidTag = + HASH_GetHashOidTagByHashType(static_cast<HASH_HashType>(aDigestType)); + if (oidTag == SEC_OID_UNKNOWN) { + rv = NS_ERROR_CMS_VERIFY_BAD_DIGEST; + goto loser; + } + + if (NSS_CMSSignedData_SetDigestValue(sigd, oidTag, &digest)) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CommonVerifySignature - bad digest")); + rv = NS_ERROR_CMS_VERIFY_BAD_DIGEST; + goto loser; + } + } + + // Import certs. Note that import failure is not a signature verification + // failure. // + if (myNSS_CMSSignedData_ImportCerts( + sigd, CERT_GetDefaultCertDB(), certUsageEmailRecipient, true, + myExtraVerificationOnCert) != SECSuccess) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CommonVerifySignature - can not import certs")); + } + + nsigners = NSS_CMSSignedData_SignerInfoCount(sigd); + PR_ASSERT(nsigners > 0); + NS_ENSURE_TRUE(nsigners > 0, NS_ERROR_UNEXPECTED); + si = NSS_CMSSignedData_GetSignerInfo(sigd, 0); + + NS_ENSURE_TRUE(si, NS_ERROR_UNEXPECTED); + NS_ENSURE_TRUE(si->cert, NS_ERROR_UNEXPECTED); + + // See bug 324474. We want to make sure the signing cert is + // still valid at the current time. + + if (myExtraVerificationOnCert(si->cert, certUsageEmailSigner) != SECSuccess) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CommonVerifySignature - signing cert not trusted " + "now")); + rv = NS_ERROR_CMS_VERIFY_UNTRUSTED; + goto loser; + } + + sigAlgTag = NSS_CMSSignerInfo_GetDigestAlgTag(si); + switch (sigAlgTag) { + case SEC_OID_SHA256: + case SEC_OID_SHA384: + case SEC_OID_SHA512: + break; + + case SEC_OID_SHA1: + if (verifyFlags & nsICMSVerifyFlags::VERIFY_ALLOW_WEAK_SHA1) { + break; + } + // else fall through to failure +#if defined(__clang__) + [[clang::fallthrough]]; +#endif + + default: + MOZ_LOG( + gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CommonVerifySignature - unsupported digest algo")); + rv = NS_ERROR_CMS_VERIFY_UNSUPPORTED_ALGO; + goto loser; + }; + + // We verify the first signer info, only // + // XXX: NSS_CMSSignedData_VerifySignerInfo calls CERT_VerifyCert, which + // requires NSS's certificate verification configuration to be done in + // order to work well (e.g. honoring OCSP preferences and proxy settings + // for OCSP requests), but Gecko stopped doing that configuration. Something + // similar to what was done for Gecko bug 1028643 needs to be done here too. + if (NSS_CMSSignedData_VerifySignerInfo(sigd, 0, CERT_GetDefaultCertDB(), + certUsageEmailSigner) != SECSuccess) { + MOZ_LOG( + gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CommonVerifySignature - unable to verify signature")); + + if (NSSCMSVS_SigningCertNotFound == si->verificationStatus) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CommonVerifySignature - signing cert not found")); + rv = NS_ERROR_CMS_VERIFY_NOCERT; + } else if (NSSCMSVS_SigningCertNotTrusted == si->verificationStatus) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CommonVerifySignature - signing cert not trusted " + "at signing time")); + rv = NS_ERROR_CMS_VERIFY_UNTRUSTED; + } else if (NSSCMSVS_Unverified == si->verificationStatus) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CommonVerifySignature - can not verify")); + rv = NS_ERROR_CMS_VERIFY_ERROR_UNVERIFIED; + } else if (NSSCMSVS_ProcessingError == si->verificationStatus) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CommonVerifySignature - processing error")); + rv = NS_ERROR_CMS_VERIFY_ERROR_PROCESSING; + } else if (NSSCMSVS_BadSignature == si->verificationStatus) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CommonVerifySignature - bad signature")); + rv = NS_ERROR_CMS_VERIFY_BAD_SIGNATURE; + } else if (NSSCMSVS_DigestMismatch == si->verificationStatus) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CommonVerifySignature - digest mismatch")); + rv = NS_ERROR_CMS_VERIFY_DIGEST_MISMATCH; + } else if (NSSCMSVS_SignatureAlgorithmUnknown == si->verificationStatus) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CommonVerifySignature - algo unknown")); + rv = NS_ERROR_CMS_VERIFY_UNKNOWN_ALGO; + } else if (NSSCMSVS_SignatureAlgorithmUnsupported == + si->verificationStatus) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CommonVerifySignature - algo not supported")); + rv = NS_ERROR_CMS_VERIFY_UNSUPPORTED_ALGO; + } else if (NSSCMSVS_MalformedSignature == si->verificationStatus) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CommonVerifySignature - malformed signature")); + rv = NS_ERROR_CMS_VERIFY_MALFORMED_SIGNATURE; + } + + goto loser; + } + + // Save the profile. Note that save import failure is not a signature + // verification failure. // + if (NSS_SMIMESignerInfo_SaveSMIMEProfile(si) != SECSuccess) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CommonVerifySignature - unable to save smime " + "profile")); + } + + rv = NS_OK; +loser: + return rv; +} + +NS_IMETHODIMP nsCMSMessage::AsyncVerifySignature( + int32_t verifyFlags, nsISMimeVerificationListener* aListener) { + return CommonAsyncVerifySignature(verifyFlags, aListener, {}, 0); +} + +NS_IMETHODIMP nsCMSMessage::AsyncVerifyDetachedSignature( + int32_t verifyFlags, nsISMimeVerificationListener* aListener, + const nsTArray<uint8_t>& aDigestData, int16_t aDigestType) { + if (aDigestData.IsEmpty()) return NS_ERROR_FAILURE; + + return CommonAsyncVerifySignature(verifyFlags, aListener, aDigestData, + aDigestType); +} + +class SMimeVerificationTask final : public CryptoTask { + public: + SMimeVerificationTask(nsICMSMessage* aMessage, int32_t verifyFlags, + nsISMimeVerificationListener* aListener, + const nsTArray<uint8_t>& aDigestData, + int16_t aDigestType) + : mMessage(aMessage), + mListener(aListener), + mDigestData(aDigestData.Clone()), + mDigestType(aDigestType), + mVerifyFlags(verifyFlags) { + MOZ_ASSERT(NS_IsMainThread()); + } + + private: + virtual nsresult CalculateResult() override { + MOZ_ASSERT(!NS_IsMainThread()); + + // Because the S/MIME code and related certificate processing isn't + // sufficiently threadsafe (see bug 1529003), we want this code to + // never run in parallel (see bug 1386601). + mozilla::StaticMutexAutoLock lock(sMutex); + nsresult rv; + if (mDigestData.IsEmpty()) { + rv = mMessage->VerifySignature(mVerifyFlags); + } else { + rv = mMessage->VerifyDetachedSignature(mVerifyFlags, mDigestData, + mDigestType); + } + + return rv; + } + virtual void CallCallback(nsresult rv) override { + MOZ_ASSERT(NS_IsMainThread()); + mListener->Notify(mMessage, rv); + } + + nsCOMPtr<nsICMSMessage> mMessage; + nsCOMPtr<nsISMimeVerificationListener> mListener; + nsTArray<uint8_t> mDigestData; + int16_t mDigestType; + int32_t mVerifyFlags; + + static mozilla::StaticMutex sMutex; +}; + +mozilla::StaticMutex SMimeVerificationTask::sMutex; + +nsresult nsCMSMessage::CommonAsyncVerifySignature( + int32_t verifyFlags, nsISMimeVerificationListener* aListener, + const nsTArray<uint8_t>& aDigestData, int16_t aDigestType) { + RefPtr<CryptoTask> task = new SMimeVerificationTask( + this, verifyFlags, aListener, aDigestData, aDigestType); + return task->Dispatch(); +} + +class nsZeroTerminatedCertArray { + public: + nsZeroTerminatedCertArray() : mCerts(nullptr), mPoolp(nullptr), mSize(0) {} + + ~nsZeroTerminatedCertArray() { + if (mCerts) { + for (uint32_t i = 0; i < mSize; i++) { + if (mCerts[i]) { + CERT_DestroyCertificate(mCerts[i]); + } + } + } + + if (mPoolp) PORT_FreeArena(mPoolp, false); + } + + bool allocate(uint32_t count) { + // only allow allocation once + if (mPoolp) return false; + + mSize = count; + + if (!mSize) return false; + + mPoolp = PORT_NewArena(1024); + if (!mPoolp) return false; + + mCerts = (CERTCertificate**)PORT_ArenaZAlloc( + mPoolp, (count + 1) * sizeof(CERTCertificate*)); + + if (!mCerts) return false; + + // null array, including zero termination + for (uint32_t i = 0; i < count + 1; i++) { + mCerts[i] = nullptr; + } + + return true; + } + + void set(uint32_t i, CERTCertificate* c) { + if (i >= mSize) return; + + if (mCerts[i]) { + CERT_DestroyCertificate(mCerts[i]); + } + + mCerts[i] = CERT_DupCertificate(c); + } + + CERTCertificate* get(uint32_t i) { + if (i >= mSize) return nullptr; + + return CERT_DupCertificate(mCerts[i]); + } + + CERTCertificate** getRawArray() { return mCerts; } + + private: + CERTCertificate** mCerts; + PLArenaPool* mPoolp; + uint32_t mSize; +}; + +NS_IMETHODIMP nsCMSMessage::CreateEncrypted( + const nsTArray<RefPtr<nsIX509Cert>>& aRecipientCerts) { + MOZ_LOG(gCMSLog, LogLevel::Debug, ("nsCMSMessage::CreateEncrypted")); + NSSCMSContentInfo* cinfo; + NSSCMSEnvelopedData* envd; + NSSCMSRecipientInfo* recipientInfo; + nsZeroTerminatedCertArray recipientCerts; + SECOidTag bulkAlgTag; + int keySize; + uint32_t i; + nsresult rv = NS_ERROR_FAILURE; + + // Check the recipient certificates // + uint32_t recipientCertCount = aRecipientCerts.Length(); + PR_ASSERT(recipientCertCount > 0); + + if (!recipientCerts.allocate(recipientCertCount)) { + goto loser; + } + + for (i = 0; i < recipientCertCount; i++) { + nsIX509Cert* x509cert = aRecipientCerts[i]; + + if (!x509cert) return NS_ERROR_FAILURE; + + UniqueCERTCertificate c(x509cert->GetCert()); + recipientCerts.set(i, c.get()); + } + + // Find a bulk key algorithm // + if (NSS_SMIMEUtil_FindBulkAlgForRecipients( + recipientCerts.getRawArray(), &bulkAlgTag, &keySize) != SECSuccess) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CreateEncrypted - can't find bulk alg for " + "recipients")); + rv = NS_ERROR_CMS_ENCRYPT_NO_BULK_ALG; + goto loser; + } + + m_cmsMsg = NSS_CMSMessage_Create(nullptr); + if (!m_cmsMsg) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CreateEncrypted - can't create new cms message")); + rv = NS_ERROR_OUT_OF_MEMORY; + goto loser; + } + + if ((envd = NSS_CMSEnvelopedData_Create(m_cmsMsg, bulkAlgTag, keySize)) == + nullptr) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CreateEncrypted - can't create enveloped data")); + goto loser; + } + + cinfo = NSS_CMSMessage_GetContentInfo(m_cmsMsg); + if (NSS_CMSContentInfo_SetContent_EnvelopedData(m_cmsMsg, cinfo, envd) != + SECSuccess) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CreateEncrypted - can't create content enveloped " + "data")); + goto loser; + } + + cinfo = NSS_CMSEnvelopedData_GetContentInfo(envd); + if (NSS_CMSContentInfo_SetContent_Data(m_cmsMsg, cinfo, nullptr, false) != + SECSuccess) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CreateEncrypted - can't set content data")); + goto loser; + } + + // Create and attach recipient information // + for (i = 0; i < recipientCertCount; i++) { + UniqueCERTCertificate rc(recipientCerts.get(i)); + if ((recipientInfo = NSS_CMSRecipientInfo_Create(m_cmsMsg, rc.get())) == + nullptr) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CreateEncrypted - can't create recipient info")); + goto loser; + } + if (NSS_CMSEnvelopedData_AddRecipient(envd, recipientInfo) != SECSuccess) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CreateEncrypted - can't add recipient info")); + goto loser; + } + } + + return NS_OK; +loser: + if (m_cmsMsg) { + NSS_CMSMessage_Destroy(m_cmsMsg); + m_cmsMsg = nullptr; + } + + return rv; +} + +bool nsCMSMessage::IsAllowedHash(const int16_t aCryptoHashInt) { + switch (aCryptoHashInt) { + case nsICryptoHash::SHA1: + case nsICryptoHash::SHA256: + case nsICryptoHash::SHA384: + case nsICryptoHash::SHA512: + return true; + default: + return false; + } +} + +NS_IMETHODIMP +nsCMSMessage::CreateSigned(nsIX509Cert* aSigningCert, nsIX509Cert* aEncryptCert, + const nsTArray<uint8_t>& aDigestData, + int16_t aDigestType) { + NS_ENSURE_ARG(aSigningCert); + MOZ_LOG(gCMSLog, LogLevel::Debug, ("nsCMSMessage::CreateSigned")); + NSSCMSContentInfo* cinfo; + NSSCMSSignedData* sigd; + NSSCMSSignerInfo* signerinfo; + UniqueCERTCertificate scert(aSigningCert->GetCert()); + UniqueCERTCertificate ecert; + nsresult rv = NS_ERROR_FAILURE; + + if (!scert) { + return NS_ERROR_FAILURE; + } + + if (aEncryptCert) { + ecert = UniqueCERTCertificate(aEncryptCert->GetCert()); + } + + if (!IsAllowedHash(aDigestType)) { + return NS_ERROR_INVALID_ARG; + } + + SECOidTag digestType = + HASH_GetHashOidTagByHashType(static_cast<HASH_HashType>(aDigestType)); + if (digestType == SEC_OID_UNKNOWN) { + return NS_ERROR_INVALID_ARG; + } + + /* + * create the message object + */ + m_cmsMsg = + NSS_CMSMessage_Create(nullptr); /* create a message on its own pool */ + if (!m_cmsMsg) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CreateSigned - can't create new message")); + rv = NS_ERROR_OUT_OF_MEMORY; + goto loser; + } + + /* + * build chain of objects: message->signedData->data + */ + if ((sigd = NSS_CMSSignedData_Create(m_cmsMsg)) == nullptr) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CreateSigned - can't create signed data")); + goto loser; + } + cinfo = NSS_CMSMessage_GetContentInfo(m_cmsMsg); + if (NSS_CMSContentInfo_SetContent_SignedData(m_cmsMsg, cinfo, sigd) != + SECSuccess) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CreateSigned - can't set content signed data")); + goto loser; + } + + cinfo = NSS_CMSSignedData_GetContentInfo(sigd); + + /* we're always passing data in and detaching optionally */ + if (NSS_CMSContentInfo_SetContent_Data(m_cmsMsg, cinfo, nullptr, true) != + SECSuccess) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CreateSigned - can't set content data")); + goto loser; + } + + /* + * create & attach signer information + */ + signerinfo = NSS_CMSSignerInfo_Create(m_cmsMsg, scert.get(), digestType); + if (!signerinfo) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CreateSigned - can't create signer info")); + goto loser; + } + + /* we want the cert chain included for this one */ + if (NSS_CMSSignerInfo_IncludeCerts(signerinfo, NSSCMSCM_CertChain, + certUsageEmailSigner) != SECSuccess) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CreateSigned - can't include signer cert chain")); + goto loser; + } + + if (NSS_CMSSignerInfo_AddSigningTime(signerinfo, PR_Now()) != SECSuccess) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CreateSigned - can't add signing time")); + goto loser; + } + + if (NSS_CMSSignerInfo_AddSMIMECaps(signerinfo) != SECSuccess) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CreateSigned - can't add smime caps")); + goto loser; + } + + if (ecert) { + if (NSS_CMSSignerInfo_AddSMIMEEncKeyPrefs( + signerinfo, ecert.get(), CERT_GetDefaultCertDB()) != SECSuccess) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CreateSigned - can't add smime enc key prefs")); + goto loser; + } + + if (NSS_CMSSignerInfo_AddMSSMIMEEncKeyPrefs( + signerinfo, ecert.get(), CERT_GetDefaultCertDB()) != SECSuccess) { + MOZ_LOG( + gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CreateSigned - can't add MS smime enc key prefs")); + goto loser; + } + + // If signing and encryption cert are identical, don't add it twice. + bool addEncryptionCert = + (ecert && (!scert || !CERT_CompareCerts(ecert.get(), scert.get()))); + + if (addEncryptionCert && + NSS_CMSSignedData_AddCertificate(sigd, ecert.get()) != SECSuccess) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CreateSigned - can't add own encryption " + "certificate")); + goto loser; + } + } + + if (NSS_CMSSignedData_AddSignerInfo(sigd, signerinfo) != SECSuccess) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CreateSigned - can't add signer info")); + goto loser; + } + + // Finally, add the pre-computed digest if passed in + if (!aDigestData.IsEmpty()) { + SECItem digest; + + // NSS_CMSSignedData_SetDigestValue() takes a copy and won't mutate our + // data, so we're OK to cast away the const here. + digest.data = const_cast<uint8_t*>(aDigestData.Elements()); + digest.len = aDigestData.Length(); + if (NSS_CMSSignedData_SetDigestValue(sigd, digestType, &digest) != + SECSuccess) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSMessage::CreateSigned - can't set digest value")); + goto loser; + } + } + + return NS_OK; +loser: + if (m_cmsMsg) { + NSS_CMSMessage_Destroy(m_cmsMsg); + m_cmsMsg = nullptr; + } + return rv; +} + +NS_IMPL_ISUPPORTS(nsCMSDecoder, nsICMSDecoder) +NS_IMPL_ISUPPORTS(nsCMSDecoderJS, nsICMSDecoderJS) + +nsCMSDecoder::nsCMSDecoder() : m_dcx(nullptr) {} +nsCMSDecoderJS::nsCMSDecoderJS() : m_dcx(nullptr) {} + +nsCMSDecoder::~nsCMSDecoder() { + if (m_dcx) { + NSS_CMSDecoder_Cancel(m_dcx); + m_dcx = nullptr; + } +} + +nsCMSDecoderJS::~nsCMSDecoderJS() { + if (m_dcx) { + NSS_CMSDecoder_Cancel(m_dcx); + m_dcx = nullptr; + } +} + +nsresult nsCMSDecoder::Init() { + nsresult rv; + nsCOMPtr<nsISupports> nssInitialized = + do_GetService("@mozilla.org/psm;1", &rv); + return rv; +} + +nsresult nsCMSDecoderJS::Init() { + nsresult rv; + nsCOMPtr<nsISupports> nssInitialized = + do_GetService("@mozilla.org/psm;1", &rv); + return rv; +} + +/* void start (in NSSCMSContentCallback cb, in voidPtr arg); */ +NS_IMETHODIMP nsCMSDecoder::Start(NSSCMSContentCallback cb, void* arg) { + MOZ_LOG(gCMSLog, LogLevel::Debug, ("nsCMSDecoder::Start")); + m_ctx = new PipUIContext(); + + m_dcx = NSS_CMSDecoder_Start(0, cb, arg, 0, m_ctx, 0, 0); + if (!m_dcx) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSDecoder::Start - can't start decoder")); + return NS_ERROR_FAILURE; + } + return NS_OK; +} + +/* void update (in string bug, in long len); */ +NS_IMETHODIMP nsCMSDecoder::Update(const char* buf, int32_t len) { + NSS_CMSDecoder_Update(m_dcx, (char*)buf, len); + return NS_OK; +} + +/* void finish (); */ +NS_IMETHODIMP nsCMSDecoder::Finish(nsICMSMessage** aCMSMsg) { + MOZ_LOG(gCMSLog, LogLevel::Debug, ("nsCMSDecoder::Finish")); + NSSCMSMessage* cmsMsg; + cmsMsg = NSS_CMSDecoder_Finish(m_dcx); + m_dcx = nullptr; + if (cmsMsg) { + nsCMSMessage* obj = new nsCMSMessage(cmsMsg); + // The NSS object cmsMsg still carries a reference to the context + // we gave it on construction. + // Make sure the context will live long enough. + obj->referenceContext(m_ctx); + NS_ADDREF(*aCMSMsg = obj); + } + return NS_OK; +} + +void nsCMSDecoderJS::content_callback(void* arg, const char* input, + unsigned long length) { + nsCMSDecoderJS* self = reinterpret_cast<nsCMSDecoderJS*>(arg); + self->mDecryptedData.AppendElements(input, length); +} + +NS_IMETHODIMP nsCMSDecoderJS::Decrypt(const nsTArray<uint8_t>& aInput, + nsTArray<uint8_t>& _retval) { + if (aInput.IsEmpty()) { + return NS_ERROR_FAILURE; + } + + m_ctx = new PipUIContext(); + + m_dcx = NSS_CMSDecoder_Start(0, nsCMSDecoderJS::content_callback, this, 0, + m_ctx, 0, 0); + if (!m_dcx) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSDecoderJS::Start - can't start decoder")); + return NS_ERROR_FAILURE; + } + + NSS_CMSDecoder_Update(m_dcx, (char*)aInput.Elements(), aInput.Length()); + + NSSCMSMessage* cmsMsg; + cmsMsg = NSS_CMSDecoder_Finish(m_dcx); + m_dcx = nullptr; + if (cmsMsg) { + nsCMSMessage* obj = new nsCMSMessage(cmsMsg); + // The NSS object cmsMsg still carries a reference to the context + // we gave it on construction. + // Make sure the context will live long enough. + obj->referenceContext(m_ctx); + mCMSMessage = obj; + } + + _retval = mDecryptedData.Clone(); + return NS_OK; +} + +NS_IMPL_ISUPPORTS(nsCMSEncoder, nsICMSEncoder) + +nsCMSEncoder::nsCMSEncoder() : m_ecx(nullptr) {} + +nsCMSEncoder::~nsCMSEncoder() { + if (m_ecx) NSS_CMSEncoder_Cancel(m_ecx); +} + +nsresult nsCMSEncoder::Init() { + nsresult rv; + nsCOMPtr<nsISupports> nssInitialized = + do_GetService("@mozilla.org/psm;1", &rv); + return rv; +} + +/* void start (); */ +NS_IMETHODIMP nsCMSEncoder::Start(nsICMSMessage* aMsg, NSSCMSContentCallback cb, + void* arg) { + MOZ_LOG(gCMSLog, LogLevel::Debug, ("nsCMSEncoder::Start")); + nsCMSMessage* cmsMsg = static_cast<nsCMSMessage*>(aMsg); + m_ctx = new PipUIContext(); + + m_ecx = NSS_CMSEncoder_Start(cmsMsg->getCMS(), cb, arg, 0, 0, 0, m_ctx, 0, 0, + 0, 0); + if (!m_ecx) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSEncoder::Start - can't start encoder")); + return NS_ERROR_FAILURE; + } + return NS_OK; +} + +/* void update (in string aBuf, in long aLen); */ +NS_IMETHODIMP nsCMSEncoder::Update(const char* aBuf, int32_t aLen) { + MOZ_LOG(gCMSLog, LogLevel::Debug, ("nsCMSEncoder::Update")); + if (!m_ecx || NSS_CMSEncoder_Update(m_ecx, aBuf, aLen) != SECSuccess) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSEncoder::Update - can't update encoder")); + return NS_ERROR_FAILURE; + } + return NS_OK; +} + +/* void finish (); */ +NS_IMETHODIMP nsCMSEncoder::Finish() { + nsresult rv = NS_OK; + MOZ_LOG(gCMSLog, LogLevel::Debug, ("nsCMSEncoder::Finish")); + if (!m_ecx || NSS_CMSEncoder_Finish(m_ecx) != SECSuccess) { + MOZ_LOG(gCMSLog, LogLevel::Debug, + ("nsCMSEncoder::Finish - can't finish encoder")); + rv = NS_ERROR_FAILURE; + } + m_ecx = nullptr; + return rv; +} + +/* void encode (in nsICMSMessage aMsg); */ +NS_IMETHODIMP nsCMSEncoder::Encode(nsICMSMessage* aMsg) { + MOZ_LOG(gCMSLog, LogLevel::Debug, ("nsCMSEncoder::Encode")); + return NS_ERROR_NOT_IMPLEMENTED; +} diff --git a/comm/mailnews/extensions/smime/nsCMS.h b/comm/mailnews/extensions/smime/nsCMS.h new file mode 100644 index 0000000000..14ed2cd07f --- /dev/null +++ b/comm/mailnews/extensions/smime/nsCMS.h @@ -0,0 +1,123 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef __NS_CMS_H__ +#define __NS_CMS_H__ + +#include "nsISupports.h" +#include "nsCOMPtr.h" +#include "nsIInterfaceRequestor.h" +#include "nsICMSMessage.h" +#include "nsICMSEncoder.h" +#include "nsICMSDecoder.h" +#include "nsICMSDecoderJS.h" +#include "sechash.h" +#include "cms.h" + +#define NS_CMSMESSAGE_CID \ + { \ + 0xa4557478, 0xae16, 0x11d5, { \ + 0xba, 0x4b, 0x00, 0x10, 0x83, 0x03, 0xb1, 0x17 \ + } \ + } + +class nsCMSMessage : public nsICMSMessage { + public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSICMSMESSAGE + + nsCMSMessage(); + explicit nsCMSMessage(NSSCMSMessage* aCMSMsg); + nsresult Init(); + + void referenceContext(nsIInterfaceRequestor* aContext) { m_ctx = aContext; } + NSSCMSMessage* getCMS() { return m_cmsMsg; } + + private: + virtual ~nsCMSMessage(); + nsCOMPtr<nsIInterfaceRequestor> m_ctx; + NSSCMSMessage* m_cmsMsg; + NSSCMSSignerInfo* GetTopLevelSignerInfo(); + nsresult CommonVerifySignature(int32_t verifyFlags, + const nsTArray<uint8_t>& aDigestData, + int16_t aDigestType); + + nsresult CommonAsyncVerifySignature(int32_t verifyFlags, + nsISMimeVerificationListener* aListener, + const nsTArray<uint8_t>& aDigestData, + int16_t aDigestType); + bool IsAllowedHash(const int16_t aCryptoHashInt); +}; + +// =============================================== +// nsCMSDecoder - implementation of nsICMSDecoder +// =============================================== + +#define NS_CMSDECODER_CID \ + { \ + 0x9dcef3a4, 0xa3bc, 0x11d5, { \ + 0xba, 0x47, 0x00, 0x10, 0x83, 0x03, 0xb1, 0x17 \ + } \ + } + +class nsCMSDecoder : public nsICMSDecoder { + public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSICMSDECODER + + nsCMSDecoder(); + nsresult Init(); + + private: + virtual ~nsCMSDecoder(); + nsCOMPtr<nsIInterfaceRequestor> m_ctx; + NSSCMSDecoderContext* m_dcx; +}; + +class nsCMSDecoderJS : public nsICMSDecoderJS { + public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSICMSDECODERJS + + nsCMSDecoderJS(); + nsresult Init(); + + private: + virtual ~nsCMSDecoderJS(); + nsCOMPtr<nsIInterfaceRequestor> m_ctx; + NSSCMSDecoderContext* m_dcx; + + nsTArray<uint8_t> mDecryptedData; + nsCOMPtr<nsICMSMessage> mCMSMessage; + + static void content_callback(void* arg, const char* input, + unsigned long length); +}; + +// =============================================== +// nsCMSEncoder - implementation of nsICMSEncoder +// =============================================== + +#define NS_CMSENCODER_CID \ + { \ + 0xa15789aa, 0x8903, 0x462b, { \ + 0x81, 0xe9, 0x4a, 0xa2, 0xcf, 0xf4, 0xd5, 0xcb \ + } \ + } +class nsCMSEncoder : public nsICMSEncoder { + public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSICMSENCODER + + nsCMSEncoder(); + nsresult Init(); + + private: + virtual ~nsCMSEncoder(); + nsCOMPtr<nsIInterfaceRequestor> m_ctx; + NSSCMSEncoderContext* m_ecx; +}; + +#endif diff --git a/comm/mailnews/extensions/smime/nsCMSSecureMessage.cpp b/comm/mailnews/extensions/smime/nsCMSSecureMessage.cpp new file mode 100644 index 0000000000..c2c220a2f4 --- /dev/null +++ b/comm/mailnews/extensions/smime/nsCMSSecureMessage.cpp @@ -0,0 +1,92 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsCMSSecureMessage.h" + +#include <string.h> + +#include "ScopedNSSTypes.h" +#include "SharedCertVerifier.h" +#include "cms.h" +#include "mozilla/Logging.h" +#include "mozilla/RefPtr.h" +#include "nsCOMPtr.h" +#include "nsDependentSubstring.h" +#include "nsIInterfaceRequestor.h" +#include "nsServiceManagerUtils.h" +#include "nsISupports.h" +#include "nsIX509Cert.h" +#include "nsIX509CertDB.h" +#include "nsNSSComponent.h" +#include "nsNSSHelper.h" +#include "plbase64.h" + +using namespace mozilla; +using namespace mozilla::psm; + +// Standard ISupports implementation +// NOTE: Should these be the thread-safe versions? + +/***** + * nsCMSSecureMessage + *****/ + +// Standard ISupports implementation +NS_IMPL_ISUPPORTS(nsCMSSecureMessage, nsICMSSecureMessage) + +// nsCMSSecureMessage constructor +nsCMSSecureMessage::nsCMSSecureMessage() { + // initialize superclass +} + +// nsCMSMessage destructor +nsCMSSecureMessage::~nsCMSSecureMessage() {} + +nsresult nsCMSSecureMessage::Init() { + nsresult rv; + nsCOMPtr<nsISupports> nssInitialized = + do_GetService("@mozilla.org/psm;1", &rv); + return rv; +} + +nsresult nsCMSSecureMessage::CheckUsageOk(nsIX509Cert* aCert, + SECCertificateUsage aUsage, + bool* aCanBeUsed) { + NS_ENSURE_ARG_POINTER(aCert); + *aCanBeUsed = false; + + nsresult rv; + nsCOMPtr<nsIX509CertDB> certdb = + do_GetService("@mozilla.org/security/x509certdb;1", &rv); + NS_ENSURE_SUCCESS(rv, rv); + + nsTArray<uint8_t> certBytes; + rv = aCert->GetRawDER(certBytes); + NS_ENSURE_SUCCESS(rv, rv); + + RefPtr<SharedCertVerifier> certVerifier(GetDefaultCertVerifier()); + NS_ENSURE_TRUE(certVerifier, NS_ERROR_UNEXPECTED); + + nsTArray<nsTArray<uint8_t>> unusedBuiltChain; + // It's fine to skip OCSP, because this is called only from code + // for selecting the user's own configured cert. + if (certVerifier->VerifyCert(certBytes, aUsage, mozilla::pkix::Now(), nullptr, + nullptr, unusedBuiltChain, + CertVerifier::FLAG_LOCAL_ONLY) == + mozilla::pkix::Success) { + *aCanBeUsed = true; + } + return NS_OK; +} + +NS_IMETHODIMP nsCMSSecureMessage::CanBeUsedForEmailEncryption( + nsIX509Cert* aCert, bool* aCanBeUsed) { + return CheckUsageOk(aCert, certificateUsageEmailRecipient, aCanBeUsed); +} + +NS_IMETHODIMP nsCMSSecureMessage::CanBeUsedForEmailSigning(nsIX509Cert* aCert, + bool* aCanBeUsed) { + return CheckUsageOk(aCert, certificateUsageEmailSigner, aCanBeUsed); +} diff --git a/comm/mailnews/extensions/smime/nsCMSSecureMessage.h b/comm/mailnews/extensions/smime/nsCMSSecureMessage.h new file mode 100644 index 0000000000..186dba525f --- /dev/null +++ b/comm/mailnews/extensions/smime/nsCMSSecureMessage.h @@ -0,0 +1,39 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef nsCMSSecureMessage_h +#define nsCMSSecureMessage_h + +#include "nsICMSSecureMessage.h" +#include "cert.h" + +// =============================================== +// nsCMSManager - implementation of nsICMSManager +// =============================================== + +#define NS_CMSSECUREMESSAGE_CID \ + { \ + 0x5fb907e0, 0x1dd2, 0x11b2, { \ + 0xa7, 0xc0, 0xf1, 0x4c, 0x41, 0x6a, 0x62, 0xa1 \ + } \ + } + +class nsCMSSecureMessage : public nsICMSSecureMessage { + public: + NS_DECL_ISUPPORTS + NS_DECL_NSICMSSECUREMESSAGE + + nsCMSSecureMessage(); + nsresult Init(); + + private: + virtual ~nsCMSSecureMessage(); + nsresult encode(const unsigned char* data, int32_t dataLen, char** _retval); + nsresult decode(const char* data, unsigned char** result, int32_t* _retval); + nsresult CheckUsageOk(nsIX509Cert* cert, SECCertificateUsage usage, + bool* _retval); +}; + +#endif // nsCMSSecureMessage_h diff --git a/comm/mailnews/extensions/smime/nsCertPicker.cpp b/comm/mailnews/extensions/smime/nsCertPicker.cpp new file mode 100644 index 0000000000..7224762eef --- /dev/null +++ b/comm/mailnews/extensions/smime/nsCertPicker.cpp @@ -0,0 +1,410 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsCertPicker.h" + +#include "MainThreadUtils.h" +#include "ScopedNSSTypes.h" +#include "cert.h" +#include "mozilla/RefPtr.h" +#include "nsCOMPtr.h" +#include "nsICertPickDialogs.h" +#include "nsIDialogParamBlock.h" +#include "nsIInterfaceRequestor.h" +#include "nsIX509CertValidity.h" +#include "nsMemory.h" +#include "nsMsgComposeSecure.h" +#include "nsNSSCertificate.h" +#include "nsNSSComponent.h" +#include "nsNSSDialogHelper.h" +#include "nsNSSHelper.h" +#include "nsNSSCertHelper.h" +#include "nsReadableUtils.h" +#include "nsComponentManagerUtils.h" // for do_CreateInstance +#include "nsString.h" +#include "mozpkix/pkixtypes.h" +#include "mozilla/Maybe.h" +#include "mozilla/Unused.h" +#include "mozilla/intl/AppDateTimeFormat.h" + +using namespace mozilla; + +// Copied from security/manager/ssl/nsCertTree.cpp +static void PRTimeToLocalDateString(PRTime time, nsAString& result) { + PRExplodedTime explodedTime; + PR_ExplodeTime(time, PR_LocalTimeParameters, &explodedTime); + mozilla::intl::DateTimeFormat::StyleBag style; + style.date = mozilla::Some(mozilla::intl::DateTimeFormat::Style::Long); + style.time = mozilla::Nothing(); + mozilla::Unused << intl::AppDateTimeFormat::Format(style, &explodedTime, + result); +} + +MOZ_TYPE_SPECIFIC_UNIQUE_PTR_TEMPLATE(UniqueCERTCertNicknames, + CERTCertNicknames, CERT_FreeNicknames) + +CERTCertNicknames* getNSSCertNicknamesFromCertList( + const UniqueCERTCertList& certList) { + nsAutoString expiredString, notYetValidString; + nsAutoString expiredStringLeadingSpace, notYetValidStringLeadingSpace; + + GetPIPNSSBundleString("NicknameExpired", expiredString); + GetPIPNSSBundleString("NicknameNotYetValid", notYetValidString); + + expiredStringLeadingSpace.Append(' '); + expiredStringLeadingSpace.Append(expiredString); + + notYetValidStringLeadingSpace.Append(' '); + notYetValidStringLeadingSpace.Append(notYetValidString); + + NS_ConvertUTF16toUTF8 aUtf8ExpiredString(expiredStringLeadingSpace); + NS_ConvertUTF16toUTF8 aUtf8NotYetValidString(notYetValidStringLeadingSpace); + + return CERT_NicknameStringsFromCertList( + certList.get(), const_cast<char*>(aUtf8ExpiredString.get()), + const_cast<char*>(aUtf8NotYetValidString.get())); +} + +nsresult FormatUIStrings(nsIX509Cert* cert, const nsAutoString& nickname, + nsAutoString& nickWithSerial, nsAutoString& details) { + if (!NS_IsMainThread()) { + NS_ERROR("nsNSSCertificate::FormatUIStrings called off the main thread"); + return NS_ERROR_NOT_SAME_THREAD; + } + + RefPtr<nsMsgComposeSecure> mcs = new nsMsgComposeSecure; + if (!mcs) { + return NS_ERROR_FAILURE; + } + + nsAutoString info; + nsAutoString temp1; + + nickWithSerial.Append(nickname); + + if (NS_SUCCEEDED(mcs->GetSMIMEBundleString(u"CertInfoIssuedFor", info))) { + details.Append(info); + details.Append(char16_t(' ')); + if (NS_SUCCEEDED(cert->GetSubjectName(temp1)) && !temp1.IsEmpty()) { + details.Append(temp1); + } + details.Append(char16_t('\n')); + } + + if (NS_SUCCEEDED(cert->GetSerialNumber(temp1)) && !temp1.IsEmpty()) { + details.AppendLiteral(" "); + if (NS_SUCCEEDED(mcs->GetSMIMEBundleString(u"CertDumpSerialNo", info))) { + details.Append(info); + details.AppendLiteral(": "); + } + details.Append(temp1); + + nickWithSerial.AppendLiteral(" ["); + nickWithSerial.Append(temp1); + nickWithSerial.Append(char16_t(']')); + + details.Append(char16_t('\n')); + } + + nsCOMPtr<nsIX509CertValidity> validity; + nsresult rv = cert->GetValidity(getter_AddRefs(validity)); + if (NS_SUCCEEDED(rv) && validity) { + details.AppendLiteral(" "); + if (NS_SUCCEEDED(mcs->GetSMIMEBundleString(u"CertInfoValid", info))) { + details.Append(info); + } + + PRTime notBefore; + rv = validity->GetNotBefore(¬Before); + if (NS_SUCCEEDED(rv)) { + details.Append(char16_t(' ')); + if (NS_SUCCEEDED(mcs->GetSMIMEBundleString(u"CertInfoFrom", info))) { + details.Append(info); + details.Append(char16_t(' ')); + } + PRTimeToLocalDateString(notBefore, temp1); + details.Append(temp1); + } + + PRTime notAfter; + rv = validity->GetNotAfter(¬After); + if (NS_SUCCEEDED(rv)) { + details.Append(char16_t(' ')); + if (NS_SUCCEEDED(mcs->GetSMIMEBundleString(u"CertInfoTo", info))) { + details.Append(info); + details.Append(char16_t(' ')); + } + PRTimeToLocalDateString(notAfter, temp1); + details.Append(temp1); + } + details.Append(char16_t('\n')); + } + + UniqueCERTCertificate nssCert(cert->GetCert()); + if (!nssCert) { + return NS_ERROR_FAILURE; + } + + nsAutoString firstEmail; + const char* aWalkAddr; + for (aWalkAddr = CERT_GetFirstEmailAddress(nssCert.get()); aWalkAddr; + aWalkAddr = CERT_GetNextEmailAddress(nssCert.get(), aWalkAddr)) { + NS_ConvertUTF8toUTF16 email(aWalkAddr); + if (email.IsEmpty()) continue; + + if (firstEmail.IsEmpty()) { + // If the first email address from the subject DN is also present + // in the subjectAltName extension, GetEmailAddresses() will return + // it twice (as received from NSS). Remember the first address so that + // we can filter out duplicates later on. + firstEmail = email; + + details.AppendLiteral(" "); + if (NS_SUCCEEDED(mcs->GetSMIMEBundleString(u"CertInfoEmail", info))) { + details.Append(info); + details.AppendLiteral(": "); + } + details.Append(email); + } else { + // Append current address if it's different from the first one. + if (!firstEmail.Equals(email)) { + details.AppendLiteral(", "); + details.Append(email); + } + } + } + + if (!firstEmail.IsEmpty()) { + // We got at least one email address, so we want a newline + details.Append(char16_t('\n')); + } + + if (NS_SUCCEEDED(mcs->GetSMIMEBundleString(u"CertInfoIssuedBy", info))) { + details.Append(info); + details.Append(char16_t(' ')); + + if (NS_SUCCEEDED(cert->GetIssuerName(temp1)) && !temp1.IsEmpty()) { + details.Append(temp1); + } + + details.Append(char16_t('\n')); + } + + if (NS_SUCCEEDED(mcs->GetSMIMEBundleString(u"CertInfoStoredIn", info))) { + details.Append(info); + details.Append(char16_t(' ')); + + if (NS_SUCCEEDED(cert->GetTokenName(temp1)) && !temp1.IsEmpty()) { + details.Append(temp1); + } + } + + // the above produces the following output: + // + // Issued to: $subjectName + // Serial number: $serialNumber + // Valid from: $starting_date to $expiration_date + // Certificate Key usage: $usages + // Email: $address(es) + // Issued by: $issuerName + // Stored in: $token + + return rv; +} + +NS_IMPL_ISUPPORTS(nsCertPicker, nsICertPickDialogs, nsIUserCertPicker) + +nsCertPicker::nsCertPicker() {} + +nsCertPicker::~nsCertPicker() {} + +nsresult nsCertPicker::Init() { + nsresult rv; + nsCOMPtr<nsISupports> psm = do_GetService("@mozilla.org/psm;1", &rv); + return rv; +} + +NS_IMETHODIMP +nsCertPicker::PickCertificate(nsIInterfaceRequestor* ctx, + const nsTArray<nsString>& certNickList, + const nsTArray<nsString>& certDetailsList, + int32_t* selectedIndex, bool* canceled) { + nsresult rv; + uint32_t i; + MOZ_ASSERT(certNickList.Length() == certDetailsList.Length()); + const uint32_t count = certNickList.Length(); + + *canceled = false; + + nsCOMPtr<nsIDialogParamBlock> block = + do_CreateInstance(NS_DIALOGPARAMBLOCK_CONTRACTID); + if (!block) return NS_ERROR_FAILURE; + + block->SetNumberStrings(1 + count * 2); + + for (i = 0; i < count; i++) { + rv = block->SetString(i, ToNewUnicode(certNickList[i])); + NS_ENSURE_SUCCESS(rv, rv); + } + + for (i = 0; i < count; i++) { + rv = block->SetString(i + count, ToNewUnicode(certDetailsList[i])); + NS_ENSURE_SUCCESS(rv, rv); + } + + rv = block->SetInt(0, count); + NS_ENSURE_SUCCESS(rv, rv); + + rv = block->SetInt(1, *selectedIndex); + NS_ENSURE_SUCCESS(rv, rv); + + rv = nsNSSDialogHelper::openDialog( + nullptr, "chrome://messenger/content/certpicker.xhtml", block); + NS_ENSURE_SUCCESS(rv, rv); + + int32_t status; + + rv = block->GetInt(0, &status); + NS_ENSURE_SUCCESS(rv, rv); + + *canceled = (status == 0) ? true : false; + if (!*canceled) { + rv = block->GetInt(1, selectedIndex); + } + return rv; +} + +NS_IMETHODIMP nsCertPicker::PickByUsage(nsIInterfaceRequestor* ctx, + const char16_t* selectedNickname, + int32_t certUsage, bool allowInvalid, + bool allowDuplicateNicknames, + const nsAString& emailAddress, + bool* canceled, nsIX509Cert** _retval) { + int32_t selectedIndex = -1; + bool selectionFound = false; + CERTCertListNode* node = nullptr; + nsresult rv = NS_OK; + + { + // Iterate over all certs. This assures that user is logged in to all + // hardware tokens. + nsCOMPtr<nsIInterfaceRequestor> ctx = new PipUIContext(); + UniqueCERTCertList allcerts(PK11_ListCerts(PK11CertListUnique, ctx)); + } + + /* find all user certs that are valid for the specified usage */ + /* note that we are allowing expired certs in this list */ + UniqueCERTCertList certList(CERT_FindUserCertsByUsage( + CERT_GetDefaultCertDB(), (SECCertUsage)certUsage, + !allowDuplicateNicknames, !allowInvalid, ctx)); + if (!certList) { + return NS_ERROR_NOT_AVAILABLE; + } + + /* if a (non-empty) emailAddress argument is supplied to PickByUsage, */ + /* remove non-matching certificates from the candidate list */ + + if (!emailAddress.IsEmpty()) { + node = CERT_LIST_HEAD(certList); + while (!CERT_LIST_END(node, certList)) { + /* if the cert has at least one e-mail address, check if suitable */ + if (CERT_GetFirstEmailAddress(node->cert)) { + RefPtr<nsNSSCertificate> tempCert(new nsNSSCertificate(node->cert)); + bool match = false; + rv = tempCert->ContainsEmailAddress(emailAddress, &match); + if (NS_FAILED(rv)) { + return rv; + } + if (!match) { + /* doesn't contain the specified address, so remove from the list */ + CERTCertListNode* freenode = node; + node = CERT_LIST_NEXT(node); + CERT_RemoveCertListNode(freenode); + continue; + } + } + node = CERT_LIST_NEXT(node); + } + } + + UniqueCERTCertNicknames nicknames(getNSSCertNicknamesFromCertList(certList)); + if (!nicknames) { + return NS_ERROR_NOT_AVAILABLE; + } + + nsTArray<nsString> certNicknameList(nicknames->numnicknames); + nsTArray<nsString> certDetailsList(nicknames->numnicknames); + + int32_t CertsToUse; + + for (CertsToUse = 0, node = CERT_LIST_HEAD(certList.get()); + !CERT_LIST_END(node, certList.get()) && + CertsToUse < nicknames->numnicknames; + node = CERT_LIST_NEXT(node)) { + RefPtr<nsNSSCertificate> tempCert(new nsNSSCertificate(node->cert)); + + if (tempCert) { + nsAutoString i_nickname( + NS_ConvertUTF8toUTF16(nicknames->nicknames[CertsToUse])); + nsAutoString nickWithSerial; + nsAutoString details; + + if (!selectionFound) { + /* for the case when selectedNickname refers to a bare nickname */ + if (i_nickname == nsDependentString(selectedNickname)) { + selectedIndex = CertsToUse; + selectionFound = true; + } + } + + if (NS_SUCCEEDED( + FormatUIStrings(tempCert, i_nickname, nickWithSerial, details))) { + certNicknameList.AppendElement(nickWithSerial); + certDetailsList.AppendElement(details); + if (!selectionFound) { + /* for the case when selectedNickname refers to nickname + serial */ + if (nickWithSerial == nsDependentString(selectedNickname)) { + selectedIndex = CertsToUse; + selectionFound = true; + } + } + } else { + // Placeholder, to keep the indexes valid. + certNicknameList.AppendElement(u""_ns); + certDetailsList.AppendElement(u""_ns); + } + + ++CertsToUse; + } + } + + if (certNicknameList.IsEmpty()) { + return NS_ERROR_NOT_AVAILABLE; + } + + nsCOMPtr<nsICertPickDialogs> dialogs; + rv = getNSSDialogs(getter_AddRefs(dialogs), NS_GET_IID(nsICertPickDialogs), + NS_CERTPICKDIALOGS_CONTRACTID); + + if (NS_SUCCEEDED(rv)) { + // Show the cert picker dialog and get the index of the selected cert. + rv = dialogs->PickCertificate(ctx, certNicknameList, certDetailsList, + &selectedIndex, canceled); + } + + if (NS_SUCCEEDED(rv) && !*canceled) { + int32_t i; + for (i = 0, node = CERT_LIST_HEAD(certList); !CERT_LIST_END(node, certList); + ++i, node = CERT_LIST_NEXT(node)) { + if (i == selectedIndex) { + RefPtr<nsNSSCertificate> cert = new nsNSSCertificate(node->cert); + cert.forget(_retval); + break; + } + } + } + + return rv; +} diff --git a/comm/mailnews/extensions/smime/nsCertPicker.h b/comm/mailnews/extensions/smime/nsCertPicker.h new file mode 100644 index 0000000000..04b5270eda --- /dev/null +++ b/comm/mailnews/extensions/smime/nsCertPicker.h @@ -0,0 +1,32 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef nsCertPicker_h +#define nsCertPicker_h + +#include "nsICertPickDialogs.h" +#include "nsIUserCertPicker.h" + +#define NS_CERT_PICKER_CID \ + { \ + 0x735959a1, 0xaf01, 0x447e, { \ + 0xb0, 0x2d, 0x56, 0xe9, 0x68, 0xfa, 0x52, 0xb4 \ + } \ + } + +class nsCertPicker : public nsICertPickDialogs, public nsIUserCertPicker { + public: + NS_DECL_ISUPPORTS + NS_DECL_NSICERTPICKDIALOGS + NS_DECL_NSIUSERCERTPICKER + + nsCertPicker(); + nsresult Init(); + + protected: + virtual ~nsCertPicker(); +}; + +#endif // nsCertPicker_h diff --git a/comm/mailnews/extensions/smime/nsEncryptedSMIMEURIsService.cpp b/comm/mailnews/extensions/smime/nsEncryptedSMIMEURIsService.cpp new file mode 100644 index 0000000000..ecd428f29b --- /dev/null +++ b/comm/mailnews/extensions/smime/nsEncryptedSMIMEURIsService.cpp @@ -0,0 +1,32 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsEncryptedSMIMEURIsService.h" + +NS_IMPL_ISUPPORTS(nsEncryptedSMIMEURIsService, nsIEncryptedSMIMEURIsService) + +nsEncryptedSMIMEURIsService::nsEncryptedSMIMEURIsService() {} + +nsEncryptedSMIMEURIsService::~nsEncryptedSMIMEURIsService() {} + +NS_IMETHODIMP nsEncryptedSMIMEURIsService::RememberEncrypted( + const nsACString& uri) { + // Assuming duplicates are allowed. + mEncryptedURIs.AppendElement(uri); + return NS_OK; +} + +NS_IMETHODIMP nsEncryptedSMIMEURIsService::ForgetEncrypted( + const nsACString& uri) { + // Assuming, this will only remove one copy of the string, if the array + // contains multiple copies of the same string. + mEncryptedURIs.RemoveElement(uri); + return NS_OK; +} + +NS_IMETHODIMP nsEncryptedSMIMEURIsService::IsEncrypted(const nsACString& uri, + bool* _retval) { + *_retval = mEncryptedURIs.Contains(uri); + return NS_OK; +} diff --git a/comm/mailnews/extensions/smime/nsEncryptedSMIMEURIsService.h b/comm/mailnews/extensions/smime/nsEncryptedSMIMEURIsService.h new file mode 100644 index 0000000000..dfce321f52 --- /dev/null +++ b/comm/mailnews/extensions/smime/nsEncryptedSMIMEURIsService.h @@ -0,0 +1,24 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef _nsEncryptedSMIMEURIsService_H_ +#define _nsEncryptedSMIMEURIsService_H_ + +#include "nsIEncryptedSMIMEURIsSrvc.h" +#include "nsTArray.h" +#include "nsString.h" + +class nsEncryptedSMIMEURIsService : public nsIEncryptedSMIMEURIsService { + public: + NS_DECL_ISUPPORTS + NS_DECL_NSIENCRYPTEDSMIMEURISSERVICE + + nsEncryptedSMIMEURIsService(); + + protected: + virtual ~nsEncryptedSMIMEURIsService(); + nsTArray<nsCString> mEncryptedURIs; +}; + +#endif diff --git a/comm/mailnews/extensions/smime/nsICMSDecoder.idl b/comm/mailnews/extensions/smime/nsICMSDecoder.idl new file mode 100644 index 0000000000..5764707c59 --- /dev/null +++ b/comm/mailnews/extensions/smime/nsICMSDecoder.idl @@ -0,0 +1,29 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" + +%{ C++ +typedef void (*NSSCMSContentCallback)(void *arg, const char *buf, unsigned long len); + +#define NS_CMSDECODER_CONTRACTID "@mozilla.org/nsCMSDecoder;1" +%} + +native NSSCMSContentCallback(NSSCMSContentCallback); + +interface nsICMSMessage; + +/** + * nsICMSDecoder + * Streaming interface to decode an CMS message, the input data may be + * passed in chunks. Cannot be called from JS. + */ +[uuid(c7c7033b-f341-4065-aadd-7eef55ce0dda)] +interface nsICMSDecoder : nsISupports +{ + void start(in NSSCMSContentCallback cb, in voidPtr arg); + void update(in string aBuf, in long aLen); + void finish(out nsICMSMessage msg); +}; diff --git a/comm/mailnews/extensions/smime/nsICMSDecoderJS.idl b/comm/mailnews/extensions/smime/nsICMSDecoderJS.idl new file mode 100644 index 0000000000..83ff66e12d --- /dev/null +++ b/comm/mailnews/extensions/smime/nsICMSDecoderJS.idl @@ -0,0 +1,24 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" + +interface nsICMSMessage; + +/** + * nsICMSDecoderJS + * Interface to decode a CMS message, can be called from JavaScript. + */ +[scriptable,uuid(7e0a6708-17f4-4573-8115-1ca14f1442e0)] +interface nsICMSDecoderJS : nsISupports +{ + /** + * Return the result of decoding/decrypting the given CMS message. + * + * @param aInput The bytes of a CMS message. + * @return The decoded data + */ + Array<uint8_t> decrypt(in Array<uint8_t> input); +}; diff --git a/comm/mailnews/extensions/smime/nsICMSEncoder.idl b/comm/mailnews/extensions/smime/nsICMSEncoder.idl new file mode 100644 index 0000000000..23fe086acb --- /dev/null +++ b/comm/mailnews/extensions/smime/nsICMSEncoder.idl @@ -0,0 +1,29 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" + +%{ C++ +typedef void (*NSSCMSContentCallback)(void *arg, const char *buf, unsigned long len); + +#define NS_CMSENCODER_CONTRACTID "@mozilla.org/nsCMSEncoder;1" +%} + +native NSSCMSContentCallback(NSSCMSContentCallback); + +interface nsICMSMessage; + +/** + * nsICMSEncoder + * Interface to Encode an CMS message + */ +[uuid(17dc4fb4-e379-4e56-a4a4-57cdcc74816f)] +interface nsICMSEncoder : nsISupports +{ + void start(in nsICMSMessage aMsg, in NSSCMSContentCallback cb, in voidPtr arg); + void update(in string aBuf, in long aLen); + void finish(); + void encode(in nsICMSMessage aMsg); +}; diff --git a/comm/mailnews/extensions/smime/nsICMSMessage.idl b/comm/mailnews/extensions/smime/nsICMSMessage.idl new file mode 100644 index 0000000000..ed863e5418 --- /dev/null +++ b/comm/mailnews/extensions/smime/nsICMSMessage.idl @@ -0,0 +1,96 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" + +interface nsISMimeVerificationListener; + +%{ C++ +#define NS_CMSMESSAGE_CONTRACTID "@mozilla.org/nsCMSMessage;1" +%} + +/* + * At the time the ptr type is eliminated in both interfaces, both should be + * made scriptable. + */ +[ptr] native UnsignedCharPtr(unsigned char); + +interface nsIX509Cert; + +[uuid(cd76ec81-02f0-41a3-8852-c0acce0bab53)] +interface nsICMSVerifyFlags : nsISupports +{ + const long NONE = 0; + const long VERIFY_ALLOW_WEAK_SHA1 = 1 << 0; +}; + +/** + * nsICMSMessage + * Interface to a CMS Message + */ +[uuid(c6d51c22-73e9-4dad-86b9-bde584e33c63)] +interface nsICMSMessage : nsISupports +{ + void contentIsSigned(out boolean aSigned); + void contentIsEncrypted(out boolean aEncrypted); + void getSignerCommonName(out string aName); + void getSignerEmailAddress(out string aEmail); + void getSignerCert(out nsIX509Cert scert); + void getEncryptionCert(out nsIX509Cert ecert); + void getSigningTime(out PRTime aTime); + + /** + * @param verifyFlags - Optional flags from nsICMSVerifyFlags. + */ + void verifySignature(in long verifyFlags); + + /** + * @param verifyFlags - Optional flags from nsICMSVerifyFlags. + */ + void verifyDetachedSignature(in long verifyFlags, + in Array<octet> aDigestData, + in int16_t aDigestType); + void CreateEncrypted(in Array<nsIX509Cert> aRecipientCerts); + + /* The parameter aDigestType must be one of the values in nsICryptoHash */ + void CreateSigned(in nsIX509Cert scert, in nsIX509Cert ecert, + in Array<octet> aDigestData, in int16_t aDigestType); + + /** + * Async version of nsICMSMessage::VerifySignature. + * Code will be executed on a background thread and + * availability of results will be notified using a + * call to nsISMimeVerificationListener. + */ + void asyncVerifySignature(in long verifyFlags, + in nsISMimeVerificationListener listener); + + /** + * Async version of nsICMSMessage::VerifyDetachedSignature. + * Code will be executed on a background thread and + * availability of results will be notified using a + * call to nsISMimeVerificationListener. + * + * Set aDigestType to one of the values from nsICryptoHash. + */ + void asyncVerifyDetachedSignature(in long verifyFlags, + in nsISMimeVerificationListener listener, + in Array<octet> aDigestData, + in int16_t aDigestType); +}; + +[uuid(5226d698-0773-4f25-b94c-7944b3fc01d3)] +interface nsISMimeVerificationListener : nsISupports { + + /** + * Notify that results are ready, that have been requested + * using nsICMSMessage::asyncVerify[Detached]Signature() + * + * verificationResultCode matches synchronous result code from + * nsICMSMessage::verify[Detached]Signature + */ + void notify(in nsICMSMessage verifiedMessage, + in nsresult verificationResultCode); +}; diff --git a/comm/mailnews/extensions/smime/nsICMSMessageErrors.idl b/comm/mailnews/extensions/smime/nsICMSMessageErrors.idl new file mode 100644 index 0000000000..7378185f50 --- /dev/null +++ b/comm/mailnews/extensions/smime/nsICMSMessageErrors.idl @@ -0,0 +1,36 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" + +/** + * nsICMSMessageErrors + * Scriptable error constants for nsICMSMessage + */ +[scriptable,uuid(267f1a5b-88f7-413b-bc49-487e745282f1)] +interface nsICMSMessageErrors : nsISupports +{ + const long SUCCESS = 0; + const long GENERAL_ERROR = 1; + const long VERIFY_NOT_SIGNED = 1024; + const long VERIFY_NO_CONTENT_INFO = 1025; + const long VERIFY_BAD_DIGEST = 1026; + const long VERIFY_NOCERT = 1028; + const long VERIFY_UNTRUSTED = 1029; + const long VERIFY_ERROR_UNVERIFIED = 1031; + const long VERIFY_ERROR_PROCESSING = 1032; + const long VERIFY_BAD_SIGNATURE = 1033; + const long VERIFY_DIGEST_MISMATCH = 1034; + const long VERIFY_UNKNOWN_ALGO = 1035; + const long VERIFY_UNSUPPORTED_ALGO = 1036; + const long VERIFY_MALFORMED_SIGNATURE = 1037; + const long VERIFY_HEADER_MISMATCH = 1038; + const long VERIFY_NOT_YET_ATTEMPTED = 1039; + const long VERIFY_CERT_WITHOUT_ADDRESS = 1040; + const long VERIFY_TIME_MISMATCH = 1041; + + const long ENCRYPT_NO_BULK_ALG = 1056; + const long ENCRYPT_INCOMPLETE = 1057; +}; diff --git a/comm/mailnews/extensions/smime/nsICMSSecureMessage.idl b/comm/mailnews/extensions/smime/nsICMSSecureMessage.idl new file mode 100644 index 0000000000..d41fc04743 --- /dev/null +++ b/comm/mailnews/extensions/smime/nsICMSSecureMessage.idl @@ -0,0 +1,30 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" + +interface nsIX509Cert; + +/** + * nsICMSManager (service) + * Interface to access users certificate store + */ +[scriptable, uuid(17103436-0111-4819-a751-0fc4aa6e3d79)] +interface nsICMSSecureMessage : nsISupports +{ + /** + * Return true if the certificate can be used for encrypting emails. + */ + bool canBeUsedForEmailEncryption(in nsIX509Cert cert); + + /** + * Return true if the certificate can be used for signing emails. + */ + bool canBeUsedForEmailSigning(in nsIX509Cert cert); +}; + +%{C++ +#define NS_CMSSECUREMESSAGE_CONTRACTID "@mozilla.org/nsCMSSecureMessage;1" +%} diff --git a/comm/mailnews/extensions/smime/nsICertPickDialogs.idl b/comm/mailnews/extensions/smime/nsICertPickDialogs.idl new file mode 100644 index 0000000000..534d47fa77 --- /dev/null +++ b/comm/mailnews/extensions/smime/nsICertPickDialogs.idl @@ -0,0 +1,27 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" + +interface nsIInterfaceRequestor; + +/** + * nsICertPickDialogs provides a generic UI for choosing a certificate. + */ +[scriptable, uuid(51d59b08-1dd2-11b2-ad4a-a51b92f8a184)] +interface nsICertPickDialogs : nsISupports +{ + /** + * General purpose certificate prompter. + */ + void pickCertificate(in nsIInterfaceRequestor ctx, + in Array<AString> certNickList, + in Array<AString> certDetailsList, + inout long selectedIndex, + out boolean canceled); +}; + +%{C++ +#define NS_CERTPICKDIALOGS_CONTRACTID "@mozilla.org/nsCertPickDialogs;1" +%} diff --git a/comm/mailnews/extensions/smime/nsIEncryptedSMIMEURIsSrvc.idl b/comm/mailnews/extensions/smime/nsIEncryptedSMIMEURIsSrvc.idl new file mode 100644 index 0000000000..53e305c977 --- /dev/null +++ b/comm/mailnews/extensions/smime/nsIEncryptedSMIMEURIsSrvc.idl @@ -0,0 +1,24 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* This is a private interface used exclusively by SMIME. + It provides functionality to the JS UI code, + that is only accessible from C++. +*/ + +#include "nsISupports.idl" + +[scriptable, uuid(f86e55c9-530b-483f-91a7-10fb5b852488)] +interface nsIEncryptedSMIMEURIsService : nsISupports +{ + /// Remember that this URI is encrypted. + void rememberEncrypted(in AUTF8String uri); + + /// Forget that this URI is encrypted. + void forgetEncrypted(in AUTF8String uri); + + /// Check if this URI is encrypted. + boolean isEncrypted(in AUTF8String uri); +}; diff --git a/comm/mailnews/extensions/smime/nsIMsgSMIMEHeaderSink.idl b/comm/mailnews/extensions/smime/nsIMsgSMIMEHeaderSink.idl new file mode 100644 index 0000000000..5a0a9b838b --- /dev/null +++ b/comm/mailnews/extensions/smime/nsIMsgSMIMEHeaderSink.idl @@ -0,0 +1,30 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + + +/* This is a private interface used exclusively by SMIME. NO ONE outside of extensions/smime + or the hard coded smime decryption files in mime/src should have any knowledge nor should + be referring to this interface. +*/ + +#include "nsISupports.idl" + +interface nsIX509Cert; + +[scriptable, uuid(25380FA1-E70C-4e82-B0BC-F31C2F41C470)] +interface nsIMsgSMIMEHeaderSink : nsISupports +{ + void signedStatus(in long aNestingLevel, + in long aSignatureStatus, + in nsIX509Cert aSignerCert, + in AUTF8String aMsgNeckoURL, + in ACString originMimePartNumber); + void encryptionStatus(in long aNestingLevel, + in long aEncryptionStatus, + in nsIX509Cert aReceipientCert, + in AUTF8String aMsgNeckoURL, + in ACString originMimePartNumber); + void ignoreStatusFrom(in ACString originMimePartNumber); +}; diff --git a/comm/mailnews/extensions/smime/nsIUserCertPicker.idl b/comm/mailnews/extensions/smime/nsIUserCertPicker.idl new file mode 100644 index 0000000000..666941c297 --- /dev/null +++ b/comm/mailnews/extensions/smime/nsIUserCertPicker.idl @@ -0,0 +1,28 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" + +interface nsIX509Cert; +interface nsIInterfaceRequestor; + +[scriptable, uuid(92396323-23f2-49e0-bf98-a25a725231ab)] +interface nsIUserCertPicker : nsISupports { + nsIX509Cert pickByUsage(in nsIInterfaceRequestor ctx, + in wstring selectedNickname, + in long certUsage, // as defined by NSS enum SECCertUsage + in boolean allowInvalid, + in boolean allowDuplicateNicknames, + in AString emailAddress, // optional - if non-empty, + // skip certificates which + // have at least one e-mail + // address but do not + // include this specific one + out boolean canceled); +}; + +%{C++ +#define NS_CERT_PICKER_CONTRACTID "@mozilla.org/user_cert_picker;1" +%} diff --git a/comm/mailnews/extensions/smime/nsMsgComposeSecure.cpp b/comm/mailnews/extensions/smime/nsMsgComposeSecure.cpp new file mode 100644 index 0000000000..4ecf73783d --- /dev/null +++ b/comm/mailnews/extensions/smime/nsMsgComposeSecure.cpp @@ -0,0 +1,1277 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsMsgComposeSecure.h" + +#include "ScopedNSSTypes.h" +#include "cert.h" +#include "keyhi.h" +#include "mozilla/RefPtr.h" +#include "mozilla/Components.h" +#include "mozilla/mailnews/MimeEncoder.h" +#include "mozilla/mailnews/MimeHeaderParser.h" +#include "msgCore.h" +#include "nsComponentManagerUtils.h" +#include "nsICryptoHash.h" +#include "nsIMimeConverter.h" +#include "nsIMsgCompFields.h" +#include "nsIMsgIdentity.h" +#include "nsIX509CertDB.h" +#include "nsMemory.h" +#include "nsMimeTypes.h" +#include "nsNSSComponent.h" +#include "nsServiceManagerUtils.h" +#include "nspr.h" +#include "mozpkix/Result.h" +#include "nsNSSCertificate.h" +#include "nsNSSHelper.h" +#include "CryptoTask.h" + +using namespace mozilla::mailnews; +using namespace mozilla; +using namespace mozilla::psm; + +#define MK_MIME_ERROR_WRITING_FILE -1 + +#define SMIME_STRBUNDLE_URL "chrome://messenger/locale/am-smime.properties" + +// It doesn't make sense to encode the message because the message will be +// displayed only if the MUA doesn't support MIME. +// We need to consider what to do in case the server doesn't support 8BITMIME. +// In short, we can't use non-ASCII characters here. +static const char crypto_multipart_blurb[] = + "This is a cryptographically signed message in MIME format."; + +static void mime_crypto_write_base64(void* closure, const char* buf, + unsigned long size); +static nsresult mime_encoder_output_fn(const char* buf, int32_t size, + void* closure); +static nsresult mime_nested_encoder_output_fn(const char* buf, int32_t size, + void* closure); +static nsresult make_multipart_signed_header_string(bool outer_p, + char** header_return, + char** boundary_return, + int16_t hash_type); +static char* mime_make_separator(const char* prefix); + +static void GenerateGlobalRandomBytes(unsigned char* buf, int32_t len) { + static bool firstTime = true; + + if (firstTime) { + // Seed the random-number generator with current time so that + // the numbers will be different every time we run. + srand((unsigned)PR_Now()); + firstTime = false; + } + + for (int32_t i = 0; i < len; i++) buf[i] = rand() % 10; +} + +char* mime_make_separator(const char* prefix) { + unsigned char rand_buf[13]; + GenerateGlobalRandomBytes(rand_buf, 12); + + return PR_smprintf( + "------------%s" + "%02X%02X%02X%02X" + "%02X%02X%02X%02X" + "%02X%02X%02X%02X", + prefix, rand_buf[0], rand_buf[1], rand_buf[2], rand_buf[3], rand_buf[4], + rand_buf[5], rand_buf[6], rand_buf[7], rand_buf[8], rand_buf[9], + rand_buf[10], rand_buf[11]); +} + +// end of copied code which needs fixed.... + +///////////////////////////////////////////////////////////////////////////////////////// +// Implementation of nsMsgComposeSecure +///////////////////////////////////////////////////////////////////////////////////////// + +NS_IMPL_ISUPPORTS(nsMsgComposeSecure, nsIMsgComposeSecure) + +nsMsgComposeSecure::nsMsgComposeSecure() + : mSignMessage(false), + mAlwaysEncryptMessage(false), + mCryptoState(mime_crypto_none), + mHashType(0), + mMultipartSignedBoundary(nullptr), + mIsDraft(false), + mBuffer(nullptr), + mBufferedBytes(0), + mErrorAlreadyReported(false) {} + +nsMsgComposeSecure::~nsMsgComposeSecure() { + /* destructor code */ + if (mEncryptionContext) { + if (mBufferedBytes) { + mEncryptionContext->Update(mBuffer, mBufferedBytes); + mBufferedBytes = 0; + } + mEncryptionContext->Finish(); + mEncryptionContext = nullptr; + } + + delete[] mBuffer; + mBuffer = nullptr; + + PR_FREEIF(mMultipartSignedBoundary); +} + +NS_IMETHODIMP nsMsgComposeSecure::SetSignMessage(bool value) { + mSignMessage = value; + return NS_OK; +} + +NS_IMETHODIMP nsMsgComposeSecure::GetSignMessage(bool* _retval) { + *_retval = mSignMessage; + return NS_OK; +} + +NS_IMETHODIMP nsMsgComposeSecure::SetRequireEncryptMessage(bool value) { + mAlwaysEncryptMessage = value; + return NS_OK; +} + +NS_IMETHODIMP nsMsgComposeSecure::GetRequireEncryptMessage(bool* _retval) { + *_retval = mAlwaysEncryptMessage; + return NS_OK; +} + +NS_IMETHODIMP nsMsgComposeSecure::RequiresCryptoEncapsulation( + nsIMsgIdentity* aIdentity, nsIMsgCompFields* aCompFields, + bool* aRequiresEncryptionWork) { + NS_ENSURE_ARG_POINTER(aRequiresEncryptionWork); + + *aRequiresEncryptionWork = false; + + bool alwaysEncryptMessages = false; + bool signMessage = false; + nsresult rv = ExtractEncryptionState(aIdentity, aCompFields, &signMessage, + &alwaysEncryptMessages); + NS_ENSURE_SUCCESS(rv, rv); + + if (alwaysEncryptMessages || signMessage) *aRequiresEncryptionWork = true; + + return NS_OK; +} + +nsresult nsMsgComposeSecure::GetSMIMEBundleString(const char16_t* name, + nsString& outString) { + outString.Truncate(); + + NS_ENSURE_ARG_POINTER(name); + + NS_ENSURE_TRUE(InitializeSMIMEBundle(), NS_ERROR_FAILURE); + + return mSMIMEBundle->GetStringFromName(NS_ConvertUTF16toUTF8(name).get(), + outString); +} + +nsresult nsMsgComposeSecure::SMIMEBundleFormatStringFromName( + const char* name, nsTArray<nsString>& params, nsAString& outString) { + NS_ENSURE_ARG_POINTER(name); + + if (!InitializeSMIMEBundle()) return NS_ERROR_FAILURE; + + return mSMIMEBundle->FormatStringFromName(name, params, outString); +} + +bool nsMsgComposeSecure::InitializeSMIMEBundle() { + if (mSMIMEBundle) return true; + + nsCOMPtr<nsIStringBundleService> bundleService = + mozilla::components::StringBundle::Service(); + nsresult rv = bundleService->CreateBundle(SMIME_STRBUNDLE_URL, + getter_AddRefs(mSMIMEBundle)); + NS_ENSURE_SUCCESS(rv, false); + + return true; +} + +void nsMsgComposeSecure::SetError(nsIMsgSendReport* sendReport, + const char16_t* bundle_string) { + if (!sendReport || !bundle_string) return; + + if (mErrorAlreadyReported) return; + + mErrorAlreadyReported = true; + + nsString errorString; + nsresult res = GetSMIMEBundleString(bundle_string, errorString); + if (NS_SUCCEEDED(res) && !errorString.IsEmpty()) { + sendReport->SetMessage(nsIMsgSendReport::process_Current, errorString.get(), + true); + } +} + +void nsMsgComposeSecure::SetErrorWithParam(nsIMsgSendReport* sendReport, + const char* bundle_string, + const char* param) { + if (!sendReport || !bundle_string || !param) return; + + if (mErrorAlreadyReported) return; + + mErrorAlreadyReported = true; + + nsString errorString; + nsresult res; + AutoTArray<nsString, 1> params; + CopyASCIItoUTF16(MakeStringSpan(param), *params.AppendElement()); + res = SMIMEBundleFormatStringFromName(bundle_string, params, errorString); + + if (NS_SUCCEEDED(res) && !errorString.IsEmpty()) { + sendReport->SetMessage(nsIMsgSendReport::process_Current, errorString.get(), + true); + } +} + +nsresult nsMsgComposeSecure::ExtractEncryptionState( + nsIMsgIdentity* aIdentity, nsIMsgCompFields* aComposeFields, + bool* aSignMessage, bool* aEncrypt) { + if (!aComposeFields && !aIdentity) + return NS_ERROR_FAILURE; // kick out...invalid args.... + + NS_ENSURE_ARG_POINTER(aSignMessage); + NS_ENSURE_ARG_POINTER(aEncrypt); + + this->GetSignMessage(aSignMessage); + this->GetRequireEncryptMessage(aEncrypt); + + return NS_OK; +} + +// Select a hash algorithm to sign message +// based on subject public key type and size. +static nsresult GetSigningHashFunction(nsIX509Cert* aSigningCert, + int16_t* hashType) { + // Get the signing certificate + CERTCertificate* scert = nullptr; + if (aSigningCert) { + scert = aSigningCert->GetCert(); + } + if (!scert) { + return NS_ERROR_FAILURE; + } + + UniqueSECKEYPublicKey scertPublicKey(CERT_ExtractPublicKey(scert)); + if (!scertPublicKey) { + return mozilla::MapSECStatus(SECFailure); + } + KeyType subjectPublicKeyType = SECKEY_GetPublicKeyType(scertPublicKey.get()); + + // Get the length of the signature in bits. + unsigned siglen = SECKEY_SignatureLen(scertPublicKey.get()) * 8; + if (!siglen) { + return mozilla::MapSECStatus(SECFailure); + } + + // Select a hash function for signature generation whose security strength + // meets or exceeds the security strength of the public key, using NIST + // Special Publication 800-57, Recommendation for Key Management - Part 1: + // General (Revision 3), where Table 2 specifies the security strength of + // the public key and Table 3 lists acceptable hash functions. (The security + // strength of the hash (for digital signatures) is half the length of the + // output.) + // [SP 800-57 is available at http://csrc.nist.gov/publications/PubsSPs.html.] + if (subjectPublicKeyType == rsaKey) { + // For RSA, siglen is the same as the length of the modulus. + + // SHA-1 provides equivalent security strength for up to 1024 bits + // SHA-256 provides equivalent security strength for up to 3072 bits + + if (siglen > 3072) { + *hashType = nsICryptoHash::SHA512; + } else if (siglen > 1024) { + *hashType = nsICryptoHash::SHA256; + } else { + *hashType = nsICryptoHash::SHA1; + } + } else if (subjectPublicKeyType == dsaKey) { + // For DSA, siglen is twice the length of the q parameter of the key. + // The security strength of the key is half the length (in bits) of + // the q parameter of the key. + + // NSS only supports SHA-1, SHA-224, and SHA-256 for DSA signatures. + // The S/MIME code does not support SHA-224. + + if (siglen >= 512) { // 512-bit signature = 256-bit q parameter + *hashType = nsICryptoHash::SHA256; + } else { + *hashType = nsICryptoHash::SHA1; + } + } else if (subjectPublicKeyType == ecKey) { + // For ECDSA, siglen is twice the length of the field size. The security + // strength of the key is half the length (in bits) of the field size. + + if (siglen >= 1024) { // 1024-bit signature = 512-bit field size + *hashType = nsICryptoHash::SHA512; + } else if (siglen >= 768) { // 768-bit signature = 384-bit field size + *hashType = nsICryptoHash::SHA384; + } else if (siglen >= 512) { // 512-bit signature = 256-bit field size + *hashType = nsICryptoHash::SHA256; + } else { + *hashType = nsICryptoHash::SHA1; + } + } else { + // Unknown key type + *hashType = nsICryptoHash::SHA256; + NS_WARNING("GetSigningHashFunction: Subject public key type unknown."); + } + return NS_OK; +} + +/* void beginCryptoEncapsulation (in nsOutputFileStream aStream, in boolean + * aEncrypt, in boolean aSign, in string aRecipeints, in boolean aIsDraft); */ +NS_IMETHODIMP nsMsgComposeSecure::BeginCryptoEncapsulation( + nsIOutputStream* aStream, const char* aRecipients, + nsIMsgCompFields* aCompFields, nsIMsgIdentity* aIdentity, + nsIMsgSendReport* sendReport, bool aIsDraft) { + mErrorAlreadyReported = false; + nsresult rv = NS_OK; + + // CryptoEncapsulation should be synchronous, therefore it must + // avoid cert verification or looking up certs, which often involves + // async OCSP. The message composer should already have looked up + // and verified certificates whenever the user modified the recipient + // list, and should have used CacheValidCertForEmail to make those + // certificates known to us. + // (That code may use the AsyncFindCertByEmailAddr API which allows + // lookup and validation to be performed on a background thread, + // which is required when using OCSP.) + + bool encryptMessages = false; + bool signMessage = false; + ExtractEncryptionState(aIdentity, aCompFields, &signMessage, + &encryptMessages); + + if (!signMessage && !encryptMessages) return NS_ERROR_FAILURE; + + mStream = aStream; + mIsDraft = aIsDraft; + + if (encryptMessages && signMessage) + mCryptoState = mime_crypto_signed_encrypted; + else if (encryptMessages) + mCryptoState = mime_crypto_encrypted; + else if (signMessage) + mCryptoState = mime_crypto_clear_signed; + else + PR_ASSERT(0); + + aIdentity->GetUnicharAttribute("signing_cert_name", mSigningCertName); + aIdentity->GetCharAttribute("signing_cert_dbkey", mSigningCertDBKey); + aIdentity->GetUnicharAttribute("encryption_cert_name", mEncryptionCertName); + aIdentity->GetCharAttribute("encryption_cert_dbkey", mEncryptionCertDBKey); + + rv = MimeCryptoHackCerts(aRecipients, sendReport, encryptMessages, + signMessage, aIdentity); + if (NS_FAILED(rv)) { + goto FAIL; + } + + if (signMessage && mSelfSigningCert) { + rv = GetSigningHashFunction(mSelfSigningCert, &mHashType); + NS_ENSURE_SUCCESS(rv, rv); + } + + switch (mCryptoState) { + case mime_crypto_clear_signed: + rv = MimeInitMultipartSigned(true, sendReport); + break; + case mime_crypto_opaque_signed: + PR_ASSERT(0); /* #### no api for this yet */ + rv = NS_ERROR_NOT_IMPLEMENTED; + break; + case mime_crypto_signed_encrypted: + rv = MimeInitEncryption(true, sendReport); + break; + case mime_crypto_encrypted: + rv = MimeInitEncryption(false, sendReport); + break; + case mime_crypto_none: + /* This can happen if mime_crypto_hack_certs() decided to turn off + encryption (by asking the user.) */ + // XXX 1 is not a valid nsresult + rv = static_cast<nsresult>(1); + break; + default: + PR_ASSERT(0); + break; + } + +FAIL: + return rv; +} + +/* void finishCryptoEncapsulation (in boolean aAbort); */ +NS_IMETHODIMP nsMsgComposeSecure::FinishCryptoEncapsulation( + bool aAbort, nsIMsgSendReport* sendReport) { + nsresult rv = NS_OK; + + if (!aAbort) { + switch (mCryptoState) { + case mime_crypto_clear_signed: + rv = MimeFinishMultipartSigned(true, sendReport); + break; + case mime_crypto_opaque_signed: + PR_ASSERT(0); /* #### no api for this yet */ + rv = NS_ERROR_FAILURE; + break; + case mime_crypto_signed_encrypted: + rv = MimeFinishEncryption(true, sendReport); + break; + case mime_crypto_encrypted: + rv = MimeFinishEncryption(false, sendReport); + break; + default: + PR_ASSERT(0); + rv = NS_ERROR_FAILURE; + break; + } + } + return rv; +} + +nsresult nsMsgComposeSecure::MimeInitMultipartSigned( + bool aOuter, nsIMsgSendReport* sendReport) { + /* First, construct and write out the multipart/signed MIME header data. + */ + nsresult rv = NS_OK; + char* header = 0; + uint32_t L; + + rv = make_multipart_signed_header_string( + aOuter, &header, &mMultipartSignedBoundary, mHashType); + + NS_ENSURE_SUCCESS(rv, rv); + + L = strlen(header); + + if (aOuter) { + /* If this is the outer block, write it to the file. */ + uint32_t n; + rv = mStream->Write(header, L, &n); + if (NS_FAILED(rv) || n < L) { + // XXX This is -1, not an nsresult + rv = static_cast<nsresult>(MK_MIME_ERROR_WRITING_FILE); + } + } else { + /* If this is an inner block, feed it through the crypto stream. */ + rv = MimeCryptoWriteBlock(header, L); + } + + PR_Free(header); + NS_ENSURE_SUCCESS(rv, rv); + + /* Now initialize the crypto library, so that we can compute a hash + on the object which we are signing. + */ + + PR_SetError(0, 0); + mDataHash = do_CreateInstance("@mozilla.org/security/hash;1", &rv); + NS_ENSURE_SUCCESS(rv, rv); + + rv = mDataHash->Init(mHashType); + NS_ENSURE_SUCCESS(rv, rv); + + PR_SetError(0, 0); + return rv; +} + +nsresult nsMsgComposeSecure::MimeInitEncryption(bool aSign, + nsIMsgSendReport* sendReport) { + nsresult rv; + nsCOMPtr<nsIStringBundleService> bundleSvc = + mozilla::components::StringBundle::Service(); + NS_ENSURE_TRUE(bundleSvc, NS_ERROR_UNEXPECTED); + + nsCOMPtr<nsIStringBundle> sMIMEBundle; + nsString mime_smime_enc_content_desc; + + bundleSvc->CreateBundle(SMIME_STRBUNDLE_URL, getter_AddRefs(sMIMEBundle)); + + if (!sMIMEBundle) return NS_ERROR_FAILURE; + + sMIMEBundle->GetStringFromName("mime_smimeEncryptedContentDesc", + mime_smime_enc_content_desc); + NS_ConvertUTF16toUTF8 enc_content_desc_utf8(mime_smime_enc_content_desc); + + nsCOMPtr<nsIMimeConverter> mimeConverter = + do_GetService("@mozilla.org/messenger/mimeconverter;1", &rv); + NS_ENSURE_SUCCESS(rv, rv); + nsCString encodedContentDescription; + mimeConverter->EncodeMimePartIIStr_UTF8( + enc_content_desc_utf8, false, sizeof("Content-Description: "), + nsIMimeConverter::MIME_ENCODED_WORD_SIZE, encodedContentDescription); + + /* First, construct and write out the opaque-crypto-blob MIME header data. + */ + + char* s = PR_smprintf("Content-Type: " APPLICATION_PKCS7_MIME + "; name=\"smime.p7m\"; smime-type=enveloped-data" CRLF + "Content-Transfer-Encoding: " ENCODING_BASE64 CRLF + "Content-Disposition: attachment" + "; filename=\"smime.p7m\"" CRLF + "Content-Description: %s" CRLF CRLF, + encodedContentDescription.get()); + + uint32_t L; + if (!s) return NS_ERROR_OUT_OF_MEMORY; + L = strlen(s); + uint32_t n; + rv = mStream->Write(s, L, &n); + if (NS_FAILED(rv) || n < L) { + return NS_ERROR_FAILURE; + } + PR_Free(s); + s = 0; + + /* Now initialize the crypto library, so that we can filter the object + to be encrypted through it. + */ + + if (!mIsDraft) { + PR_ASSERT(!mCerts.IsEmpty()); + if (mCerts.IsEmpty()) return NS_ERROR_FAILURE; + } + + // If a previous call to MimeInitEncryption (this function) failed, + // the mEncryptionContext already exists and references our + // mCryptoEncoder. Destroy mEncryptionContext to release the + // reference prior to resetting mCryptoEncoder. + if (mEncryptionContext) { + mEncryptionContext->Finish(); + mEncryptionContext = nullptr; + } + + // Initialize the base64 encoder + mCryptoEncoder.reset( + MimeEncoder::GetBase64Encoder(mime_encoder_output_fn, this)); + + /* Initialize the encrypter (and add the sender's cert.) */ + PR_ASSERT(mSelfEncryptionCert); + PR_SetError(0, 0); + mEncryptionCinfo = do_CreateInstance(NS_CMSMESSAGE_CONTRACTID, &rv); + if (NS_FAILED(rv)) return rv; + rv = mEncryptionCinfo->CreateEncrypted(mCerts); + if (NS_FAILED(rv)) { + SetError(sendReport, u"ErrorEncryptMail"); + goto FAIL; + } + + mEncryptionContext = do_CreateInstance(NS_CMSENCODER_CONTRACTID, &rv); + if (NS_FAILED(rv)) return rv; + + if (!mBuffer) { + mBuffer = new char[eBufferSize]; + if (!mBuffer) return NS_ERROR_OUT_OF_MEMORY; + } + + mBufferedBytes = 0; + + rv = mEncryptionContext->Start(mEncryptionCinfo, mime_crypto_write_base64, + mCryptoEncoder.get()); + if (NS_FAILED(rv)) { + SetError(sendReport, u"ErrorEncryptMail"); + goto FAIL; + } + + /* If we're signing, tack a multipart/signed header onto the front of + the data to be encrypted, and initialize the sign-hashing code too. + */ + if (aSign) { + rv = MimeInitMultipartSigned(false, sendReport); + if (NS_FAILED(rv)) goto FAIL; + } + +FAIL: + return rv; +} + +nsresult nsMsgComposeSecure::MimeFinishMultipartSigned( + bool aOuter, nsIMsgSendReport* sendReport) { + nsresult rv; + nsCOMPtr<nsICMSMessage> cinfo = + do_CreateInstance(NS_CMSMESSAGE_CONTRACTID, &rv); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr<nsICMSEncoder> encoder = + do_CreateInstance(NS_CMSENCODER_CONTRACTID, &rv); + NS_ENSURE_SUCCESS(rv, rv); + + char* header = nullptr; + nsCOMPtr<nsIStringBundleService> bundleSvc = + mozilla::components::StringBundle::Service(); + NS_ENSURE_TRUE(bundleSvc, NS_ERROR_UNEXPECTED); + + nsCOMPtr<nsIStringBundle> sMIMEBundle; + nsString mime_smime_sig_content_desc; + + bundleSvc->CreateBundle(SMIME_STRBUNDLE_URL, getter_AddRefs(sMIMEBundle)); + + if (!sMIMEBundle) return NS_ERROR_FAILURE; + + sMIMEBundle->GetStringFromName("mime_smimeSignatureContentDesc", + mime_smime_sig_content_desc); + + NS_ConvertUTF16toUTF8 sig_content_desc_utf8(mime_smime_sig_content_desc); + + /* Compute the hash... + */ + + NS_ENSURE_STATE(mDataHash); + nsAutoCString hashString; + rv = mDataHash->Finish(false, hashString); + mDataHash = nullptr; + NS_ENSURE_SUCCESS(rv, rv); + if (PR_GetError() < 0) return NS_ERROR_FAILURE; + + /* Write out the headers for the signature. + */ + uint32_t L; + header = PR_smprintf( + CRLF "--%s" CRLF "Content-Type: " APPLICATION_PKCS7_SIGNATURE + "; name=\"smime.p7s\"" CRLF + "Content-Transfer-Encoding: " ENCODING_BASE64 CRLF + "Content-Disposition: attachment; " + "filename=\"smime.p7s\"" CRLF "Content-Description: %s" CRLF CRLF, + mMultipartSignedBoundary, sig_content_desc_utf8.get()); + + if (!header) { + return NS_ERROR_OUT_OF_MEMORY; + } + + L = strlen(header); + if (aOuter) { + /* If this is the outer block, write it to the file. */ + uint32_t n; + rv = mStream->Write(header, L, &n); + if (NS_FAILED(rv) || n < L) { + // XXX This is -1, not an nsresult + rv = static_cast<nsresult>(MK_MIME_ERROR_WRITING_FILE); + } + } else { + /* If this is an inner block, feed it through the crypto stream. */ + rv = MimeCryptoWriteBlock(header, L); + } + + PR_Free(header); + NS_ENSURE_SUCCESS(rv, rv); + + /* Create the signature... + */ + + NS_ASSERTION(mHashType, "Hash function for signature has not been set."); + + PR_ASSERT(mSelfSigningCert); + PR_SetError(0, 0); + + nsTArray<uint8_t> digest; + digest.AppendElements(hashString.get(), hashString.Length()); + + rv = cinfo->CreateSigned(mSelfSigningCert, mSelfEncryptionCert, digest, + mHashType); + if (NS_FAILED(rv)) { + SetError(sendReport, u"ErrorCanNotSignMail"); + return rv; + } + + // Initialize the base64 encoder for the signature data. + MOZ_ASSERT(!mSigEncoder, "Shouldn't already have a mSigEncoder"); + mSigEncoder.reset(MimeEncoder::GetBase64Encoder( + (aOuter ? mime_encoder_output_fn : mime_nested_encoder_output_fn), this)); + + /* Write out the signature. + */ + PR_SetError(0, 0); + rv = encoder->Start(cinfo, mime_crypto_write_base64, mSigEncoder.get()); + if (NS_FAILED(rv)) { + SetError(sendReport, u"ErrorCanNotSignMail"); + return rv; + } + + // We're not passing in any data, so no update needed. + rv = encoder->Finish(); + if (NS_FAILED(rv)) { + SetError(sendReport, u"ErrorCanNotSignMail"); + return rv; + } + + // Shut down the sig's base64 encoder. + rv = mSigEncoder->Flush(); + mSigEncoder.reset(); + NS_ENSURE_SUCCESS(rv, rv); + + /* Now write out the terminating boundary. + */ + { + uint32_t L; + char* header = PR_smprintf(CRLF "--%s--" CRLF, mMultipartSignedBoundary); + PR_Free(mMultipartSignedBoundary); + mMultipartSignedBoundary = 0; + + if (!header) { + return NS_ERROR_OUT_OF_MEMORY; + } + L = strlen(header); + if (aOuter) { + /* If this is the outer block, write it to the file. */ + uint32_t n; + rv = mStream->Write(header, L, &n); + if (NS_FAILED(rv) || n < L) + // XXX This is -1, not an nsresult + rv = static_cast<nsresult>(MK_MIME_ERROR_WRITING_FILE); + } else { + /* If this is an inner block, feed it through the crypto stream. */ + rv = MimeCryptoWriteBlock(header, L); + } + } + + return rv; +} + +/* Helper function for mime_finish_crypto_encapsulation() to close off + an opaque crypto object (for encrypted or signed-and-encrypted messages.) + */ +nsresult nsMsgComposeSecure::MimeFinishEncryption( + bool aSign, nsIMsgSendReport* sendReport) { + nsresult rv; + + /* If this object is both encrypted and signed, close off the + signature first (since it's inside.) */ + if (aSign) { + rv = MimeFinishMultipartSigned(false, sendReport); + NS_ENSURE_SUCCESS(rv, rv); + } + + /* Close off the opaque encrypted blob. + */ + PR_ASSERT(mEncryptionContext); + + if (mBufferedBytes) { + rv = mEncryptionContext->Update(mBuffer, mBufferedBytes); + mBufferedBytes = 0; + if (NS_FAILED(rv)) { + PR_ASSERT(PR_GetError() < 0); + return rv; + } + } + + rv = mEncryptionContext->Finish(); + mEncryptionContext = nullptr; + + if (NS_FAILED(rv)) { + SetError(sendReport, u"ErrorEncryptMail"); + return rv; + } + + NS_ENSURE_TRUE(mEncryptionCinfo, NS_ERROR_UNEXPECTED); + + mEncryptionCinfo = nullptr; + + // Shut down the base64 encoder. + mCryptoEncoder->Flush(); + mCryptoEncoder.reset(); + + uint32_t n; + rv = mStream->Write(CRLF, 2, &n); + if (NS_FAILED(rv) || n < 2) rv = NS_ERROR_FAILURE; + + return rv; +} + +/* Used to figure out what certs should be used when encrypting this message. + */ +nsresult nsMsgComposeSecure::MimeCryptoHackCerts(const char* aRecipients, + nsIMsgSendReport* sendReport, + bool aEncrypt, bool aSign, + nsIMsgIdentity* aIdentity) { + nsCOMPtr<nsIX509CertDB> certdb = do_GetService(NS_X509CERTDB_CONTRACTID); + nsresult res = NS_OK; + + PR_ASSERT(aEncrypt || aSign); + + /* + Signing and encryption certs use the following (per-identity) preferences: + - "signing_cert_name"/"encryption_cert_name": a string specifying the + nickname of the certificate + - "signing_cert_dbkey"/"encryption_cert_dbkey": a Base64 encoded blob + specifying an nsIX509Cert dbKey (represents serial number + and issuer DN, which is considered to be unique for X.509 certificates) + */ + + RefPtr<SharedCertVerifier> certVerifier(GetDefaultCertVerifier()); + NS_ENSURE_TRUE(certVerifier, NS_ERROR_UNEXPECTED); + + // Calling CERT_GetCertNicknames has the desired side effect of + // traversing all tokens, and bringing up prompts to unlock them. + nsCOMPtr<nsIInterfaceRequestor> ctx = new PipUIContext(); + CERTCertNicknames* result_unused = CERT_GetCertNicknames( + CERT_GetDefaultCertDB(), SEC_CERT_NICKNAMES_USER, ctx); + CERT_FreeNicknames(result_unused); + + nsTArray<nsTArray<uint8_t>> builtChain; + if (!mEncryptionCertDBKey.IsEmpty()) { + res = certdb->FindCertByDBKey(mEncryptionCertDBKey, + getter_AddRefs(mSelfEncryptionCert)); + + if (NS_SUCCEEDED(res) && mSelfEncryptionCert) { + nsTArray<uint8_t> certBytes; + res = mSelfEncryptionCert->GetRawDER(certBytes); + NS_ENSURE_SUCCESS(res, res); + + if (certVerifier->VerifyCert( + certBytes, certificateUsageEmailRecipient, mozilla::pkix::Now(), + nullptr, nullptr, builtChain, + // Only local checks can run on the main thread. + // Skipping OCSP for the user's own cert seems accaptable. + CertVerifier::FLAG_LOCAL_ONLY) != mozilla::pkix::Success) { + // not suitable for encryption, so unset cert and clear pref + mSelfEncryptionCert = nullptr; + mEncryptionCertDBKey.Truncate(); + aIdentity->SetCharAttribute("encryption_cert_dbkey", + mEncryptionCertDBKey); + } + } + } + + // same procedure for the signing cert + if (!mSigningCertDBKey.IsEmpty()) { + res = certdb->FindCertByDBKey(mSigningCertDBKey, + getter_AddRefs(mSelfSigningCert)); + if (NS_SUCCEEDED(res) && mSelfSigningCert) { + nsTArray<uint8_t> certBytes; + res = mSelfSigningCert->GetRawDER(certBytes); + NS_ENSURE_SUCCESS(res, res); + + if (certVerifier->VerifyCert( + certBytes, certificateUsageEmailSigner, mozilla::pkix::Now(), + nullptr, nullptr, builtChain, + // Only local checks can run on the main thread. + // Skipping OCSP for the user's own cert seems accaptable. + CertVerifier::FLAG_LOCAL_ONLY) != mozilla::pkix::Success) { + // not suitable for signing, so unset cert and clear pref + mSelfSigningCert = nullptr; + mSigningCertDBKey.Truncate(); + aIdentity->SetCharAttribute("signing_cert_dbkey", mSigningCertDBKey); + } + } + } + + // must have both the signing and encryption certs to sign + if (!mSelfSigningCert && aSign) { + SetError(sendReport, u"NoSenderSigningCert"); + return NS_ERROR_FAILURE; + } + + if (!mSelfEncryptionCert && aEncrypt) { + SetError(sendReport, u"NoSenderEncryptionCert"); + return NS_ERROR_FAILURE; + } + + if (aEncrypt && mSelfEncryptionCert) { + // Make sure self's configured cert is prepared for being used + // as an email recipient cert. + UniqueCERTCertificate nsscert(mSelfEncryptionCert->GetCert()); + if (!nsscert) { + return NS_ERROR_FAILURE; + } + // XXX: This does not respect the nsNSSShutDownObject protocol. + if (CERT_SaveSMimeProfile(nsscert.get(), nullptr, nullptr) != SECSuccess) { + return NS_ERROR_FAILURE; + } + } + + /* If the message is to be encrypted, then get the recipient certs */ + if (aEncrypt) { + nsTArray<nsCString> mailboxes; + ExtractEmails(EncodedHeader(nsDependentCString(aRecipients)), + UTF16ArrayAdapter<>(mailboxes)); + uint32_t count = mailboxes.Length(); + + bool already_added_self_cert = false; + + for (uint32_t i = 0; i < count; i++) { + nsCString mailbox_lowercase; + ToLowerCase(mailboxes[i], mailbox_lowercase); + nsCOMPtr<nsIX509Cert> cert; + + nsCString dbKey; + res = GetCertDBKeyForEmail(mailbox_lowercase, dbKey); + if (NS_SUCCEEDED(res)) { + res = certdb->FindCertByDBKey(dbKey, getter_AddRefs(cert)); + } + + if (NS_FAILED(res) || !cert) { + // Failure to find a valid encryption cert is fatal. + // Here I assume that mailbox is ascii rather than utf8. + SetErrorWithParam(sendReport, "MissingRecipientEncryptionCert", + mailboxes[i].get()); + return res; + } + + /* #### see if recipient requests `signedData'. + if (...) no_clearsigning_p = true; + (This is the only reason we even bother looking up the certs + of the recipients if we're sending a signed-but-not-encrypted + message.) + */ + + if (cert.get() == mSelfEncryptionCert.get()) { + already_added_self_cert = true; + } + + mCerts.AppendElement(cert); + } + + if (!already_added_self_cert) { + mCerts.AppendElement(mSelfEncryptionCert); + } + } + return res; +} + +NS_IMETHODIMP nsMsgComposeSecure::MimeCryptoWriteBlock(const char* buf, + int32_t size) { + int status = 0; + nsresult rv; + + /* If this is a From line, mangle it before signing it. You just know + that something somewhere is going to mangle it later, and that's + going to cause the signature check to fail. + + (This assumes that, in the cases where From-mangling must happen, + this function is called a line at a time. That happens to be the + case.) + */ + if (size >= 5 && buf[0] == 'F' && !strncmp(buf, "From ", 5)) { + char mangle[] = ">"; + nsresult res = MimeCryptoWriteBlock(mangle, 1); + if (NS_FAILED(res)) return res; + // This value will actually be cast back to an nsresult before use, so this + // cast is reasonable under the circumstances. + status = static_cast<int>(res); + } + + /* If we're signing, or signing-and-encrypting, feed this data into + the computation of the hash. */ + if (mDataHash) { + PR_SetError(0, 0); + mDataHash->Update((const uint8_t*)buf, size); + status = PR_GetError(); + if (status < 0) goto FAIL; + } + + PR_SetError(0, 0); + if (mEncryptionContext) { + /* If we're encrypting, or signing-and-encrypting, write this data + by filtering it through the crypto library. */ + + /* We want to create equally sized encryption strings */ + const char* inputBytesIterator = buf; + uint32_t inputBytesLeft = size; + + while (inputBytesLeft) { + const uint32_t spaceLeftInBuffer = eBufferSize - mBufferedBytes; + const uint32_t bytesToAppend = + std::min(inputBytesLeft, spaceLeftInBuffer); + + memcpy(mBuffer + mBufferedBytes, inputBytesIterator, bytesToAppend); + mBufferedBytes += bytesToAppend; + + inputBytesIterator += bytesToAppend; + inputBytesLeft -= bytesToAppend; + + if (eBufferSize == mBufferedBytes) { + rv = mEncryptionContext->Update(mBuffer, mBufferedBytes); + mBufferedBytes = 0; + if (NS_FAILED(rv)) { + status = PR_GetError(); + PR_ASSERT(status < 0); + if (status >= 0) status = -1; + goto FAIL; + } + } + } + } else { + /* If we're not encrypting (presumably just signing) then write this + data directly to the file. */ + + uint32_t n; + rv = mStream->Write(buf, size, &n); + if (NS_FAILED(rv) || n < (uint32_t)size) { + // XXX MK_MIME_ERROR_WRITING_FILE is -1, which is not a valid nsresult + return static_cast<nsresult>(MK_MIME_ERROR_WRITING_FILE); + } + } +FAIL: + // XXX status sometimes has invalid nsresults like -1 or PR_GetError() + // assigned to it + return static_cast<nsresult>(status); +} + +/* Returns a string consisting of a Content-Type header, and a boundary + string, suitable for moving from the header block, down into the body + of a multipart object. The boundary itself is also returned (so that + the caller knows what to write to close it off.) + */ +static nsresult make_multipart_signed_header_string(bool outer_p, + char** header_return, + char** boundary_return, + int16_t hash_type) { + const char* hashStr; + *header_return = 0; + *boundary_return = mime_make_separator("ms"); + + if (!*boundary_return) return NS_ERROR_OUT_OF_MEMORY; + + switch (hash_type) { + case nsICryptoHash::SHA1: + hashStr = PARAM_MICALG_SHA1; + break; + case nsICryptoHash::SHA256: + hashStr = PARAM_MICALG_SHA256; + break; + case nsICryptoHash::SHA384: + hashStr = PARAM_MICALG_SHA384; + break; + case nsICryptoHash::SHA512: + hashStr = PARAM_MICALG_SHA512; + break; + default: + return NS_ERROR_INVALID_ARG; + } + + *header_return = PR_smprintf("Content-Type: " MULTIPART_SIGNED + "; " + "protocol=\"" APPLICATION_PKCS7_SIGNATURE + "\"; " + "micalg=%s; " + "boundary=\"%s\"" CRLF CRLF + "%s%s" + "--%s" CRLF, + hashStr, *boundary_return, + (outer_p ? crypto_multipart_blurb : ""), + (outer_p ? CRLF CRLF : ""), *boundary_return); + + if (!*header_return) { + PR_Free(*boundary_return); + *boundary_return = 0; + return NS_ERROR_OUT_OF_MEMORY; + } + + return NS_OK; +} + +/* Used as the output function of a SEC_PKCS7EncoderContext -- we feed + plaintext into the crypto engine, and it calls this function with encrypted + data; then this function writes a base64-encoded representation of that + data to the file (by filtering it through the given MimeEncoder object.) + + Also used as the output function of SEC_PKCS7Encode() -- but in that case, + it's used to write the encoded representation of the signature. The only + difference is which MimeEncoder object is used. + */ +static void mime_crypto_write_base64(void* closure, const char* buf, + unsigned long size) { + MimeEncoder* encoder = (MimeEncoder*)closure; + nsresult rv = encoder->Write(buf, size); + PR_SetError(NS_FAILED(rv) ? static_cast<uint32_t>(rv) : 0, 0); +} + +/* Used as the output function of MimeEncoder -- when we have generated + the signature for a multipart/signed object, this is used to write the + base64-encoded representation of the signature to the file. + */ +// TODO: size should probably be converted to uint32_t +nsresult mime_encoder_output_fn(const char* buf, int32_t size, void* closure) { + nsMsgComposeSecure* state = (nsMsgComposeSecure*)closure; + nsCOMPtr<nsIOutputStream> stream; + state->GetOutputStream(getter_AddRefs(stream)); + uint32_t n; + nsresult rv = stream->Write((char*)buf, size, &n); + if (NS_FAILED(rv) || n < (uint32_t)size) + return NS_ERROR_FAILURE; + else + return NS_OK; +} + +/* Like mime_encoder_output_fn, except this is used for the case where we + are both signing and encrypting -- the base64-encoded output of the + signature should be fed into the crypto engine, rather than being written + directly to the file. + */ +static nsresult mime_nested_encoder_output_fn(const char* buf, int32_t size, + void* closure) { + nsMsgComposeSecure* state = (nsMsgComposeSecure*)closure; + + // Copy to new null-terminated string so JS glue doesn't crash when + // MimeCryptoWriteBlock() is implemented in JS. + nsCString bufWithNull; + bufWithNull.Assign(buf, size); + return state->MimeCryptoWriteBlock(bufWithNull.get(), size); +} + +class FindSMimeCertTask final : public CryptoTask { + public: + FindSMimeCertTask(const nsACString& email, + nsIDoneFindCertForEmailCallback* listener) + : mEmail(email), mListener(listener) { + MOZ_ASSERT(NS_IsMainThread()); + } + ~FindSMimeCertTask(); + + private: + virtual nsresult CalculateResult() override; + virtual void CallCallback(nsresult rv) override; + + const nsCString mEmail; + nsCOMPtr<nsIX509Cert> mCert; + nsCOMPtr<nsIDoneFindCertForEmailCallback> mListener; + + static mozilla::StaticMutex sMutex; +}; + +mozilla::StaticMutex FindSMimeCertTask::sMutex; + +void FindSMimeCertTask::CallCallback(nsresult rv) { + MOZ_ASSERT(NS_IsMainThread()); + nsCOMPtr<nsIX509Cert> cert; + nsCOMPtr<nsIDoneFindCertForEmailCallback> listener; + { + mozilla::StaticMutexAutoLock lock(sMutex); + if (!mListener) { + return; + } + // We won't need these objects after leaving this function, so let's + // destroy them early. Also has the benefit that we're already + // on the main thread. By destroying the listener here, we avoid + // dispatching in the destructor. + mCert.swap(cert); + mListener.swap(listener); + } + listener->FindCertDone(mEmail, cert); +} + +/* +called by: + GetValidCertInfo + GetRecipientCertsInfo + GetNoCertAddresses +*/ +nsresult FindSMimeCertTask::CalculateResult() { + MOZ_ASSERT(!NS_IsMainThread()); + + nsresult rv = BlockUntilLoadableCertsLoaded(); + if (NS_FAILED(rv)) { + return rv; + } + + RefPtr<SharedCertVerifier> certVerifier(GetDefaultCertVerifier()); + NS_ENSURE_TRUE(certVerifier, NS_ERROR_UNEXPECTED); + + const nsCString& flatEmailAddress = PromiseFlatCString(mEmail); + UniqueCERTCertList certlist( + PK11_FindCertsFromEmailAddress(flatEmailAddress.get(), nullptr)); + if (!certlist) return NS_ERROR_FAILURE; + + // certlist now contains certificates with the right email address, + // but they might not have the correct usage or might even be invalid + + if (CERT_LIST_END(CERT_LIST_HEAD(certlist), certlist)) + return NS_ERROR_FAILURE; // no certs found + + CERTCertListNode* node; + // search for a valid certificate + for (node = CERT_LIST_HEAD(certlist); !CERT_LIST_END(node, certlist); + node = CERT_LIST_NEXT(node)) { + nsTArray<uint8_t> certBytes(node->cert->derCert.data, + node->cert->derCert.len); + nsTArray<nsTArray<uint8_t>> unusedCertChain; + + mozilla::pkix::Result result = certVerifier->VerifyCert( + certBytes, certificateUsageEmailRecipient, mozilla::pkix::Now(), + nullptr /*XXX pinarg*/, nullptr /*hostname*/, unusedCertChain); + if (result == mozilla::pkix::Success) { + mozilla::StaticMutexAutoLock lock(sMutex); + mCert = new nsNSSCertificate(node->cert); + break; + } + } + + return NS_OK; +} + +/* + * We need to ensure that the callback is destroyed on the main thread. + */ +class ProxyListenerDestructor final : public mozilla::Runnable { + public: + explicit ProxyListenerDestructor( + nsCOMPtr<nsIDoneFindCertForEmailCallback>&& aListener) + : mozilla::Runnable("ProxyListenerDestructor"), + mListener(std::move(aListener)) {} + + NS_IMETHODIMP + Run() override { + MOZ_ASSERT(NS_IsMainThread()); + // Release the object referenced by mListener. + mListener = nullptr; + return NS_OK; + } + + private: + nsCOMPtr<nsIDoneFindCertForEmailCallback> mListener; +}; + +FindSMimeCertTask::~FindSMimeCertTask() { + // Unless we already cleaned up inside CallCallback, we must release + // the listener on the main thread. + if (mListener && !NS_IsMainThread()) { + RefPtr<ProxyListenerDestructor> runnable = + new ProxyListenerDestructor(std::move(mListener)); + MOZ_ALWAYS_SUCCEEDS(NS_DispatchToMainThread(runnable)); + } +} + +NS_IMETHODIMP +nsMsgComposeSecure::CacheValidCertForEmail(const nsACString& email, + const nsACString& certDBKey) { + mozilla::StaticMutexAutoLock lock(sMutex); + mValidCertForEmailAddr.InsertOrUpdate(email, certDBKey); + return NS_OK; +} + +NS_IMETHODIMP +nsMsgComposeSecure::HaveValidCertForEmail(const nsACString& email, + bool* _retval) { + mozilla::StaticMutexAutoLock lock(sMutex); + *_retval = mValidCertForEmailAddr.Get(email, nullptr); + return NS_OK; +} + +NS_IMETHODIMP +nsMsgComposeSecure::GetCertDBKeyForEmail(const nsACString& email, + nsACString& _retval) { + mozilla::StaticMutexAutoLock lock(sMutex); + nsCString dbKey; + bool found = mValidCertForEmailAddr.Get(email, &dbKey); + if (found) { + _retval = dbKey; + } else { + _retval.Truncate(); + } + return NS_OK; +} + +NS_IMETHODIMP +nsMsgComposeSecure::AsyncFindCertByEmailAddr( + const nsACString& email, nsIDoneFindCertForEmailCallback* callback) { + RefPtr<CryptoTask> task = new FindSMimeCertTask(email, callback); + return task->Dispatch(); +} + +mozilla::StaticMutex nsMsgComposeSecure::sMutex; diff --git a/comm/mailnews/extensions/smime/nsMsgComposeSecure.h b/comm/mailnews/extensions/smime/nsMsgComposeSecure.h new file mode 100644 index 0000000000..14f7b54157 --- /dev/null +++ b/comm/mailnews/extensions/smime/nsMsgComposeSecure.h @@ -0,0 +1,103 @@ +/* -*- Mode: idl; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#ifndef _nsMsgComposeSecure_H_ +#define _nsMsgComposeSecure_H_ + +#include "nsIMsgComposeSecure.h" +#include "nsCOMPtr.h" +#include "nsICMSEncoder.h" +#include "nsIX509Cert.h" +#include "nsIStringBundle.h" +#include "nsICryptoHash.h" +#include "nsICMSMessage.h" +#include "nsString.h" +#include "nsTHashMap.h" +#include "nsIOutputStream.h" +#include "mozilla/UniquePtr.h" +#include "mozilla/StaticMutex.h" + +class nsIMsgCompFields; +namespace mozilla { +namespace mailnews { +class MimeEncoder; +} +} // namespace mozilla + +typedef enum { + mime_crypto_none, /* normal unencapsulated MIME message */ + mime_crypto_clear_signed, /* multipart/signed encapsulation */ + mime_crypto_opaque_signed, /* application/x-pkcs7-mime (signedData) */ + mime_crypto_encrypted, /* application/x-pkcs7-mime */ + mime_crypto_signed_encrypted /* application/x-pkcs7-mime */ +} mimeDeliveryCryptoState; + +class nsMsgComposeSecure : public nsIMsgComposeSecure { + public: + NS_DECL_ISUPPORTS + NS_DECL_NSIMSGCOMPOSESECURE + + nsMsgComposeSecure(); + + void GetOutputStream(nsIOutputStream** stream) { + NS_IF_ADDREF(*stream = mStream); + } + nsresult GetSMIMEBundleString(const char16_t* name, nsString& outString); + + private: + virtual ~nsMsgComposeSecure(); + typedef mozilla::mailnews::MimeEncoder MimeEncoder; + nsresult MimeInitMultipartSigned(bool aOuter, nsIMsgSendReport* sendReport); + nsresult MimeInitEncryption(bool aSign, nsIMsgSendReport* sendReport); + nsresult MimeFinishMultipartSigned(bool aOuter, nsIMsgSendReport* sendReport); + nsresult MimeFinishEncryption(bool aSign, nsIMsgSendReport* sendReport); + nsresult MimeCryptoHackCerts(const char* aRecipients, + nsIMsgSendReport* sendReport, bool aEncrypt, + bool aSign, nsIMsgIdentity* aIdentity); + bool InitializeSMIMEBundle(); + nsresult SMIMEBundleFormatStringFromName(const char* name, + nsTArray<nsString>& params, + nsAString& outString); + nsresult ExtractEncryptionState(nsIMsgIdentity* aIdentity, + nsIMsgCompFields* aComposeFields, + bool* aSignMessage, bool* aEncrypt); + + bool mSignMessage; + bool mAlwaysEncryptMessage; + mimeDeliveryCryptoState mCryptoState; + nsCOMPtr<nsIOutputStream> mStream; + int16_t mHashType; + nsCOMPtr<nsICryptoHash> mDataHash; + mozilla::UniquePtr<MimeEncoder> mSigEncoder; + char* mMultipartSignedBoundary; + nsString mSigningCertName; + nsAutoCString mSigningCertDBKey; + nsCOMPtr<nsIX509Cert> mSelfSigningCert; + nsString mEncryptionCertName; + nsAutoCString mEncryptionCertDBKey; + nsCOMPtr<nsIX509Cert> mSelfEncryptionCert; + nsTArray<RefPtr<nsIX509Cert>> mCerts; + nsCOMPtr<nsICMSMessage> mEncryptionCinfo; + nsCOMPtr<nsICMSEncoder> mEncryptionContext; + nsCOMPtr<nsIStringBundle> mSMIMEBundle; + + // Maps email address to nsIX509Cert.dbKey of a verified certificate. + nsTHashMap<nsCStringHashKey, nsCString> mValidCertForEmailAddr; + static mozilla::StaticMutex sMutex; + + mozilla::UniquePtr<MimeEncoder> mCryptoEncoder; + bool mIsDraft; + + enum { eBufferSize = 8192 }; + char* mBuffer; + uint32_t mBufferedBytes; + + bool mErrorAlreadyReported; + void SetError(nsIMsgSendReport* sendReport, const char16_t* bundle_string); + void SetErrorWithParam(nsIMsgSendReport* sendReport, + const char* bundle_string, const char* param); +}; + +#endif |