diff options
Diffstat (limited to 'comm/mailnews/base/src/nsMsgFolderCache.cpp')
| -rw-r--r-- | comm/mailnews/base/src/nsMsgFolderCache.cpp | 570 |
1 files changed, 570 insertions, 0 deletions
diff --git a/comm/mailnews/base/src/nsMsgFolderCache.cpp b/comm/mailnews/base/src/nsMsgFolderCache.cpp new file mode 100644 index 0000000000..c843730f24 --- /dev/null +++ b/comm/mailnews/base/src/nsMsgFolderCache.cpp @@ -0,0 +1,570 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "msgCore.h" +#include "nsMsgFolderCache.h" +#include "nsIMsgFolderCacheElement.h" +#include "nsNetUtil.h" +#include "nsStreamUtils.h" +#include "nsIOutputStream.h" +#include "nsIInputStream.h" +#include "nsISafeOutputStream.h" +#include "prprf.h" +#include "mozilla/Logging.h" +// Mork-related includes. +#include "nsIMdbFactoryFactory.h" +#include "mdb.h" +// Includes for jsoncpp. +#include "json/json.h" +#include <string> + +using namespace mozilla; + +static LazyLogModule sFolderCacheLog("MsgFolderCache"); + +// Helper functions for migration of legacy pancea.dat files. +static nsresult importFromMork(PathString const& dbName, Json::Value& root); +static nsresult convertTable(nsIMdbEnv* env, nsIMdbStore* store, + nsIMdbTable* table, Json::Value& root); +static void applyEntry(nsCString const& name, nsCString const& val, + Json::Value& obj); + +/* + * nsMsgFolderCacheElement + * Folders are given this to let them manipulate their cache data. + */ +class nsMsgFolderCacheElement : public nsIMsgFolderCacheElement { + public: + nsMsgFolderCacheElement(nsMsgFolderCache* owner, nsACString const& key) + : mOwner(owner), mKey(key) {} + nsMsgFolderCacheElement() = delete; + + NS_DECL_ISUPPORTS + NS_IMETHOD GetKey(nsACString& key) override { + key = mKey; + return NS_OK; + } + + NS_IMETHOD GetCachedString(const char* name, nsACString& _retval) override { + if (!Obj().isMember(name)) return NS_ERROR_NOT_AVAILABLE; + Json::Value& o = Obj()[name]; + if (o.isConvertibleTo(Json::stringValue)) { + _retval = o.asString().c_str(); + return NS_OK; + } + // Leave _retval unchanged if an error occurs. + return NS_ERROR_NOT_AVAILABLE; + } + + NS_IMETHOD GetCachedInt32(const char* name, int32_t* _retval) override { + if (!Obj().isMember(name)) return NS_ERROR_NOT_AVAILABLE; + Json::Value& o = Obj()[name]; + if (o.isConvertibleTo(Json::intValue)) { + *_retval = o.asInt(); + return NS_OK; + } + // Leave _retval unchanged if an error occurs. + return NS_ERROR_NOT_AVAILABLE; + } + + NS_IMETHOD GetCachedUInt32(const char* name, uint32_t* _retval) override { + if (!Obj().isMember(name)) return NS_ERROR_NOT_AVAILABLE; + Json::Value& o = Obj()[name]; + if (o.isConvertibleTo(Json::uintValue)) { + *_retval = o.asUInt(); + return NS_OK; + } + // Leave _retval unchanged if an error occurs. + return NS_ERROR_NOT_AVAILABLE; + } + + NS_IMETHOD GetCachedInt64(const char* name, int64_t* _retval) override { + if (!Obj().isMember(name)) return NS_ERROR_NOT_AVAILABLE; + Json::Value& o = Obj()[name]; + // isConvertibleTo() doesn't seem to support Int64. Hence multiple checks. + if (o.isNumeric() || o.isNull() || o.isBool()) { + *_retval = o.asInt64(); + return NS_OK; + } + // Leave _retval unchanged if an error occurs. + return NS_ERROR_NOT_AVAILABLE; + } + + NS_IMETHOD SetCachedString(const char* name, + const nsACString& value) override { + if (Obj()[name] != PromiseFlatCString(value).get()) { + Obj()[name] = PromiseFlatCString(value).get(); + mOwner->SetModified(); + } + return NS_OK; + } + + NS_IMETHOD SetCachedInt32(const char* name, int32_t value) override { + if (Obj()[name] != value) { + Obj()[name] = value; + mOwner->SetModified(); + } + return NS_OK; + } + + NS_IMETHOD SetCachedUInt32(const char* name, uint32_t value) override { + if (Obj()[name] != value) { + Obj()[name] = value; + mOwner->SetModified(); + } + return NS_OK; + } + NS_IMETHOD SetCachedInt64(const char* name, int64_t value) override { + if (Obj()[name] != value) { + Obj()[name] = value; + mOwner->SetModified(); + } + return NS_OK; + } + + protected: + virtual ~nsMsgFolderCacheElement() {} + RefPtr<nsMsgFolderCache> mOwner; + nsAutoCString mKey; + + // Helper to get the Json object for this nsFolderCacheElement, + // creating it if it doesn't already exist. + Json::Value& Obj() { + Json::Value& root = *mOwner->mRoot; + // This will create an empty object if it doesn't already exist. + Json::Value& v = root[mKey.get()]; + if (v.isObject()) { + return v; + } + // uhoh... either the folder entry doesn't exist (expected) or + // the json file wasn't the structure we were expecting. + // We _really_ don't want jsoncpp to be throwing exceptions, so in either + // case we'll create a fresh new empty object there. + root[mKey.get()] = Json::Value(Json::objectValue); + return root[mKey.get()]; + } +}; + +NS_IMPL_ISUPPORTS(nsMsgFolderCacheElement, nsIMsgFolderCacheElement) + +/* + * nsMsgFolderCache implementation + */ + +NS_IMPL_ISUPPORTS(nsMsgFolderCache, nsIMsgFolderCache) + +// mRoot dynamically allocated here to avoid exposing Json in header file. +nsMsgFolderCache::nsMsgFolderCache() + : mRoot(new Json::Value(Json::objectValue)), + mSavePending(false), + mSaveTimer(NS_NewTimer()) {} + +NS_IMETHODIMP nsMsgFolderCache::Init(nsIFile* cacheFile, nsIFile* legacyFile) { + mCacheFile = cacheFile; + // Is there a JSON file to load? + bool exists; + nsresult rv = cacheFile->Exists(&exists); + if (NS_SUCCEEDED(rv) && exists) { + rv = LoadFolderCache(cacheFile); + if (NS_FAILED(rv)) { + MOZ_LOG( + sFolderCacheLog, LogLevel::Error, + ("Failed to load %s (code 0x%x)", + cacheFile->HumanReadablePath().get(), static_cast<uint32_t>(rv))); + } + // Ignore error. If load fails, we'll just start off with empty cache. + return NS_OK; + } + + MOZ_LOG(sFolderCacheLog, LogLevel::Debug, ("No cache file found.")); + + // No sign of new-style JSON file. Maybe there's an old panacea.dat we can + // migrate? + rv = legacyFile->Exists(&exists); + if (NS_SUCCEEDED(rv) && exists) { + MOZ_LOG(sFolderCacheLog, LogLevel::Debug, + ("Found %s. Attempting migration.", + legacyFile->HumanReadablePath().get())); + Json::Value root(Json::objectValue); + rv = importFromMork(legacyFile->NativePath(), root); + if (NS_SUCCEEDED(rv)) { + *mRoot = root; + MOZ_LOG(sFolderCacheLog, LogLevel::Debug, + ("Migration: Legacy cache imported")); + // Migrate it to JSON. + rv = SaveFolderCache(cacheFile); + if (NS_SUCCEEDED(rv)) { + // We're done with the legacy panacea.dat - remove it. + legacyFile->Remove(false); + } else { + MOZ_LOG( + sFolderCacheLog, LogLevel::Error, + ("Migration: save failed (code 0x%x)", static_cast<uint32_t>(rv))); + } + } else { + MOZ_LOG( + sFolderCacheLog, LogLevel::Error, + ("Migration: import failed (code 0x%x)", static_cast<uint32_t>(rv))); + } + } + // Never fails. + return NS_OK; +} + +nsMsgFolderCache::~nsMsgFolderCache() { + Flush(); + delete mRoot; +} + +NS_IMETHODIMP nsMsgFolderCache::Flush() { + if (mSavePending) { + mSaveTimer->Cancel(); + mSavePending = false; + MOZ_LOG(sFolderCacheLog, LogLevel::Debug, ("Forced save.")); + nsresult rv = SaveFolderCache(mCacheFile); + if (NS_FAILED(rv)) { + MOZ_LOG( + sFolderCacheLog, LogLevel::Error, + ("Failed to write to %s (code 0x%x)", + mCacheFile->HumanReadablePath().get(), static_cast<uint32_t>(rv))); + } + } + return NS_OK; +} + +// Read the cache data from inFile. +// It's atomic - if a failure occurs, the cache data will be left unchanged. +nsresult nsMsgFolderCache::LoadFolderCache(nsIFile* inFile) { + MOZ_LOG(sFolderCacheLog, LogLevel::Debug, + ("Loading %s", inFile->HumanReadablePath().get())); + + nsCOMPtr<nsIInputStream> inStream; + nsresult rv = NS_NewLocalFileInputStream(getter_AddRefs(inStream), inFile); + NS_ENSURE_SUCCESS(rv, rv); + + nsCString data; + rv = NS_ConsumeStream(inStream, UINT32_MAX, data); + if (NS_FAILED(rv)) { + MOZ_LOG(sFolderCacheLog, LogLevel::Error, ("Read failed.")); + return rv; + } + + Json::Value root; + Json::CharReaderBuilder builder; + std::unique_ptr<Json::CharReader> const reader(builder.newCharReader()); + if (!reader->parse(data.BeginReading(), data.EndReading(), &root, nullptr)) { + MOZ_LOG(sFolderCacheLog, LogLevel::Error, ("Error parsing JSON")); + return NS_ERROR_FAILURE; // parsing failed. + } + if (!root.isObject()) { + MOZ_LOG(sFolderCacheLog, LogLevel::Error, ("JSON root is not an object")); + return NS_ERROR_FAILURE; // bad format. + } + *mRoot = root; + return NS_OK; +} + +// Write the cache data to outFile. +nsresult nsMsgFolderCache::SaveFolderCache(nsIFile* outFile) { + MOZ_LOG(sFolderCacheLog, LogLevel::Debug, + ("Save to %s", outFile->HumanReadablePath().get())); + + // Serialise the data. + Json::StreamWriterBuilder b; + // b["indentation"] = ""; + std::string out = Json::writeString(b, *mRoot); + + // Safe stream, writes to a tempfile first then moves into proper place when + // Finish() is called. Could use NS_NewAtomicFileOutputStream, but seems hard + // to justify a full filesystem flush). + nsCOMPtr<nsIOutputStream> outputStream; + nsresult rv = + NS_NewSafeLocalFileOutputStream(getter_AddRefs(outputStream), outFile, + PR_CREATE_FILE | PR_TRUNCATE | PR_WRONLY); + NS_ENSURE_SUCCESS(rv, rv); + + const char* ptr = out.data(); + uint32_t remaining = out.length(); + while (remaining > 0) { + uint32_t written = 0; + rv = outputStream->Write(ptr, remaining, &written); + NS_ENSURE_SUCCESS(rv, rv); + remaining -= written; + ptr += written; + } + nsCOMPtr<nsISafeOutputStream> safeStream = do_QueryInterface(outputStream); + MOZ_ASSERT(safeStream); + rv = safeStream->Finish(); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +NS_IMETHODIMP nsMsgFolderCache::GetCacheElement( + const nsACString& pathKey, bool createIfMissing, + nsIMsgFolderCacheElement** result) { + nsAutoCString key(pathKey); + if (mRoot->isMember(key.get()) || createIfMissing) { + nsCOMPtr<nsIMsgFolderCacheElement> element = + new nsMsgFolderCacheElement(this, pathKey); + element.forget(result); + return NS_OK; + } + return NS_ERROR_NOT_AVAILABLE; +} + +NS_IMETHODIMP nsMsgFolderCache::RemoveElement(const nsACString& key) { + mRoot->removeMember(PromiseFlatCString(key).get()); + return NS_OK; +} + +void nsMsgFolderCache::SetModified() { + if (mSavePending) { + return; + } + nsresult rv = mSaveTimer->InitWithNamedFuncCallback( + doSave, (void*)this, kSaveDelayMs, nsITimer::TYPE_ONE_SHOT, + "msgFolderCache::doSave"); + if (NS_SUCCEEDED(rv)) { + MOZ_LOG(sFolderCacheLog, LogLevel::Debug, + ("AutoSave in %ds", kSaveDelayMs / 1000)); + mSavePending = true; + } +} + +// static +void nsMsgFolderCache::doSave(nsITimer*, void* closure) { + MOZ_LOG(sFolderCacheLog, LogLevel::Debug, ("AutoSave")); + nsMsgFolderCache* that = static_cast<nsMsgFolderCache*>(closure); + nsresult rv = that->SaveFolderCache(that->mCacheFile); + if (NS_FAILED(rv)) { + MOZ_LOG(sFolderCacheLog, LogLevel::Error, + ("Failed writing %s (code 0x%x)", + that->mCacheFile->HumanReadablePath().get(), + static_cast<uint32_t>(rv))); + } + that->mSavePending = false; +} + +// Helper to apply a legacy property to the new JSON format. +static void applyEntry(nsCString const& name, nsCString const& val, + Json::Value& obj) { + // The old mork version stored all numbers as hex, so we need to convert + // them into proper Json numeric types. But there's no type info in the + // database so we just have to know which values to convert. + // We can find a list of all the numeric values by grepping the codebase + // for GetCacheInt32/GetCachedInt64. We treat everything else as a string. + // It's much harder to get a definitive list of possible keys for strings, + // because nsIMsgFolderCache is also used to cache nsDBFolderInfo data - + // see nsMsgDBFolder::GetStringProperty(). + + // One of the Int32 properties? + if (name.EqualsLiteral("hierDelim") || + name.EqualsLiteral("lastSyncTimeInSec") || + name.EqualsLiteral("nextUID") || name.EqualsLiteral("pendingMsgs") || + name.EqualsLiteral("pendingUnreadMsgs") || + name.EqualsLiteral("serverRecent") || name.EqualsLiteral("serverTotal") || + name.EqualsLiteral("serverUnseen") || name.EqualsLiteral("totalMsgs") || + name.EqualsLiteral("totalUnreadMsgs")) { + if (val.IsEmpty()) { + return; + } + int32_t i32; + if (PR_sscanf(val.get(), "%x", &i32) != 1) { + return; + } + obj[name.get()] = i32; + return; + } + + // Flags were int32. But the upper bit can be set, meaning we'll get + // annoying negative values, which isn't what we want. Not so much of an + // issue with legacy pancea.dat as it was all hex strings anyway. But let's + // fix it up as we go to JSON. + if (name.EqualsLiteral("aclFlags") || name.EqualsLiteral("boxFlags") || + name.EqualsLiteral("flags")) { + uint32_t u32; + if (PR_sscanf(val.get(), "%x", &u32) != 1) { + return; + } + obj[name.get()] = u32; + return; + } + + // One of the Int64 properties? + if (name.EqualsLiteral("expungedBytes") || name.EqualsLiteral("folderSize")) { + if (val.IsEmpty()) { + return; + } + int64_t i64; + if (PR_sscanf(val.get(), "%llx", &i64) != 1) { + return; + } + obj[name.get()] = i64; + return; + } + + // Assume anything else is a string. + obj[name.get()] = val.get(); +} + +// Import an old panacea.dat mork file, converting it into our JSON form. +// The flow of this is taken from the old implementation. There are a couple +// of steps that may not be strictly required, but have been left in anyway +// on the grounds that it works. +static nsresult importFromMork(PathString const& dbName, Json::Value& root) { + nsresult rv; + nsCOMPtr<nsIMdbFactoryService> factoryService = + do_GetService("@mozilla.org/db/mork;1", &rv); + NS_ENSURE_SUCCESS(rv, rv); + nsCOMPtr<nsIMdbFactory> factory; + rv = factoryService->GetMdbFactory(getter_AddRefs(factory)); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr<nsIMdbEnv> env; + rv = factory->MakeEnv(nullptr, getter_AddRefs(env)); + NS_ENSURE_SUCCESS(rv, rv); + + env->SetAutoClear(true); + nsCOMPtr<nsIMdbFile> dbFile; + rv = factory->OpenOldFile(env, + nullptr, // Use default heap alloc fns. + dbName.get(), + mdbBool_kTrue, // Frozen (read only). + getter_AddRefs(dbFile)); + NS_ENSURE_SUCCESS(rv, rv); + + // Unsure if we actually need this... + mdb_bool canOpen; + mdbYarn outFormatVersion; + rv = factory->CanOpenFilePort(env, dbFile, &canOpen, &outFormatVersion); + NS_ENSURE_SUCCESS(rv, rv); + if (!canOpen) { + return NS_MSG_ERROR_FOLDER_SUMMARY_OUT_OF_DATE; + } + + mdbOpenPolicy inOpenPolicy; + inOpenPolicy.mOpenPolicy_ScopePlan.mScopeStringSet_Count = 0; + inOpenPolicy.mOpenPolicy_MinMemory = 0; + inOpenPolicy.mOpenPolicy_MaxLazy = 0; + + nsCOMPtr<nsIMdbThumb> thumb; + rv = factory->OpenFileStore(env, + nullptr, // Use default heap alloc fns. + dbFile, &inOpenPolicy, getter_AddRefs(thumb)); + NS_ENSURE_SUCCESS(rv, rv); + + // Unsure what this is doing. Applying appended-but-unapplied writes? + { + mdb_count outTotal; // total somethings to do in operation + mdb_count outCurrent; // subportion of total completed so far + mdb_bool outDone; // is operation finished? + mdb_bool outBroken; // is operation irreparably dead and broken? + do { + rv = thumb->DoMore(env, &outTotal, &outCurrent, &outDone, &outBroken); + NS_ENSURE_SUCCESS(rv, rv); + } while (!outBroken && !outDone); + } + + // Finally, open the store. + nsCOMPtr<nsIMdbStore> store; + rv = factory->ThumbToOpenStore(env, thumb, getter_AddRefs(store)); + NS_ENSURE_SUCCESS(rv, rv); + + // Resolve some tokens we'll need. + const char* kFoldersScope = "ns:msg:db:row:scope:folders:all"; + mdb_token folderRowScopeToken; + rv = store->StringToToken(env, kFoldersScope, &folderRowScopeToken); + NS_ENSURE_SUCCESS(rv, rv); + + // Find the table. Only one, and we assume id=1. Eek! But original code + // did this too... + mdbOid allFoldersTableOID{folderRowScopeToken, 1}; + nsCOMPtr<nsIMdbTable> allFoldersTable; + rv = store->GetTable(env, &allFoldersTableOID, + getter_AddRefs(allFoldersTable)); + NS_ENSURE_SUCCESS(rv, rv); + // GetTable() can return null even without an error. + NS_ENSURE_STATE(allFoldersTable); + + rv = convertTable(env, store, allFoldersTable, root); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +// The legacy panacea.dat mork db has a single table, with a row per +// folder. This function reads it in and writes it into our Json::Value +// object. +static nsresult convertTable(nsIMdbEnv* env, nsIMdbStore* store, + nsIMdbTable* table, Json::Value& root) { + MOZ_ASSERT(root.isObject()); + MOZ_ASSERT(table); + + nsresult rv; + nsCOMPtr<nsIMdbTableRowCursor> rowCursor; + rv = table->GetTableRowCursor(env, -1, getter_AddRefs(rowCursor)); + NS_ENSURE_SUCCESS(rv, rv); + // For each row in the table... + while (true) { + nsCOMPtr<nsIMdbRow> row; + mdb_pos pos; + rv = rowCursor->NextRow(env, getter_AddRefs(row), &pos); + NS_ENSURE_SUCCESS(rv, rv); + if (!row) { + break; // That's all the rows done. + } + + nsCOMPtr<nsIMdbRowCellCursor> cellCursor; + rv = row->GetRowCellCursor(env, -1, getter_AddRefs(cellCursor)); + NS_ENSURE_SUCCESS(rv, rv); + + Json::Value obj(Json::objectValue); + // For each cell in the row... + nsAutoCString rowKey; + while (true) { + nsCOMPtr<nsIMdbCell> cell; + mdb_column column; + rv = cellCursor->NextCell(env, getter_AddRefs(cell), &column, nullptr); + NS_ENSURE_SUCCESS(rv, rv); + if (!cell) { + break; // No more cells. + } + + // Get the column name + nsAutoCString colName; + { + char buf[100]; + mdbYarn colYarn{buf, 0, sizeof(buf), 0, 0, nullptr}; + // Get the column of the cell + nsresult rv = store->TokenToString(env, column, &colYarn); + NS_ENSURE_SUCCESS(rv, rv); + + colName.Assign((const char*)colYarn.mYarn_Buf, colYarn.mYarn_Fill); + } + // Get the value + nsAutoCString colValue; + { + mdbYarn yarn; + cell->AliasYarn(env, &yarn); + colValue.Assign((const char*)yarn.mYarn_Buf, yarn.mYarn_Fill); + } + if (colName.EqualsLiteral("key")) { + rowKey = colValue; + } else { + applyEntry(colName, colValue, obj); + } + } + if (rowKey.IsEmpty()) { + continue; + } + + MOZ_LOG(sFolderCacheLog, LogLevel::Debug, + ("Migration: migrated key '%s' (%d properties)", rowKey.get(), + (int)obj.size())); + root[rowKey.get()] = obj; + } + return NS_OK; +} |