path: root/comm/mailnews/base/src/nsMsgFolderCache.cpp

/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#include "msgCore.h"
#include "nsMsgFolderCache.h"
#include "nsIMsgFolderCacheElement.h"
#include "nsNetUtil.h"
#include "nsStreamUtils.h"
#include "nsIOutputStream.h"
#include "nsIInputStream.h"
#include "nsISafeOutputStream.h"
#include "prprf.h"
#include "mozilla/Logging.h"
// Mork-related includes.
#include "nsIMdbFactoryFactory.h"
#include "mdb.h"
// Includes for jsoncpp.
#include "json/json.h"
#include <string>

using namespace mozilla;

static LazyLogModule sFolderCacheLog("MsgFolderCache");

// Helper functions for migration of legacy pancea.dat files.
static nsresult importFromMork(PathString const& dbName, Json::Value& root);
static nsresult convertTable(nsIMdbEnv* env, nsIMdbStore* store,
                             nsIMdbTable* table, Json::Value& root);
static void applyEntry(nsCString const& name, nsCString const& val,
                       Json::Value& obj);

/*
 * nsMsgFolderCacheElement
 * Folders are given this to let them manipulate their cache data.
 */
class nsMsgFolderCacheElement : public nsIMsgFolderCacheElement {
 public:
  nsMsgFolderCacheElement(nsMsgFolderCache* owner, nsACString const& key)
      : mOwner(owner), mKey(key) {}
  nsMsgFolderCacheElement() = delete;

  NS_DECL_ISUPPORTS
  NS_IMETHOD GetKey(nsACString& key) override {
    key = mKey;
    return NS_OK;
  }

  NS_IMETHOD GetCachedString(const char* name, nsACString& _retval) override {
    if (!Obj().isMember(name)) return NS_ERROR_NOT_AVAILABLE;
    Json::Value& o = Obj()[name];
    if (o.isConvertibleTo(Json::stringValue)) {
      _retval = o.asString().c_str();
      return NS_OK;
    }
    // Leave _retval unchanged if an error occurs.
    return NS_ERROR_NOT_AVAILABLE;
  }

  NS_IMETHOD GetCachedInt32(const char* name, int32_t* _retval) override {
    if (!Obj().isMember(name)) return NS_ERROR_NOT_AVAILABLE;
    Json::Value& o = Obj()[name];
    if (o.isConvertibleTo(Json::intValue)) {
      *_retval = o.asInt();
      return NS_OK;
    }
    // Leave _retval unchanged if an error occurs.
    return NS_ERROR_NOT_AVAILABLE;
  }

  NS_IMETHOD GetCachedUInt32(const char* name, uint32_t* _retval) override {
    if (!Obj().isMember(name)) return NS_ERROR_NOT_AVAILABLE;
    Json::Value& o = Obj()[name];
    if (o.isConvertibleTo(Json::uintValue)) {
      *_retval = o.asUInt();
      return NS_OK;
    }
    // Leave _retval unchanged if an error occurs.
    return NS_ERROR_NOT_AVAILABLE;
  }

  NS_IMETHOD GetCachedInt64(const char* name, int64_t* _retval) override {
    if (!Obj().isMember(name)) return NS_ERROR_NOT_AVAILABLE;
    Json::Value& o = Obj()[name];
    // isConvertibleTo() doesn't seem to support Int64. Hence multiple checks.
    if (o.isNumeric() || o.isNull() || o.isBool()) {
      *_retval = o.asInt64();
      return NS_OK;
    }
    // Leave _retval unchanged if an error occurs.
    return NS_ERROR_NOT_AVAILABLE;
  }

  NS_IMETHOD SetCachedString(const char* name,
                             const nsACString& value) override {
    if (Obj()[name] != PromiseFlatCString(value).get()) {
      Obj()[name] = PromiseFlatCString(value).get();
      mOwner->SetModified();
    }
    return NS_OK;
  }

  NS_IMETHOD SetCachedInt32(const char* name, int32_t value) override {
    if (Obj()[name] != value) {
      Obj()[name] = value;
      mOwner->SetModified();
    }
    return NS_OK;
  }

  NS_IMETHOD SetCachedUInt32(const char* name, uint32_t value) override {
    if (Obj()[name] != value) {
      Obj()[name] = value;
      mOwner->SetModified();
    }
    return NS_OK;
  }
  NS_IMETHOD SetCachedInt64(const char* name, int64_t value) override {
    if (Obj()[name] != value) {
      Obj()[name] = value;
      mOwner->SetModified();
    }
    return NS_OK;
  }

 protected:
  virtual ~nsMsgFolderCacheElement() {}
  RefPtr<nsMsgFolderCache> mOwner;
  nsAutoCString mKey;

  // Helper to get the Json object for this nsFolderCacheElement,
  // creating it if it doesn't already exist.
  Json::Value& Obj() {
    Json::Value& root = *mOwner->mRoot;
    // This will create an empty object if it doesn't already exist.
    Json::Value& v = root[mKey.get()];
    if (v.isObject()) {
      return v;
    }
    // uhoh... either the folder entry doesn't exist (expected) or
    // the json file wasn't the structure we were expecting.
    // We _really_ don't want jsoncpp to be throwing exceptions, so in either
    // case we'll create a fresh new empty object there.
    root[mKey.get()] = Json::Value(Json::objectValue);
    return root[mKey.get()];
  }
};

NS_IMPL_ISUPPORTS(nsMsgFolderCacheElement, nsIMsgFolderCacheElement)

/*
 * nsMsgFolderCache implementation
 */

NS_IMPL_ISUPPORTS(nsMsgFolderCache, nsIMsgFolderCache)

// mRoot dynamically allocated here to avoid exposing Json in header file.
nsMsgFolderCache::nsMsgFolderCache()
    : mRoot(new Json::Value(Json::objectValue)),
      mSavePending(false),
      mSaveTimer(NS_NewTimer()) {}

NS_IMETHODIMP nsMsgFolderCache::Init(nsIFile* cacheFile, nsIFile* legacyFile) {
  mCacheFile = cacheFile;
  // Is there a JSON file to load?
  bool exists;
  nsresult rv = cacheFile->Exists(&exists);
  if (NS_SUCCEEDED(rv) && exists) {
    rv = LoadFolderCache(cacheFile);
    if (NS_FAILED(rv)) {
      MOZ_LOG(
          sFolderCacheLog, LogLevel::Error,
          ("Failed to load %s (code 0x%x)",
           cacheFile->HumanReadablePath().get(), static_cast<uint32_t>(rv)));
    }
    // Ignore error. If load fails, we'll just start off with empty cache.
    return NS_OK;
  }

  MOZ_LOG(sFolderCacheLog, LogLevel::Debug, ("No cache file found."));

  // No sign of new-style JSON file. Maybe there's an old panacea.dat we can
  // migrate?
  rv = legacyFile->Exists(&exists);
  if (NS_SUCCEEDED(rv) && exists) {
    MOZ_LOG(sFolderCacheLog, LogLevel::Debug,
            ("Found %s. Attempting migration.",
             legacyFile->HumanReadablePath().get()));
    Json::Value root(Json::objectValue);
    rv = importFromMork(legacyFile->NativePath(), root);
    if (NS_SUCCEEDED(rv)) {
      *mRoot = root;
      MOZ_LOG(sFolderCacheLog, LogLevel::Debug,
              ("Migration: Legacy cache imported"));
      // Migrate it to JSON.
      rv = SaveFolderCache(cacheFile);
      if (NS_SUCCEEDED(rv)) {
        // We're done with the legacy panacea.dat - remove it.
        legacyFile->Remove(false);
      } else {
        MOZ_LOG(
            sFolderCacheLog, LogLevel::Error,
            ("Migration: save failed (code 0x%x)", static_cast<uint32_t>(rv)));
      }
    } else {
      MOZ_LOG(
          sFolderCacheLog, LogLevel::Error,
          ("Migration: import failed (code 0x%x)", static_cast<uint32_t>(rv)));
    }
  }
  // Never fails.
  return NS_OK;
}

nsMsgFolderCache::~nsMsgFolderCache() {
  Flush();
  delete mRoot;
}

NS_IMETHODIMP nsMsgFolderCache::Flush() {
  if (mSavePending) {
    mSaveTimer->Cancel();
    mSavePending = false;
    MOZ_LOG(sFolderCacheLog, LogLevel::Debug, ("Forced save."));
    nsresult rv = SaveFolderCache(mCacheFile);
    if (NS_FAILED(rv)) {
      MOZ_LOG(
          sFolderCacheLog, LogLevel::Error,
          ("Failed to write to %s (code 0x%x)",
           mCacheFile->HumanReadablePath().get(), static_cast<uint32_t>(rv)));
    }
  }
  return NS_OK;
}

// Read the cache data from inFile.
// It's atomic - if a failure occurs, the cache data will be left unchanged.
nsresult nsMsgFolderCache::LoadFolderCache(nsIFile* inFile) {
  MOZ_LOG(sFolderCacheLog, LogLevel::Debug,
          ("Loading %s", inFile->HumanReadablePath().get()));

  nsCOMPtr<nsIInputStream> inStream;
  nsresult rv = NS_NewLocalFileInputStream(getter_AddRefs(inStream), inFile);
  NS_ENSURE_SUCCESS(rv, rv);

  nsCString data;
  rv = NS_ConsumeStream(inStream, UINT32_MAX, data);
  if (NS_FAILED(rv)) {
    MOZ_LOG(sFolderCacheLog, LogLevel::Error, ("Read failed."));
    return rv;
  }

  Json::Value root;
  Json::CharReaderBuilder builder;
  std::unique_ptr<Json::CharReader> const reader(builder.newCharReader());
  if (!reader->parse(data.BeginReading(), data.EndReading(), &root, nullptr)) {
    MOZ_LOG(sFolderCacheLog, LogLevel::Error, ("Error parsing JSON"));
    return NS_ERROR_FAILURE;  // parsing failed.
  }
  if (!root.isObject()) {
    MOZ_LOG(sFolderCacheLog, LogLevel::Error, ("JSON root is not an object"));
    return NS_ERROR_FAILURE;  // bad format.
  }
  *mRoot = root;
  return NS_OK;
}

// Write the cache data to outFile.
nsresult nsMsgFolderCache::SaveFolderCache(nsIFile* outFile) {
  MOZ_LOG(sFolderCacheLog, LogLevel::Debug,
          ("Save to %s", outFile->HumanReadablePath().get()));

  // Serialise the data.
  Json::StreamWriterBuilder b;
  //  b["indentation"] = "";
  std::string out = Json::writeString(b, *mRoot);

  // Safe stream, writes to a tempfile first then moves into proper place when
  // Finish() is called. Could use NS_NewAtomicFileOutputStream, but seems hard
  // to justify a full filesystem flush).
  nsCOMPtr<nsIOutputStream> outputStream;
  nsresult rv =
      NS_NewSafeLocalFileOutputStream(getter_AddRefs(outputStream), outFile,
                                      PR_CREATE_FILE | PR_TRUNCATE | PR_WRONLY);
  NS_ENSURE_SUCCESS(rv, rv);

  const char* ptr = out.data();
  uint32_t remaining = out.length();
  while (remaining > 0) {
    uint32_t written = 0;
    rv = outputStream->Write(ptr, remaining, &written);
    NS_ENSURE_SUCCESS(rv, rv);
    remaining -= written;
    ptr += written;
  }
  nsCOMPtr<nsISafeOutputStream> safeStream = do_QueryInterface(outputStream);
  MOZ_ASSERT(safeStream);
  rv = safeStream->Finish();
  NS_ENSURE_SUCCESS(rv, rv);

  return NS_OK;
}

NS_IMETHODIMP nsMsgFolderCache::GetCacheElement(
    const nsACString& pathKey, bool createIfMissing,
    nsIMsgFolderCacheElement** result) {
  nsAutoCString key(pathKey);
  if (mRoot->isMember(key.get()) || createIfMissing) {
    nsCOMPtr<nsIMsgFolderCacheElement> element =
        new nsMsgFolderCacheElement(this, pathKey);
    element.forget(result);
    return NS_OK;
  }
  return NS_ERROR_NOT_AVAILABLE;
}

NS_IMETHODIMP nsMsgFolderCache::RemoveElement(const nsACString& key) {
  mRoot->removeMember(PromiseFlatCString(key).get());
  return NS_OK;
}

void nsMsgFolderCache::SetModified() {
  if (mSavePending) {
    return;
  }
  nsresult rv = mSaveTimer->InitWithNamedFuncCallback(
      doSave, (void*)this, kSaveDelayMs, nsITimer::TYPE_ONE_SHOT,
      "msgFolderCache::doSave");
  if (NS_SUCCEEDED(rv)) {
    MOZ_LOG(sFolderCacheLog, LogLevel::Debug,
            ("AutoSave in %ds", kSaveDelayMs / 1000));
    mSavePending = true;
  }
}

// static
void nsMsgFolderCache::doSave(nsITimer*, void* closure) {
  MOZ_LOG(sFolderCacheLog, LogLevel::Debug, ("AutoSave"));
  nsMsgFolderCache* that = static_cast<nsMsgFolderCache*>(closure);
  nsresult rv = that->SaveFolderCache(that->mCacheFile);
  if (NS_FAILED(rv)) {
    MOZ_LOG(sFolderCacheLog, LogLevel::Error,
            ("Failed writing %s (code 0x%x)",
             that->mCacheFile->HumanReadablePath().get(),
             static_cast<uint32_t>(rv)));
  }
  that->mSavePending = false;
}

// Helper to apply a legacy property to the new JSON format.
static void applyEntry(nsCString const& name, nsCString const& val,
                       Json::Value& obj) {
  // The old mork version stored all numbers as hex, so we need to convert
  // them into proper Json numeric types. But there's no type info in the
  // database so we just have to know which values to convert.
  // We can find a list of all the numeric values by grepping the codebase
  // for GetCacheInt32/GetCachedInt64. We treat everything else as a string.
  // It's much harder to get a definitive list of possible keys for strings,
  // because nsIMsgFolderCache is also used to cache nsDBFolderInfo data -
  // see nsMsgDBFolder::GetStringProperty().

  // One of the Int32 properties?
  if (name.EqualsLiteral("hierDelim") ||
      name.EqualsLiteral("lastSyncTimeInSec") ||
      name.EqualsLiteral("nextUID") || name.EqualsLiteral("pendingMsgs") ||
      name.EqualsLiteral("pendingUnreadMsgs") ||
      name.EqualsLiteral("serverRecent") || name.EqualsLiteral("serverTotal") ||
      name.EqualsLiteral("serverUnseen") || name.EqualsLiteral("totalMsgs") ||
      name.EqualsLiteral("totalUnreadMsgs")) {
    if (val.IsEmpty()) {
      return;
    }
    int32_t i32;
    if (PR_sscanf(val.get(), "%x", &i32) != 1) {
      return;
    }
    obj[name.get()] = i32;
    return;
  }

  // Flags were int32. But the upper bit can be set, meaning we'll get
  // annoying negative values, which isn't what we want. Not so much of an
  // issue with legacy pancea.dat as it was all hex strings anyway. But let's
  // fix it up as we go to JSON.
  if (name.EqualsLiteral("aclFlags") || name.EqualsLiteral("boxFlags") ||
      name.EqualsLiteral("flags")) {
    uint32_t u32;
    if (PR_sscanf(val.get(), "%x", &u32) != 1) {
      return;
    }
    obj[name.get()] = u32;
    return;
  }

  // One of the Int64 properties?
  if (name.EqualsLiteral("expungedBytes") || name.EqualsLiteral("folderSize")) {
    if (val.IsEmpty()) {
      return;
    }
    int64_t i64;
    if (PR_sscanf(val.get(), "%llx", &i64) != 1) {
      return;
    }
    obj[name.get()] = i64;
    return;
  }

  // Assume anything else is a string.
  obj[name.get()] = val.get();
}

// Import an old panacea.dat mork file, converting it into our JSON form.
// The flow of this is taken from the old implementation. There are a couple
// of steps that may not be strictly required, but have been left in anyway
// on the grounds that it works.
static nsresult importFromMork(PathString const& dbName, Json::Value& root) {
  nsresult rv;
  nsCOMPtr<nsIMdbFactoryService> factoryService =
      do_GetService("@mozilla.org/db/mork;1", &rv);
  NS_ENSURE_SUCCESS(rv, rv);
  nsCOMPtr<nsIMdbFactory> factory;
  rv = factoryService->GetMdbFactory(getter_AddRefs(factory));
  NS_ENSURE_SUCCESS(rv, rv);

  nsCOMPtr<nsIMdbEnv> env;
  rv = factory->MakeEnv(nullptr, getter_AddRefs(env));
  NS_ENSURE_SUCCESS(rv, rv);

  env->SetAutoClear(true);
  nsCOMPtr<nsIMdbFile> dbFile;
  rv = factory->OpenOldFile(env,
                            nullptr,  // Use default heap alloc fns.
                            dbName.get(),
                            mdbBool_kTrue,  // Frozen (read only).
                            getter_AddRefs(dbFile));
  NS_ENSURE_SUCCESS(rv, rv);

  // Unsure if we actually need this...
  mdb_bool canOpen;
  mdbYarn outFormatVersion;
  rv = factory->CanOpenFilePort(env, dbFile, &canOpen, &outFormatVersion);
  NS_ENSURE_SUCCESS(rv, rv);
  if (!canOpen) {
    return NS_MSG_ERROR_FOLDER_SUMMARY_OUT_OF_DATE;
  }

  mdbOpenPolicy inOpenPolicy;
  inOpenPolicy.mOpenPolicy_ScopePlan.mScopeStringSet_Count = 0;
  inOpenPolicy.mOpenPolicy_MinMemory = 0;
  inOpenPolicy.mOpenPolicy_MaxLazy = 0;

  nsCOMPtr<nsIMdbThumb> thumb;
  rv = factory->OpenFileStore(env,
                              nullptr,  // Use default heap alloc fns.
                              dbFile, &inOpenPolicy, getter_AddRefs(thumb));
  NS_ENSURE_SUCCESS(rv, rv);

  // Unsure what this is doing. Applying appended-but-unapplied writes?
  {
    mdb_count outTotal;    // total somethings to do in operation
    mdb_count outCurrent;  // subportion of total completed so far
    mdb_bool outDone;      // is operation finished?
    mdb_bool outBroken;    // is operation irreparably dead and broken?
    do {
      rv = thumb->DoMore(env, &outTotal, &outCurrent, &outDone, &outBroken);
      NS_ENSURE_SUCCESS(rv, rv);
    } while (!outBroken && !outDone);
  }

  // Finally, open the store.
  nsCOMPtr<nsIMdbStore> store;
  rv = factory->ThumbToOpenStore(env, thumb, getter_AddRefs(store));
  NS_ENSURE_SUCCESS(rv, rv);

  // Resolve some tokens we'll need.
  const char* kFoldersScope = "ns:msg:db:row:scope:folders:all";
  mdb_token folderRowScopeToken;
  rv = store->StringToToken(env, kFoldersScope, &folderRowScopeToken);
  NS_ENSURE_SUCCESS(rv, rv);

  // Find the table. Only one, and we assume id=1. Eek! But original code
  // did this too...
  mdbOid allFoldersTableOID{folderRowScopeToken, 1};
  nsCOMPtr<nsIMdbTable> allFoldersTable;
  rv = store->GetTable(env, &allFoldersTableOID,
                       getter_AddRefs(allFoldersTable));
  NS_ENSURE_SUCCESS(rv, rv);
  // GetTable() can return null even without an error.
  NS_ENSURE_STATE(allFoldersTable);

  rv = convertTable(env, store, allFoldersTable, root);
  NS_ENSURE_SUCCESS(rv, rv);

  return NS_OK;
}

// The legacy panacea.dat mork db has a single table, with a row per
// folder. This function reads it in and writes it into our Json::Value
// object.
static nsresult convertTable(nsIMdbEnv* env, nsIMdbStore* store,
                             nsIMdbTable* table, Json::Value& root) {
  MOZ_ASSERT(root.isObject());
  MOZ_ASSERT(table);

  nsresult rv;
  nsCOMPtr<nsIMdbTableRowCursor> rowCursor;
  rv = table->GetTableRowCursor(env, -1, getter_AddRefs(rowCursor));
  NS_ENSURE_SUCCESS(rv, rv);
  // For each row in the table...
  while (true) {
    nsCOMPtr<nsIMdbRow> row;
    mdb_pos pos;
    rv = rowCursor->NextRow(env, getter_AddRefs(row), &pos);
    NS_ENSURE_SUCCESS(rv, rv);
    if (!row) {
      break;  // That's all the rows done.
    }

    nsCOMPtr<nsIMdbRowCellCursor> cellCursor;
    rv = row->GetRowCellCursor(env, -1, getter_AddRefs(cellCursor));
    NS_ENSURE_SUCCESS(rv, rv);

    Json::Value obj(Json::objectValue);
    // For each cell in the row...
    nsAutoCString rowKey;
    while (true) {
      nsCOMPtr<nsIMdbCell> cell;
      mdb_column column;
      rv = cellCursor->NextCell(env, getter_AddRefs(cell), &column, nullptr);
      NS_ENSURE_SUCCESS(rv, rv);
      if (!cell) {
        break;  // No more cells.
      }

      // Get the column name
      nsAutoCString colName;
      {
        char buf[100];
        mdbYarn colYarn{buf, 0, sizeof(buf), 0, 0, nullptr};
        // Get the column of the cell
        nsresult rv = store->TokenToString(env, column, &colYarn);
        NS_ENSURE_SUCCESS(rv, rv);

        colName.Assign((const char*)colYarn.mYarn_Buf, colYarn.mYarn_Fill);
      }
      // Get the value
      nsAutoCString colValue;
      {
        mdbYarn yarn;
        cell->AliasYarn(env, &yarn);
        colValue.Assign((const char*)yarn.mYarn_Buf, yarn.mYarn_Fill);
      }
      if (colName.EqualsLiteral("key")) {
        rowKey = colValue;
      } else {
        applyEntry(colName, colValue, obj);
      }
    }
    if (rowKey.IsEmpty()) {
      continue;
    }

    MOZ_LOG(sFolderCacheLog, LogLevel::Debug,
            ("Migration: migrated key '%s' (%d properties)", rowKey.get(),
             (int)obj.size()));
    root[rowKey.get()] = obj;
  }
  return NS_OK;
}