diff options
Diffstat (limited to 'netwerk/cache2/CacheIndex.cpp')
| -rw-r--r-- | netwerk/cache2/CacheIndex.cpp | 3921 |
1 files changed, 3921 insertions, 0 deletions
diff --git a/netwerk/cache2/CacheIndex.cpp b/netwerk/cache2/CacheIndex.cpp new file mode 100644 index 0000000000..c0cb63ef6d --- /dev/null +++ b/netwerk/cache2/CacheIndex.cpp @@ -0,0 +1,3921 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "CacheIndex.h" + +#include "CacheLog.h" +#include "CacheFileIOManager.h" +#include "CacheFileMetadata.h" +#include "CacheFileUtils.h" +#include "CacheIndexIterator.h" +#include "CacheIndexContextIterator.h" +#include "nsThreadUtils.h" +#include "nsISizeOf.h" +#include "nsPrintfCString.h" +#include "mozilla/DebugOnly.h" +#include "prinrval.h" +#include "nsIFile.h" +#include "nsITimer.h" +#include "mozilla/AutoRestore.h" +#include <algorithm> +#include "mozilla/StaticPrefs_network.h" +#include "mozilla/Telemetry.h" +#include "mozilla/Unused.h" + +#define kMinUnwrittenChanges 300 +#define kMinDumpInterval 20000 // in milliseconds +#define kMaxBufSize 16384 +#define kIndexVersion 0x0000000A +#define kUpdateIndexStartDelay 50000 // in milliseconds +#define kTelemetryReportBytesLimit (2U * 1024U * 1024U * 1024U) // 2GB + +#define INDEX_NAME "index" +#define TEMP_INDEX_NAME "index.tmp" +#define JOURNAL_NAME "index.log" + +namespace mozilla::net { + +namespace { + +class FrecencyComparator { + public: + bool Equals(const RefPtr<CacheIndexRecordWrapper>& a, + const RefPtr<CacheIndexRecordWrapper>& b) const { + if (!a || !b) { + return false; + } + + return a->Get()->mFrecency == b->Get()->mFrecency; + } + bool LessThan(const RefPtr<CacheIndexRecordWrapper>& a, + const RefPtr<CacheIndexRecordWrapper>& b) const { + // Removed (=null) entries must be at the end of the array. + if (!a) { + return false; + } + if (!b) { + return true; + } + + // Place entries with frecency 0 at the end of the non-removed entries. + if (a->Get()->mFrecency == 0) { + return false; + } + if (b->Get()->mFrecency == 0) { + return true; + } + + return a->Get()->mFrecency < b->Get()->mFrecency; + } +}; + +} // namespace + +// used to dispatch a wrapper deletion the caller's thread +// cannot be used on IOThread after shutdown begins +class DeleteCacheIndexRecordWrapper : public Runnable { + CacheIndexRecordWrapper* mWrapper; + + public: + explicit DeleteCacheIndexRecordWrapper(CacheIndexRecordWrapper* wrapper) + : Runnable("net::CacheIndex::DeleteCacheIndexRecordWrapper"), + mWrapper(wrapper) {} + NS_IMETHOD Run() override { + StaticMutexAutoLock lock(CacheIndex::sLock); + + // if somehow the item is still in the frecency array, remove it + RefPtr<CacheIndex> index = CacheIndex::gInstance; + if (index) { + bool found = index->mFrecencyArray.RecordExistedUnlocked(mWrapper); + if (found) { + LOG( + ("DeleteCacheIndexRecordWrapper::Run() - \ + record wrapper found in frecency array during deletion")); + index->mFrecencyArray.RemoveRecord(mWrapper, lock); + } + } + + delete mWrapper; + return NS_OK; + } +}; + +void CacheIndexRecordWrapper::DispatchDeleteSelfToCurrentThread() { + // Dispatch during shutdown will not trigger DeleteCacheIndexRecordWrapper + nsCOMPtr<nsIRunnable> event = new DeleteCacheIndexRecordWrapper(this); + MOZ_ALWAYS_SUCCEEDS(NS_DispatchToCurrentThread(event)); +} + +CacheIndexRecordWrapper::~CacheIndexRecordWrapper() { +#ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED + CacheIndex::sLock.AssertCurrentThreadOwns(); + RefPtr<CacheIndex> index = CacheIndex::gInstance; + if (index) { + bool found = index->mFrecencyArray.RecordExistedUnlocked(this); + MOZ_DIAGNOSTIC_ASSERT(!found); + } +#endif +} + +/** + * This helper class is responsible for keeping CacheIndex::mIndexStats and + * CacheIndex::mFrecencyArray up to date. + */ +class MOZ_RAII CacheIndexEntryAutoManage { + public: + CacheIndexEntryAutoManage(const SHA1Sum::Hash* aHash, CacheIndex* aIndex, + const StaticMutexAutoLock& aProofOfLock) + MOZ_REQUIRES(CacheIndex::sLock) + : mIndex(aIndex), mProofOfLock(aProofOfLock) { + mHash = aHash; + const CacheIndexEntry* entry = FindEntry(); + mIndex->mIndexStats.BeforeChange(entry); + if (entry && entry->IsInitialized() && !entry->IsRemoved()) { + mOldRecord = entry->mRec; + mOldFrecency = entry->mRec->Get()->mFrecency; + } + } + + ~CacheIndexEntryAutoManage() MOZ_REQUIRES(CacheIndex::sLock) { + const CacheIndexEntry* entry = FindEntry(); + mIndex->mIndexStats.AfterChange(entry); + if (!entry || !entry->IsInitialized() || entry->IsRemoved()) { + entry = nullptr; + } + + if (entry && !mOldRecord) { + mIndex->mFrecencyArray.AppendRecord(entry->mRec, mProofOfLock); + mIndex->AddRecordToIterators(entry->mRec, mProofOfLock); + } else if (!entry && mOldRecord) { + mIndex->mFrecencyArray.RemoveRecord(mOldRecord, mProofOfLock); + mIndex->RemoveRecordFromIterators(mOldRecord, mProofOfLock); + } else if (entry && mOldRecord) { + if (entry->mRec != mOldRecord) { + // record has a different address, we have to replace it + mIndex->ReplaceRecordInIterators(mOldRecord, entry->mRec, mProofOfLock); + + if (entry->mRec->Get()->mFrecency == mOldFrecency) { + // If frecency hasn't changed simply replace the pointer + mIndex->mFrecencyArray.ReplaceRecord(mOldRecord, entry->mRec, + mProofOfLock); + } else { + // Remove old pointer and insert the new one at the end of the array + mIndex->mFrecencyArray.RemoveRecord(mOldRecord, mProofOfLock); + mIndex->mFrecencyArray.AppendRecord(entry->mRec, mProofOfLock); + } + } else if (entry->mRec->Get()->mFrecency != mOldFrecency) { + // Move the element at the end of the array + mIndex->mFrecencyArray.RemoveRecord(entry->mRec, mProofOfLock); + mIndex->mFrecencyArray.AppendRecord(entry->mRec, mProofOfLock); + } + } else { + // both entries were removed or not initialized, do nothing + } + } + + // We cannot rely on nsTHashtable::GetEntry() in case we are removing entries + // while iterating. Destructor is called before the entry is removed. Caller + // must call one of following methods to skip lookup in the hashtable. + void DoNotSearchInIndex() { mDoNotSearchInIndex = true; } + void DoNotSearchInUpdates() { mDoNotSearchInUpdates = true; } + + private: + const CacheIndexEntry* FindEntry() MOZ_REQUIRES(CacheIndex::sLock) { + const CacheIndexEntry* entry = nullptr; + + switch (mIndex->mState) { + case CacheIndex::READING: + case CacheIndex::WRITING: + if (!mDoNotSearchInUpdates) { + entry = mIndex->mPendingUpdates.GetEntry(*mHash); + } + [[fallthrough]]; + case CacheIndex::BUILDING: + case CacheIndex::UPDATING: + case CacheIndex::READY: + if (!entry && !mDoNotSearchInIndex) { + entry = mIndex->mIndex.GetEntry(*mHash); + } + break; + case CacheIndex::INITIAL: + case CacheIndex::SHUTDOWN: + default: + MOZ_ASSERT(false, "Unexpected state!"); + } + + return entry; + } + + const SHA1Sum::Hash* mHash; + RefPtr<CacheIndex> mIndex; + RefPtr<CacheIndexRecordWrapper> mOldRecord; + uint32_t mOldFrecency{0}; + bool mDoNotSearchInIndex{false}; + bool mDoNotSearchInUpdates{false}; + const StaticMutexAutoLock& mProofOfLock; +}; + +class FileOpenHelper final : public CacheFileIOListener { + public: + NS_DECL_THREADSAFE_ISUPPORTS + + explicit FileOpenHelper(CacheIndex* aIndex) + : mIndex(aIndex), mCanceled(false) {} + + void Cancel() { + CacheIndex::sLock.AssertCurrentThreadOwns(); + mCanceled = true; + } + + private: + virtual ~FileOpenHelper() = default; + + NS_IMETHOD OnFileOpened(CacheFileHandle* aHandle, nsresult aResult) override; + NS_IMETHOD OnDataWritten(CacheFileHandle* aHandle, const char* aBuf, + nsresult aResult) override { + MOZ_CRASH("FileOpenHelper::OnDataWritten should not be called!"); + return NS_ERROR_UNEXPECTED; + } + NS_IMETHOD OnDataRead(CacheFileHandle* aHandle, char* aBuf, + nsresult aResult) override { + MOZ_CRASH("FileOpenHelper::OnDataRead should not be called!"); + return NS_ERROR_UNEXPECTED; + } + NS_IMETHOD OnFileDoomed(CacheFileHandle* aHandle, nsresult aResult) override { + MOZ_CRASH("FileOpenHelper::OnFileDoomed should not be called!"); + return NS_ERROR_UNEXPECTED; + } + NS_IMETHOD OnEOFSet(CacheFileHandle* aHandle, nsresult aResult) override { + MOZ_CRASH("FileOpenHelper::OnEOFSet should not be called!"); + return NS_ERROR_UNEXPECTED; + } + NS_IMETHOD OnFileRenamed(CacheFileHandle* aHandle, + nsresult aResult) override { + MOZ_CRASH("FileOpenHelper::OnFileRenamed should not be called!"); + return NS_ERROR_UNEXPECTED; + } + + RefPtr<CacheIndex> mIndex; + bool mCanceled; +}; + +NS_IMETHODIMP FileOpenHelper::OnFileOpened(CacheFileHandle* aHandle, + nsresult aResult) { + StaticMutexAutoLock lock(CacheIndex::sLock); + + if (mCanceled) { + if (aHandle) { + CacheFileIOManager::DoomFile(aHandle, nullptr); + } + + return NS_OK; + } + + mIndex->OnFileOpenedInternal(this, aHandle, aResult, lock); + + return NS_OK; +} + +NS_IMPL_ISUPPORTS(FileOpenHelper, CacheFileIOListener); + +StaticRefPtr<CacheIndex> CacheIndex::gInstance; +StaticMutex CacheIndex::sLock; + +NS_IMPL_ADDREF(CacheIndex) +NS_IMPL_RELEASE(CacheIndex) + +NS_INTERFACE_MAP_BEGIN(CacheIndex) + NS_INTERFACE_MAP_ENTRY(mozilla::net::CacheFileIOListener) + NS_INTERFACE_MAP_ENTRY(nsIRunnable) +NS_INTERFACE_MAP_END + +CacheIndex::CacheIndex() { + sLock.AssertCurrentThreadOwns(); + LOG(("CacheIndex::CacheIndex [this=%p]", this)); + MOZ_ASSERT(!gInstance, "multiple CacheIndex instances!"); +} + +CacheIndex::~CacheIndex() { + sLock.AssertCurrentThreadOwns(); + LOG(("CacheIndex::~CacheIndex [this=%p]", this)); + + ReleaseBuffer(); +} + +// static +nsresult CacheIndex::Init(nsIFile* aCacheDirectory) { + LOG(("CacheIndex::Init()")); + + MOZ_ASSERT(NS_IsMainThread()); + + StaticMutexAutoLock lock(sLock); + + if (gInstance) { + return NS_ERROR_ALREADY_INITIALIZED; + } + + RefPtr<CacheIndex> idx = new CacheIndex(); + + nsresult rv = idx->InitInternal(aCacheDirectory, lock); + NS_ENSURE_SUCCESS(rv, rv); + + gInstance = std::move(idx); + return NS_OK; +} + +nsresult CacheIndex::InitInternal(nsIFile* aCacheDirectory, + const StaticMutexAutoLock& aProofOfLock) { + nsresult rv; + sLock.AssertCurrentThreadOwns(); + + rv = aCacheDirectory->Clone(getter_AddRefs(mCacheDirectory)); + NS_ENSURE_SUCCESS(rv, rv); + + mStartTime = TimeStamp::NowLoRes(); + + ReadIndexFromDisk(aProofOfLock); + + return NS_OK; +} + +// static +nsresult CacheIndex::PreShutdown() { + MOZ_ASSERT(NS_IsMainThread()); + + StaticMutexAutoLock lock(sLock); + + LOG(("CacheIndex::PreShutdown() [gInstance=%p]", gInstance.get())); + + nsresult rv; + RefPtr<CacheIndex> index = gInstance; + + if (!index) { + return NS_ERROR_NOT_INITIALIZED; + } + + LOG( + ("CacheIndex::PreShutdown() - [state=%d, indexOnDiskIsValid=%d, " + "dontMarkIndexClean=%d]", + index->mState, index->mIndexOnDiskIsValid, index->mDontMarkIndexClean)); + + LOG(("CacheIndex::PreShutdown() - Closing iterators.")); + for (uint32_t i = 0; i < index->mIterators.Length();) { + rv = index->mIterators[i]->CloseInternal(NS_ERROR_FAILURE); + if (NS_FAILED(rv)) { + // CacheIndexIterator::CloseInternal() removes itself from mIteratos iff + // it returns success. + LOG( + ("CacheIndex::PreShutdown() - Failed to remove iterator %p. " + "[rv=0x%08" PRIx32 "]", + index->mIterators[i], static_cast<uint32_t>(rv))); + i++; + } + } + + index->mShuttingDown = true; + + if (index->mState == READY) { + return NS_OK; // nothing to do + } + + nsCOMPtr<nsIRunnable> event; + event = NewRunnableMethod("net::CacheIndex::PreShutdownInternal", index, + &CacheIndex::PreShutdownInternal); + + nsCOMPtr<nsIEventTarget> ioTarget = CacheFileIOManager::IOTarget(); + MOZ_ASSERT(ioTarget); + + // PreShutdownInternal() will be executed before any queued event on INDEX + // level. That's OK since we don't want to wait for any operation in progess. + rv = ioTarget->Dispatch(event, nsIEventTarget::DISPATCH_NORMAL); + if (NS_FAILED(rv)) { + NS_WARNING("CacheIndex::PreShutdown() - Can't dispatch event"); + LOG(("CacheIndex::PreShutdown() - Can't dispatch event")); + return rv; + } + + return NS_OK; +} + +void CacheIndex::PreShutdownInternal() { + StaticMutexAutoLock lock(sLock); + + LOG( + ("CacheIndex::PreShutdownInternal() - [state=%d, indexOnDiskIsValid=%d, " + "dontMarkIndexClean=%d]", + mState, mIndexOnDiskIsValid, mDontMarkIndexClean)); + + MOZ_ASSERT(mShuttingDown); + + if (mUpdateTimer) { + mUpdateTimer->Cancel(); + mUpdateTimer = nullptr; + } + + switch (mState) { + case WRITING: + FinishWrite(false, lock); + break; + case READY: + // nothing to do, write the journal in Shutdown() + break; + case READING: + FinishRead(false, lock); + break; + case BUILDING: + case UPDATING: + FinishUpdate(false, lock); + break; + default: + MOZ_ASSERT(false, "Implement me!"); + } + + // We should end up in READY state + MOZ_ASSERT(mState == READY); +} + +// static +nsresult CacheIndex::Shutdown() { + MOZ_ASSERT(NS_IsMainThread()); + + StaticMutexAutoLock lock(sLock); + + LOG(("CacheIndex::Shutdown() [gInstance=%p]", gInstance.get())); + + RefPtr<CacheIndex> index = gInstance.forget(); + + if (!index) { + return NS_ERROR_NOT_INITIALIZED; + } + + bool sanitize = CacheObserver::ClearCacheOnShutdown(); + + LOG( + ("CacheIndex::Shutdown() - [state=%d, indexOnDiskIsValid=%d, " + "dontMarkIndexClean=%d, sanitize=%d]", + index->mState, index->mIndexOnDiskIsValid, index->mDontMarkIndexClean, + sanitize)); + + MOZ_ASSERT(index->mShuttingDown); + + EState oldState = index->mState; + index->ChangeState(SHUTDOWN, lock); + + if (oldState != READY) { + LOG( + ("CacheIndex::Shutdown() - Unexpected state. Did posting of " + "PreShutdownInternal() fail?")); + } + + switch (oldState) { + case WRITING: + index->FinishWrite(false, lock); + [[fallthrough]]; + case READY: + if (index->mIndexOnDiskIsValid && !index->mDontMarkIndexClean) { + if (!sanitize && NS_FAILED(index->WriteLogToDisk())) { + index->RemoveJournalAndTempFile(); + } + } else { + index->RemoveJournalAndTempFile(); + } + break; + case READING: + index->FinishRead(false, lock); + break; + case BUILDING: + case UPDATING: + index->FinishUpdate(false, lock); + break; + default: + MOZ_ASSERT(false, "Unexpected state!"); + } + + if (sanitize) { + index->RemoveAllIndexFiles(); + } + + return NS_OK; +} + +// static +nsresult CacheIndex::AddEntry(const SHA1Sum::Hash* aHash) { + LOG(("CacheIndex::AddEntry() [hash=%08x%08x%08x%08x%08x]", LOGSHA1(aHash))); + + MOZ_ASSERT(CacheFileIOManager::IsOnIOThread()); + + StaticMutexAutoLock lock(sLock); + + RefPtr<CacheIndex> index = gInstance; + + if (!index) { + return NS_ERROR_NOT_INITIALIZED; + } + + if (!index->IsIndexUsable()) { + return NS_ERROR_NOT_AVAILABLE; + } + + // Getters in CacheIndexStats assert when mStateLogged is true since the + // information is incomplete between calls to BeforeChange() and AfterChange() + // (i.e. while CacheIndexEntryAutoManage exists). We need to check whether + // non-fresh entries exists outside the scope of CacheIndexEntryAutoManage. + bool updateIfNonFreshEntriesExist = false; + + { + CacheIndexEntryAutoManage entryMng(aHash, index, lock); + + CacheIndexEntry* entry = index->mIndex.GetEntry(*aHash); + bool entryRemoved = entry && entry->IsRemoved(); + CacheIndexEntryUpdate* updated = nullptr; + + if (index->mState == READY || index->mState == UPDATING || + index->mState == BUILDING) { + MOZ_ASSERT(index->mPendingUpdates.Count() == 0); + + if (entry && !entryRemoved) { + // Found entry in index that shouldn't exist. + + if (entry->IsFresh()) { + // Someone removed the file on disk while FF is running. Update + // process can fix only non-fresh entries (i.e. entries that were not + // added within this session). Start update only if we have such + // entries. + // + // TODO: This should be very rare problem. If it turns out not to be + // true, change the update process so that it also iterates all + // initialized non-empty entries and checks whether the file exists. + + LOG( + ("CacheIndex::AddEntry() - Cache file was removed outside FF " + "process!")); + + updateIfNonFreshEntriesExist = true; + } else if (index->mState == READY) { + // Index is outdated, update it. + LOG( + ("CacheIndex::AddEntry() - Found entry that shouldn't exist, " + "update is needed")); + index->mIndexNeedsUpdate = true; + } else { + // We cannot be here when building index since all entries are fresh + // during building. + MOZ_ASSERT(index->mState == UPDATING); + } + } + + if (!entry) { + entry = index->mIndex.PutEntry(*aHash); + } + } else { // WRITING, READING + updated = index->mPendingUpdates.GetEntry(*aHash); + bool updatedRemoved = updated && updated->IsRemoved(); + + if ((updated && !updatedRemoved) || + (!updated && entry && !entryRemoved && entry->IsFresh())) { + // Fresh entry found, so the file was removed outside FF + LOG( + ("CacheIndex::AddEntry() - Cache file was removed outside FF " + "process!")); + + updateIfNonFreshEntriesExist = true; + } else if (!updated && entry && !entryRemoved) { + if (index->mState == WRITING) { + LOG( + ("CacheIndex::AddEntry() - Found entry that shouldn't exist, " + "update is needed")); + index->mIndexNeedsUpdate = true; + } + // Ignore if state is READING since the index information is partial + } + + updated = index->mPendingUpdates.PutEntry(*aHash); + } + + if (updated) { + updated->InitNew(); + updated->MarkDirty(); + updated->MarkFresh(); + } else { + entry->InitNew(); + entry->MarkDirty(); + entry->MarkFresh(); + } + } + + if (updateIfNonFreshEntriesExist && + index->mIndexStats.Count() != index->mIndexStats.Fresh()) { + index->mIndexNeedsUpdate = true; + } + + index->StartUpdatingIndexIfNeeded(lock); + index->WriteIndexToDiskIfNeeded(lock); + + return NS_OK; +} + +// static +nsresult CacheIndex::EnsureEntryExists(const SHA1Sum::Hash* aHash) { + LOG(("CacheIndex::EnsureEntryExists() [hash=%08x%08x%08x%08x%08x]", + LOGSHA1(aHash))); + + MOZ_ASSERT(CacheFileIOManager::IsOnIOThread()); + + StaticMutexAutoLock lock(sLock); + + RefPtr<CacheIndex> index = gInstance; + + if (!index) { + return NS_ERROR_NOT_INITIALIZED; + } + + if (!index->IsIndexUsable()) { + return NS_ERROR_NOT_AVAILABLE; + } + + { + CacheIndexEntryAutoManage entryMng(aHash, index, lock); + + CacheIndexEntry* entry = index->mIndex.GetEntry(*aHash); + bool entryRemoved = entry && entry->IsRemoved(); + + if (index->mState == READY || index->mState == UPDATING || + index->mState == BUILDING) { + MOZ_ASSERT(index->mPendingUpdates.Count() == 0); + + if (!entry || entryRemoved) { + if (entryRemoved && entry->IsFresh()) { + // This could happen only if somebody copies files to the entries + // directory while FF is running. + LOG( + ("CacheIndex::EnsureEntryExists() - Cache file was added outside " + "FF process! Update is needed.")); + index->mIndexNeedsUpdate = true; + } else if (index->mState == READY || + (entryRemoved && !entry->IsFresh())) { + // Removed non-fresh entries can be present as a result of + // MergeJournal() + LOG( + ("CacheIndex::EnsureEntryExists() - Didn't find entry that should" + " exist, update is needed")); + index->mIndexNeedsUpdate = true; + } + + if (!entry) { + entry = index->mIndex.PutEntry(*aHash); + } + entry->InitNew(); + entry->MarkDirty(); + } + entry->MarkFresh(); + } else { // WRITING, READING + CacheIndexEntryUpdate* updated = index->mPendingUpdates.GetEntry(*aHash); + bool updatedRemoved = updated && updated->IsRemoved(); + + if (updatedRemoved || (!updated && entryRemoved && entry->IsFresh())) { + // Fresh information about missing entry found. This could happen only + // if somebody copies files to the entries directory while FF is + // running. + LOG( + ("CacheIndex::EnsureEntryExists() - Cache file was added outside " + "FF process! Update is needed.")); + index->mIndexNeedsUpdate = true; + } else if (!updated && (!entry || entryRemoved)) { + if (index->mState == WRITING) { + LOG( + ("CacheIndex::EnsureEntryExists() - Didn't find entry that should" + " exist, update is needed")); + index->mIndexNeedsUpdate = true; + } + // Ignore if state is READING since the index information is partial + } + + // We don't need entryRemoved and updatedRemoved info anymore + if (entryRemoved) entry = nullptr; + if (updatedRemoved) updated = nullptr; + + if (updated) { + updated->MarkFresh(); + } else { + if (!entry) { + // Create a new entry + updated = index->mPendingUpdates.PutEntry(*aHash); + updated->InitNew(); + updated->MarkFresh(); + updated->MarkDirty(); + } else { + if (!entry->IsFresh()) { + // To mark the entry fresh we must make a copy of index entry + // since the index is read-only. + updated = index->mPendingUpdates.PutEntry(*aHash); + *updated = *entry; + updated->MarkFresh(); + } + } + } + } + } + + index->StartUpdatingIndexIfNeeded(lock); + index->WriteIndexToDiskIfNeeded(lock); + + return NS_OK; +} + +// static +nsresult CacheIndex::InitEntry(const SHA1Sum::Hash* aHash, + OriginAttrsHash aOriginAttrsHash, + bool aAnonymous, bool aPinned) { + LOG( + ("CacheIndex::InitEntry() [hash=%08x%08x%08x%08x%08x, " + "originAttrsHash=%" PRIx64 ", anonymous=%d, pinned=%d]", + LOGSHA1(aHash), aOriginAttrsHash, aAnonymous, aPinned)); + + MOZ_ASSERT(CacheFileIOManager::IsOnIOThread()); + + StaticMutexAutoLock lock(sLock); + + RefPtr<CacheIndex> index = gInstance; + + if (!index) { + return NS_ERROR_NOT_INITIALIZED; + } + + if (!index->IsIndexUsable()) { + return NS_ERROR_NOT_AVAILABLE; + } + + { + CacheIndexEntryAutoManage entryMng(aHash, index, lock); + + CacheIndexEntry* entry = index->mIndex.GetEntry(*aHash); + CacheIndexEntryUpdate* updated = nullptr; + bool reinitEntry = false; + + if (entry && entry->IsRemoved()) { + entry = nullptr; + } + + if (index->mState == READY || index->mState == UPDATING || + index->mState == BUILDING) { + MOZ_ASSERT(index->mPendingUpdates.Count() == 0); + MOZ_ASSERT(entry); + MOZ_ASSERT(entry->IsFresh()); + + if (!entry) { + LOG(("CacheIndex::InitEntry() - Entry was not found in mIndex!")); + NS_WARNING( + ("CacheIndex::InitEntry() - Entry was not found in mIndex!")); + return NS_ERROR_UNEXPECTED; + } + + if (IsCollision(entry, aOriginAttrsHash, aAnonymous)) { + index->mIndexNeedsUpdate = + true; // TODO Does this really help in case of collision? + reinitEntry = true; + } else { + if (entry->IsInitialized()) { + return NS_OK; + } + } + } else { + updated = index->mPendingUpdates.GetEntry(*aHash); + DebugOnly<bool> removed = updated && updated->IsRemoved(); + + MOZ_ASSERT(updated || !removed); + MOZ_ASSERT(updated || entry); + + if (!updated && !entry) { + LOG( + ("CacheIndex::InitEntry() - Entry was found neither in mIndex nor " + "in mPendingUpdates!")); + NS_WARNING( + ("CacheIndex::InitEntry() - Entry was found neither in " + "mIndex nor in mPendingUpdates!")); + return NS_ERROR_UNEXPECTED; + } + + if (updated) { + MOZ_ASSERT(updated->IsFresh()); + + if (IsCollision(updated, aOriginAttrsHash, aAnonymous)) { + index->mIndexNeedsUpdate = true; + reinitEntry = true; + } else { + if (updated->IsInitialized()) { + return NS_OK; + } + } + } else { + MOZ_ASSERT(entry->IsFresh()); + + if (IsCollision(entry, aOriginAttrsHash, aAnonymous)) { + index->mIndexNeedsUpdate = true; + reinitEntry = true; + } else { + if (entry->IsInitialized()) { + return NS_OK; + } + } + + // make a copy of a read-only entry + updated = index->mPendingUpdates.PutEntry(*aHash); + *updated = *entry; + } + } + + if (reinitEntry) { + // There is a collision and we are going to rewrite this entry. Initialize + // it as a new entry. + if (updated) { + updated->InitNew(); + updated->MarkFresh(); + } else { + entry->InitNew(); + entry->MarkFresh(); + } + } + + if (updated) { + updated->Init(aOriginAttrsHash, aAnonymous, aPinned); + updated->MarkDirty(); + } else { + entry->Init(aOriginAttrsHash, aAnonymous, aPinned); + entry->MarkDirty(); + } + } + + index->StartUpdatingIndexIfNeeded(lock); + index->WriteIndexToDiskIfNeeded(lock); + + return NS_OK; +} + +// static +nsresult CacheIndex::RemoveEntry(const SHA1Sum::Hash* aHash) { + LOG(("CacheIndex::RemoveEntry() [hash=%08x%08x%08x%08x%08x]", + LOGSHA1(aHash))); + + MOZ_ASSERT(CacheFileIOManager::IsOnIOThread()); + + StaticMutexAutoLock lock(sLock); + + RefPtr<CacheIndex> index = gInstance; + + if (!index) { + return NS_ERROR_NOT_INITIALIZED; + } + + if (!index->IsIndexUsable()) { + return NS_ERROR_NOT_AVAILABLE; + } + + { + CacheIndexEntryAutoManage entryMng(aHash, index, lock); + + CacheIndexEntry* entry = index->mIndex.GetEntry(*aHash); + bool entryRemoved = entry && entry->IsRemoved(); + + if (index->mState == READY || index->mState == UPDATING || + index->mState == BUILDING) { + MOZ_ASSERT(index->mPendingUpdates.Count() == 0); + + if (!entry || entryRemoved) { + if (entryRemoved && entry->IsFresh()) { + // This could happen only if somebody copies files to the entries + // directory while FF is running. + LOG( + ("CacheIndex::RemoveEntry() - Cache file was added outside FF " + "process! Update is needed.")); + index->mIndexNeedsUpdate = true; + } else if (index->mState == READY || + (entryRemoved && !entry->IsFresh())) { + // Removed non-fresh entries can be present as a result of + // MergeJournal() + LOG( + ("CacheIndex::RemoveEntry() - Didn't find entry that should exist" + ", update is needed")); + index->mIndexNeedsUpdate = true; + } + } else { + if (entry) { + if (!entry->IsDirty() && entry->IsFileEmpty()) { + index->mIndex.RemoveEntry(entry); + entry = nullptr; + } else { + entry->MarkRemoved(); + entry->MarkDirty(); + entry->MarkFresh(); + } + } + } + } else { // WRITING, READING + CacheIndexEntryUpdate* updated = index->mPendingUpdates.GetEntry(*aHash); + bool updatedRemoved = updated && updated->IsRemoved(); + + if (updatedRemoved || (!updated && entryRemoved && entry->IsFresh())) { + // Fresh information about missing entry found. This could happen only + // if somebody copies files to the entries directory while FF is + // running. + LOG( + ("CacheIndex::RemoveEntry() - Cache file was added outside FF " + "process! Update is needed.")); + index->mIndexNeedsUpdate = true; + } else if (!updated && (!entry || entryRemoved)) { + if (index->mState == WRITING) { + LOG( + ("CacheIndex::RemoveEntry() - Didn't find entry that should exist" + ", update is needed")); + index->mIndexNeedsUpdate = true; + } + // Ignore if state is READING since the index information is partial + } + + if (!updated) { + updated = index->mPendingUpdates.PutEntry(*aHash); + updated->InitNew(); + } + + updated->MarkRemoved(); + updated->MarkDirty(); + updated->MarkFresh(); + } + } + index->StartUpdatingIndexIfNeeded(lock); + index->WriteIndexToDiskIfNeeded(lock); + + return NS_OK; +} + +// static +nsresult CacheIndex::UpdateEntry(const SHA1Sum::Hash* aHash, + const uint32_t* aFrecency, + const bool* aHasAltData, + const uint16_t* aOnStartTime, + const uint16_t* aOnStopTime, + const uint8_t* aContentType, + const uint32_t* aSize) { + LOG( + ("CacheIndex::UpdateEntry() [hash=%08x%08x%08x%08x%08x, " + "frecency=%s, hasAltData=%s, onStartTime=%s, onStopTime=%s, " + "contentType=%s, size=%s]", + LOGSHA1(aHash), aFrecency ? nsPrintfCString("%u", *aFrecency).get() : "", + aHasAltData ? (*aHasAltData ? "true" : "false") : "", + aOnStartTime ? nsPrintfCString("%u", *aOnStartTime).get() : "", + aOnStopTime ? nsPrintfCString("%u", *aOnStopTime).get() : "", + aContentType ? nsPrintfCString("%u", *aContentType).get() : "", + aSize ? nsPrintfCString("%u", *aSize).get() : "")); + + MOZ_ASSERT(CacheFileIOManager::IsOnIOThread()); + + StaticMutexAutoLock lock(sLock); + + RefPtr<CacheIndex> index = gInstance; + + if (!index) { + return NS_ERROR_NOT_INITIALIZED; + } + + if (!index->IsIndexUsable()) { + return NS_ERROR_NOT_AVAILABLE; + } + + { + CacheIndexEntryAutoManage entryMng(aHash, index, lock); + + CacheIndexEntry* entry = index->mIndex.GetEntry(*aHash); + + if (entry && entry->IsRemoved()) { + entry = nullptr; + } + + if (index->mState == READY || index->mState == UPDATING || + index->mState == BUILDING) { + MOZ_ASSERT(index->mPendingUpdates.Count() == 0); + MOZ_ASSERT(entry); + + if (!entry) { + LOG(("CacheIndex::UpdateEntry() - Entry was not found in mIndex!")); + NS_WARNING( + ("CacheIndex::UpdateEntry() - Entry was not found in mIndex!")); + return NS_ERROR_UNEXPECTED; + } + + if (!HasEntryChanged(entry, aFrecency, aHasAltData, aOnStartTime, + aOnStopTime, aContentType, aSize)) { + return NS_OK; + } + + MOZ_ASSERT(entry->IsFresh()); + MOZ_ASSERT(entry->IsInitialized()); + entry->MarkDirty(); + + if (aFrecency) { + entry->SetFrecency(*aFrecency); + } + + if (aHasAltData) { + entry->SetHasAltData(*aHasAltData); + } + + if (aOnStartTime) { + entry->SetOnStartTime(*aOnStartTime); + } + + if (aOnStopTime) { + entry->SetOnStopTime(*aOnStopTime); + } + + if (aContentType) { + entry->SetContentType(*aContentType); + } + + if (aSize) { + entry->SetFileSize(*aSize); + } + } else { + CacheIndexEntryUpdate* updated = index->mPendingUpdates.GetEntry(*aHash); + DebugOnly<bool> removed = updated && updated->IsRemoved(); + + MOZ_ASSERT(updated || !removed); + MOZ_ASSERT(updated || entry); + + if (!updated) { + if (!entry) { + LOG( + ("CacheIndex::UpdateEntry() - Entry was found neither in mIndex " + "nor in mPendingUpdates!")); + NS_WARNING( + ("CacheIndex::UpdateEntry() - Entry was found neither in " + "mIndex nor in mPendingUpdates!")); + return NS_ERROR_UNEXPECTED; + } + + // make a copy of a read-only entry + updated = index->mPendingUpdates.PutEntry(*aHash); + *updated = *entry; + } + + MOZ_ASSERT(updated->IsFresh()); + MOZ_ASSERT(updated->IsInitialized()); + updated->MarkDirty(); + + if (aFrecency) { + updated->SetFrecency(*aFrecency); + } + + if (aHasAltData) { + updated->SetHasAltData(*aHasAltData); + } + + if (aOnStartTime) { + updated->SetOnStartTime(*aOnStartTime); + } + + if (aOnStopTime) { + updated->SetOnStopTime(*aOnStopTime); + } + + if (aContentType) { + updated->SetContentType(*aContentType); + } + + if (aSize) { + updated->SetFileSize(*aSize); + } + } + } + + index->WriteIndexToDiskIfNeeded(lock); + + return NS_OK; +} + +// static +nsresult CacheIndex::RemoveAll() { + LOG(("CacheIndex::RemoveAll()")); + + MOZ_ASSERT(CacheFileIOManager::IsOnIOThread()); + + nsCOMPtr<nsIFile> file; + + { + StaticMutexAutoLock lock(sLock); + + RefPtr<CacheIndex> index = gInstance; + + if (!index) { + return NS_ERROR_NOT_INITIALIZED; + } + + MOZ_ASSERT(!index->mRemovingAll); + + if (!index->IsIndexUsable()) { + return NS_ERROR_NOT_AVAILABLE; + } + + AutoRestore<bool> saveRemovingAll(index->mRemovingAll); + index->mRemovingAll = true; + + // Doom index and journal handles but don't null them out since this will be + // done in FinishWrite/FinishRead methods. + if (index->mIndexHandle) { + CacheFileIOManager::DoomFile(index->mIndexHandle, nullptr); + } else { + // We don't have a handle to index file, so get the file here, but delete + // it outside the lock. Ignore the result since this is not fatal. + index->GetFile(nsLiteralCString(INDEX_NAME), getter_AddRefs(file)); + } + + if (index->mJournalHandle) { + CacheFileIOManager::DoomFile(index->mJournalHandle, nullptr); + } + + switch (index->mState) { + case WRITING: + index->FinishWrite(false, lock); + break; + case READY: + // nothing to do + break; + case READING: + index->FinishRead(false, lock); + break; + case BUILDING: + case UPDATING: + index->FinishUpdate(false, lock); + break; + default: + MOZ_ASSERT(false, "Unexpected state!"); + } + + // We should end up in READY state + MOZ_ASSERT(index->mState == READY); + + // There should not be any handle + MOZ_ASSERT(!index->mIndexHandle); + MOZ_ASSERT(!index->mJournalHandle); + + index->mIndexOnDiskIsValid = false; + index->mIndexNeedsUpdate = false; + + index->mIndexStats.Clear(); + index->mFrecencyArray.Clear(lock); + index->mIndex.Clear(); + + for (uint32_t i = 0; i < index->mIterators.Length();) { + nsresult rv = index->mIterators[i]->CloseInternal(NS_ERROR_NOT_AVAILABLE); + if (NS_FAILED(rv)) { + // CacheIndexIterator::CloseInternal() removes itself from mIterators + // iff it returns success. + LOG( + ("CacheIndex::RemoveAll() - Failed to remove iterator %p. " + "[rv=0x%08" PRIx32 "]", + index->mIterators[i], static_cast<uint32_t>(rv))); + i++; + } + } + } + + if (file) { + // Ignore the result. The file might not exist and the failure is not fatal. + file->Remove(false); + } + + return NS_OK; +} + +// static +nsresult CacheIndex::HasEntry( + const nsACString& aKey, EntryStatus* _retval, + const std::function<void(const CacheIndexEntry*)>& aCB) { + LOG(("CacheIndex::HasEntry() [key=%s]", PromiseFlatCString(aKey).get())); + + SHA1Sum sum; + SHA1Sum::Hash hash; + sum.update(aKey.BeginReading(), aKey.Length()); + sum.finish(hash); + + return HasEntry(hash, _retval, aCB); +} + +// static +nsresult CacheIndex::HasEntry( + const SHA1Sum::Hash& hash, EntryStatus* _retval, + const std::function<void(const CacheIndexEntry*)>& aCB) { + StaticMutexAutoLock lock(sLock); + + RefPtr<CacheIndex> index = gInstance; + + if (!index) { + return NS_ERROR_NOT_INITIALIZED; + } + + if (!index->IsIndexUsable()) { + return NS_ERROR_NOT_AVAILABLE; + } + + const CacheIndexEntry* entry = nullptr; + + switch (index->mState) { + case READING: + case WRITING: + entry = index->mPendingUpdates.GetEntry(hash); + [[fallthrough]]; + case BUILDING: + case UPDATING: + case READY: + if (!entry) { + entry = index->mIndex.GetEntry(hash); + } + break; + case INITIAL: + case SHUTDOWN: + MOZ_ASSERT(false, "Unexpected state!"); + } + + if (!entry) { + if (index->mState == READY || index->mState == WRITING) { + *_retval = DOES_NOT_EXIST; + } else { + *_retval = DO_NOT_KNOW; + } + } else { + if (entry->IsRemoved()) { + if (entry->IsFresh()) { + *_retval = DOES_NOT_EXIST; + } else { + *_retval = DO_NOT_KNOW; + } + } else { + *_retval = EXISTS; + if (aCB) { + aCB(entry); + } + } + } + + LOG(("CacheIndex::HasEntry() - result is %u", *_retval)); + return NS_OK; +} + +// static +nsresult CacheIndex::GetEntryForEviction(bool aIgnoreEmptyEntries, + SHA1Sum::Hash* aHash, uint32_t* aCnt) { + LOG(("CacheIndex::GetEntryForEviction()")); + + MOZ_ASSERT(CacheFileIOManager::IsOnIOThread()); + + StaticMutexAutoLock lock(sLock); + + RefPtr<CacheIndex> index = gInstance; + + if (!index) return NS_ERROR_NOT_INITIALIZED; + + if (!index->IsIndexUsable()) { + return NS_ERROR_NOT_AVAILABLE; + } + + if (index->mIndexStats.Size() == 0) { + return NS_ERROR_NOT_AVAILABLE; + } + + int32_t mediaUsage = + round(static_cast<double>(index->mIndexStats.SizeByType( + nsICacheEntry::CONTENT_TYPE_MEDIA)) * + 100.0 / static_cast<double>(index->mIndexStats.Size())); + int32_t mediaUsageLimit = + StaticPrefs::browser_cache_disk_content_type_media_limit(); + bool evictMedia = false; + if (mediaUsage > mediaUsageLimit) { + LOG( + ("CacheIndex::GetEntryForEviction() - media content type is over the " + "limit [mediaUsage=%d, mediaUsageLimit=%d]", + mediaUsage, mediaUsageLimit)); + evictMedia = true; + } + + SHA1Sum::Hash hash; + CacheIndexRecord* foundRecord = nullptr; + uint32_t skipped = 0; + + // find first non-forced valid and unpinned entry with the lowest frecency + index->mFrecencyArray.SortIfNeeded(lock); + + for (auto iter = index->mFrecencyArray.Iter(); !iter.Done(); iter.Next()) { + CacheIndexRecord* rec = iter.Get()->Get(); + + memcpy(&hash, rec->mHash, sizeof(SHA1Sum::Hash)); + + ++skipped; + + if (evictMedia && CacheIndexEntry::GetContentType(rec) != + nsICacheEntry::CONTENT_TYPE_MEDIA) { + continue; + } + + if (IsForcedValidEntry(&hash)) { + continue; + } + + if (CacheIndexEntry::IsPinned(rec)) { + continue; + } + + if (aIgnoreEmptyEntries && !CacheIndexEntry::GetFileSize(*rec)) { + continue; + } + + --skipped; + foundRecord = rec; + break; + } + + if (!foundRecord) return NS_ERROR_NOT_AVAILABLE; + + *aCnt = skipped; + + LOG( + ("CacheIndex::GetEntryForEviction() - returning entry " + "[hash=%08x%08x%08x%08x%08x, cnt=%u, frecency=%u, contentType=%u]", + LOGSHA1(&hash), *aCnt, foundRecord->mFrecency, + CacheIndexEntry::GetContentType(foundRecord))); + + memcpy(aHash, &hash, sizeof(SHA1Sum::Hash)); + + return NS_OK; +} + +// static +bool CacheIndex::IsForcedValidEntry(const SHA1Sum::Hash* aHash) { + RefPtr<CacheFileHandle> handle; + + CacheFileIOManager::gInstance->mHandles.GetHandle(aHash, + getter_AddRefs(handle)); + + if (!handle) return false; + + nsCString hashKey = handle->Key(); + return CacheStorageService::Self()->IsForcedValidEntry(hashKey); +} + +// static +nsresult CacheIndex::GetCacheSize(uint32_t* _retval) { + LOG(("CacheIndex::GetCacheSize()")); + + StaticMutexAutoLock lock(sLock); + + RefPtr<CacheIndex> index = gInstance; + + if (!index) return NS_ERROR_NOT_INITIALIZED; + + if (!index->IsIndexUsable()) { + return NS_ERROR_NOT_AVAILABLE; + } + + *_retval = index->mIndexStats.Size(); + LOG(("CacheIndex::GetCacheSize() - returning %u", *_retval)); + return NS_OK; +} + +// static +nsresult CacheIndex::GetEntryFileCount(uint32_t* _retval) { + LOG(("CacheIndex::GetEntryFileCount()")); + + StaticMutexAutoLock lock(sLock); + + RefPtr<CacheIndex> index = gInstance; + + if (!index) { + return NS_ERROR_NOT_INITIALIZED; + } + + if (!index->IsIndexUsable()) { + return NS_ERROR_NOT_AVAILABLE; + } + + *_retval = index->mIndexStats.ActiveEntriesCount(); + LOG(("CacheIndex::GetEntryFileCount() - returning %u", *_retval)); + return NS_OK; +} + +// static +nsresult CacheIndex::GetCacheStats(nsILoadContextInfo* aInfo, uint32_t* aSize, + uint32_t* aCount) { + LOG(("CacheIndex::GetCacheStats() [info=%p]", aInfo)); + + StaticMutexAutoLock lock(sLock); + + RefPtr<CacheIndex> index = gInstance; + + if (!index) { + return NS_ERROR_NOT_INITIALIZED; + } + + if (!index->IsIndexUsable()) { + return NS_ERROR_NOT_AVAILABLE; + } + + *aSize = 0; + *aCount = 0; + + for (auto iter = index->mFrecencyArray.Iter(); !iter.Done(); iter.Next()) { + if (aInfo && + !CacheIndexEntry::RecordMatchesLoadContextInfo(iter.Get(), aInfo)) { + continue; + } + + *aSize += CacheIndexEntry::GetFileSize(*(iter.Get()->Get())); + ++*aCount; + } + + return NS_OK; +} + +// static +nsresult CacheIndex::AsyncGetDiskConsumption( + nsICacheStorageConsumptionObserver* aObserver) { + LOG(("CacheIndex::AsyncGetDiskConsumption()")); + + StaticMutexAutoLock lock(sLock); + + RefPtr<CacheIndex> index = gInstance; + + if (!index) { + return NS_ERROR_NOT_INITIALIZED; + } + + if (!index->IsIndexUsable()) { + return NS_ERROR_NOT_AVAILABLE; + } + + RefPtr<DiskConsumptionObserver> observer = + DiskConsumptionObserver::Init(aObserver); + + NS_ENSURE_ARG(observer); + + if ((index->mState == READY || index->mState == WRITING) && + !index->mAsyncGetDiskConsumptionBlocked) { + LOG(("CacheIndex::AsyncGetDiskConsumption - calling immediately")); + // Safe to call the callback under the lock, + // we always post to the main thread. + observer->OnDiskConsumption(index->mIndexStats.Size() << 10); + return NS_OK; + } + + LOG(("CacheIndex::AsyncGetDiskConsumption - remembering callback")); + // Will be called when the index get to the READY state. + index->mDiskConsumptionObservers.AppendElement(observer); + + // Move forward with index re/building if it is pending + RefPtr<CacheIOThread> ioThread = CacheFileIOManager::IOThread(); + if (ioThread) { + ioThread->Dispatch( + NS_NewRunnableFunction("net::CacheIndex::AsyncGetDiskConsumption", + []() -> void { + StaticMutexAutoLock lock(sLock); + + RefPtr<CacheIndex> index = gInstance; + if (index && index->mUpdateTimer) { + index->mUpdateTimer->Cancel(); + index->DelayedUpdateLocked(lock); + } + }), + CacheIOThread::INDEX); + } + + return NS_OK; +} + +// static +nsresult CacheIndex::GetIterator(nsILoadContextInfo* aInfo, bool aAddNew, + CacheIndexIterator** _retval) { + LOG(("CacheIndex::GetIterator() [info=%p, addNew=%d]", aInfo, aAddNew)); + + StaticMutexAutoLock lock(sLock); + + RefPtr<CacheIndex> index = gInstance; + + if (!index) { + return NS_ERROR_NOT_INITIALIZED; + } + + if (!index->IsIndexUsable()) { + return NS_ERROR_NOT_AVAILABLE; + } + + RefPtr<CacheIndexIterator> idxIter; + if (aInfo) { + idxIter = new CacheIndexContextIterator(index, aAddNew, aInfo); + } else { + idxIter = new CacheIndexIterator(index, aAddNew); + } + + index->mFrecencyArray.SortIfNeeded(lock); + + for (auto iter = index->mFrecencyArray.Iter(); !iter.Done(); iter.Next()) { + idxIter->AddRecord(iter.Get(), lock); + } + + index->mIterators.AppendElement(idxIter); + idxIter.swap(*_retval); + return NS_OK; +} + +// static +nsresult CacheIndex::IsUpToDate(bool* _retval) { + LOG(("CacheIndex::IsUpToDate()")); + + StaticMutexAutoLock lock(sLock); + + RefPtr<CacheIndex> index = gInstance; + + if (!index) { + return NS_ERROR_NOT_INITIALIZED; + } + + if (!index->IsIndexUsable()) { + return NS_ERROR_NOT_AVAILABLE; + } + + *_retval = (index->mState == READY || index->mState == WRITING) && + !index->mIndexNeedsUpdate && !index->mShuttingDown; + + LOG(("CacheIndex::IsUpToDate() - returning %d", *_retval)); + return NS_OK; +} + +bool CacheIndex::IsIndexUsable() { + MOZ_ASSERT(mState != INITIAL); + + switch (mState) { + case INITIAL: + case SHUTDOWN: + return false; + + case READING: + case WRITING: + case BUILDING: + case UPDATING: + case READY: + break; + } + + return true; +} + +// static +bool CacheIndex::IsCollision(CacheIndexEntry* aEntry, + OriginAttrsHash aOriginAttrsHash, + bool aAnonymous) { + if (!aEntry->IsInitialized()) { + return false; + } + + if (aEntry->Anonymous() != aAnonymous || + aEntry->OriginAttrsHash() != aOriginAttrsHash) { + LOG( + ("CacheIndex::IsCollision() - Collision detected for entry hash=%08x" + "%08x%08x%08x%08x, expected values: originAttrsHash=%" PRIu64 ", " + "anonymous=%d; actual values: originAttrsHash=%" PRIu64 + ", anonymous=%d]", + LOGSHA1(aEntry->Hash()), aOriginAttrsHash, aAnonymous, + aEntry->OriginAttrsHash(), aEntry->Anonymous())); + return true; + } + + return false; +} + +// static +bool CacheIndex::HasEntryChanged( + CacheIndexEntry* aEntry, const uint32_t* aFrecency, const bool* aHasAltData, + const uint16_t* aOnStartTime, const uint16_t* aOnStopTime, + const uint8_t* aContentType, const uint32_t* aSize) { + if (aFrecency && *aFrecency != aEntry->GetFrecency()) { + return true; + } + + if (aHasAltData && *aHasAltData != aEntry->GetHasAltData()) { + return true; + } + + if (aOnStartTime && *aOnStartTime != aEntry->GetOnStartTime()) { + return true; + } + + if (aOnStopTime && *aOnStopTime != aEntry->GetOnStopTime()) { + return true; + } + + if (aContentType && *aContentType != aEntry->GetContentType()) { + return true; + } + + if (aSize && + (*aSize & CacheIndexEntry::kFileSizeMask) != aEntry->GetFileSize()) { + return true; + } + + return false; +} + +void CacheIndex::ProcessPendingOperations( + const StaticMutexAutoLock& aProofOfLock) { + sLock.AssertCurrentThreadOwns(); + LOG(("CacheIndex::ProcessPendingOperations()")); + + for (auto iter = mPendingUpdates.Iter(); !iter.Done(); iter.Next()) { + CacheIndexEntryUpdate* update = iter.Get(); + + LOG(("CacheIndex::ProcessPendingOperations() [hash=%08x%08x%08x%08x%08x]", + LOGSHA1(update->Hash()))); + + MOZ_ASSERT(update->IsFresh()); + + CacheIndexEntry* entry = mIndex.GetEntry(*update->Hash()); + { + CacheIndexEntryAutoManage emng(update->Hash(), this, aProofOfLock); + emng.DoNotSearchInUpdates(); + + if (update->IsRemoved()) { + if (entry) { + if (entry->IsRemoved()) { + MOZ_ASSERT(entry->IsFresh()); + MOZ_ASSERT(entry->IsDirty()); + } else if (!entry->IsDirty() && entry->IsFileEmpty()) { + // Entries with empty file are not stored in index on disk. Just + // remove the entry, but only in case the entry is not dirty, i.e. + // the entry file was empty when we wrote the index. + mIndex.RemoveEntry(entry); + entry = nullptr; + } else { + entry->MarkRemoved(); + entry->MarkDirty(); + entry->MarkFresh(); + } + } + } else if (entry) { + // Some information in mIndex can be newer than in mPendingUpdates (see + // bug 1074832). This will copy just those values that were really + // updated. + update->ApplyUpdate(entry); + } else { + // There is no entry in mIndex, copy all information from + // mPendingUpdates to mIndex. + entry = mIndex.PutEntry(*update->Hash()); + *entry = *update; + } + } + iter.Remove(); + } + + MOZ_ASSERT(mPendingUpdates.Count() == 0); + + EnsureCorrectStats(); +} + +bool CacheIndex::WriteIndexToDiskIfNeeded( + const StaticMutexAutoLock& aProofOfLock) { + sLock.AssertCurrentThreadOwns(); + if (mState != READY || mShuttingDown || mRWPending) { + return false; + } + + if (!mLastDumpTime.IsNull() && + (TimeStamp::NowLoRes() - mLastDumpTime).ToMilliseconds() < + kMinDumpInterval) { + return false; + } + + if (mIndexStats.Dirty() < kMinUnwrittenChanges) { + return false; + } + + WriteIndexToDisk(aProofOfLock); + return true; +} + +void CacheIndex::WriteIndexToDisk(const StaticMutexAutoLock& aProofOfLock) { + sLock.AssertCurrentThreadOwns(); + LOG(("CacheIndex::WriteIndexToDisk()")); + mIndexStats.Log(); + + nsresult rv; + + MOZ_ASSERT(mState == READY); + MOZ_ASSERT(!mRWBuf); + MOZ_ASSERT(!mRWHash); + MOZ_ASSERT(!mRWPending); + + ChangeState(WRITING, aProofOfLock); + + mProcessEntries = mIndexStats.ActiveEntriesCount(); + + mIndexFileOpener = new FileOpenHelper(this); + rv = CacheFileIOManager::OpenFile( + nsLiteralCString(TEMP_INDEX_NAME), + CacheFileIOManager::SPECIAL_FILE | CacheFileIOManager::CREATE, + mIndexFileOpener); + if (NS_FAILED(rv)) { + LOG(("CacheIndex::WriteIndexToDisk() - Can't open file [rv=0x%08" PRIx32 + "]", + static_cast<uint32_t>(rv))); + FinishWrite(false, aProofOfLock); + return; + } + + // Write index header to a buffer, it will be written to disk together with + // records in WriteRecords() once we open the file successfully. + AllocBuffer(); + mRWHash = new CacheHash(); + + mRWBufPos = 0; + // index version + NetworkEndian::writeUint32(mRWBuf + mRWBufPos, kIndexVersion); + mRWBufPos += sizeof(uint32_t); + // timestamp + NetworkEndian::writeUint32(mRWBuf + mRWBufPos, + static_cast<uint32_t>(PR_Now() / PR_USEC_PER_SEC)); + mRWBufPos += sizeof(uint32_t); + // dirty flag + NetworkEndian::writeUint32(mRWBuf + mRWBufPos, 1); + mRWBufPos += sizeof(uint32_t); + // amount of data written to the cache + NetworkEndian::writeUint32(mRWBuf + mRWBufPos, + static_cast<uint32_t>(mTotalBytesWritten >> 10)); + mRWBufPos += sizeof(uint32_t); + + mSkipEntries = 0; +} + +void CacheIndex::WriteRecords(const StaticMutexAutoLock& aProofOfLock) { + sLock.AssertCurrentThreadOwns(); + LOG(("CacheIndex::WriteRecords()")); + + nsresult rv; + + MOZ_ASSERT(mState == WRITING); + MOZ_ASSERT(!mRWPending); + + int64_t fileOffset; + + if (mSkipEntries) { + MOZ_ASSERT(mRWBufPos == 0); + fileOffset = sizeof(CacheIndexHeader); + fileOffset += sizeof(CacheIndexRecord) * mSkipEntries; + } else { + MOZ_ASSERT(mRWBufPos == sizeof(CacheIndexHeader)); + fileOffset = 0; + } + uint32_t hashOffset = mRWBufPos; + + char* buf = mRWBuf + mRWBufPos; + uint32_t skip = mSkipEntries; + uint32_t processMax = (mRWBufSize - mRWBufPos) / sizeof(CacheIndexRecord); + MOZ_ASSERT(processMax != 0 || + mProcessEntries == + 0); // TODO make sure we can write an empty index + uint32_t processed = 0; +#ifdef DEBUG + bool hasMore = false; +#endif + for (auto iter = mIndex.Iter(); !iter.Done(); iter.Next()) { + CacheIndexEntry* entry = iter.Get(); + if (entry->IsRemoved() || !entry->IsInitialized() || entry->IsFileEmpty()) { + continue; + } + + if (skip) { + skip--; + continue; + } + + if (processed == processMax) { +#ifdef DEBUG + hasMore = true; +#endif + break; + } + + entry->WriteToBuf(buf); + buf += sizeof(CacheIndexRecord); + processed++; + } + + MOZ_ASSERT(mRWBufPos != static_cast<uint32_t>(buf - mRWBuf) || + mProcessEntries == 0); + mRWBufPos = buf - mRWBuf; + mSkipEntries += processed; + MOZ_ASSERT(mSkipEntries <= mProcessEntries); + + mRWHash->Update(mRWBuf + hashOffset, mRWBufPos - hashOffset); + + if (mSkipEntries == mProcessEntries) { + MOZ_ASSERT(!hasMore); + + // We've processed all records + if (mRWBufPos + sizeof(CacheHash::Hash32_t) > mRWBufSize) { + // realloc buffer to spare another write cycle + mRWBufSize = mRWBufPos + sizeof(CacheHash::Hash32_t); + mRWBuf = static_cast<char*>(moz_xrealloc(mRWBuf, mRWBufSize)); + } + + NetworkEndian::writeUint32(mRWBuf + mRWBufPos, mRWHash->GetHash()); + mRWBufPos += sizeof(CacheHash::Hash32_t); + } else { + MOZ_ASSERT(hasMore); + } + + rv = CacheFileIOManager::Write(mIndexHandle, fileOffset, mRWBuf, mRWBufPos, + mSkipEntries == mProcessEntries, false, this); + if (NS_FAILED(rv)) { + LOG( + ("CacheIndex::WriteRecords() - CacheFileIOManager::Write() failed " + "synchronously [rv=0x%08" PRIx32 "]", + static_cast<uint32_t>(rv))); + FinishWrite(false, aProofOfLock); + } else { + mRWPending = true; + } + + mRWBufPos = 0; +} + +void CacheIndex::FinishWrite(bool aSucceeded, + const StaticMutexAutoLock& aProofOfLock) { + sLock.AssertCurrentThreadOwns(); + LOG(("CacheIndex::FinishWrite() [succeeded=%d]", aSucceeded)); + + MOZ_ASSERT((!aSucceeded && mState == SHUTDOWN) || mState == WRITING); + + // If there is write operation pending we must be cancelling writing of the + // index when shutting down or removing the whole index. + MOZ_ASSERT(!mRWPending || (!aSucceeded && (mShuttingDown || mRemovingAll))); + + mIndexHandle = nullptr; + mRWHash = nullptr; + ReleaseBuffer(); + + if (aSucceeded) { + // Opening of the file must not be in progress if writing succeeded. + MOZ_ASSERT(!mIndexFileOpener); + + for (auto iter = mIndex.Iter(); !iter.Done(); iter.Next()) { + CacheIndexEntry* entry = iter.Get(); + + bool remove = false; + { + CacheIndexEntryAutoManage emng(entry->Hash(), this, aProofOfLock); + + if (entry->IsRemoved()) { + emng.DoNotSearchInIndex(); + remove = true; + } else if (entry->IsDirty()) { + entry->ClearDirty(); + } + } + if (remove) { + iter.Remove(); + } + } + + mIndexOnDiskIsValid = true; + } else { + if (mIndexFileOpener) { + // If opening of the file is still in progress (e.g. WRITE process was + // canceled by RemoveAll()) then we need to cancel the opener to make sure + // that OnFileOpenedInternal() won't be called. + mIndexFileOpener->Cancel(); + mIndexFileOpener = nullptr; + } + } + + ProcessPendingOperations(aProofOfLock); + mIndexStats.Log(); + + if (mState == WRITING) { + ChangeState(READY, aProofOfLock); + mLastDumpTime = TimeStamp::NowLoRes(); + } +} + +nsresult CacheIndex::GetFile(const nsACString& aName, nsIFile** _retval) { + nsresult rv; + + nsCOMPtr<nsIFile> file; + rv = mCacheDirectory->Clone(getter_AddRefs(file)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = file->AppendNative(aName); + NS_ENSURE_SUCCESS(rv, rv); + + file.swap(*_retval); + return NS_OK; +} + +void CacheIndex::RemoveFile(const nsACString& aName) { + MOZ_ASSERT(mState == SHUTDOWN); + + nsresult rv; + + nsCOMPtr<nsIFile> file; + rv = GetFile(aName, getter_AddRefs(file)); + NS_ENSURE_SUCCESS_VOID(rv); + + rv = file->Remove(false); + if (NS_FAILED(rv) && rv != NS_ERROR_FILE_NOT_FOUND) { + LOG( + ("CacheIndex::RemoveFile() - Cannot remove old entry file from disk " + "[rv=0x%08" PRIx32 ", name=%s]", + static_cast<uint32_t>(rv), PromiseFlatCString(aName).get())); + } +} + +void CacheIndex::RemoveAllIndexFiles() { + LOG(("CacheIndex::RemoveAllIndexFiles()")); + RemoveFile(nsLiteralCString(INDEX_NAME)); + RemoveJournalAndTempFile(); +} + +void CacheIndex::RemoveJournalAndTempFile() { + LOG(("CacheIndex::RemoveJournalAndTempFile()")); + RemoveFile(nsLiteralCString(TEMP_INDEX_NAME)); + RemoveFile(nsLiteralCString(JOURNAL_NAME)); +} + +class WriteLogHelper { + public: + explicit WriteLogHelper(PRFileDesc* aFD) + : mFD(aFD), mBufSize(kMaxBufSize), mBufPos(0) { + mHash = new CacheHash(); + mBuf = static_cast<char*>(moz_xmalloc(mBufSize)); + } + + ~WriteLogHelper() { free(mBuf); } + + nsresult AddEntry(CacheIndexEntry* aEntry); + nsresult Finish(); + + private: + nsresult FlushBuffer(); + + PRFileDesc* mFD; + char* mBuf; + uint32_t mBufSize; + int32_t mBufPos; + RefPtr<CacheHash> mHash; +}; + +nsresult WriteLogHelper::AddEntry(CacheIndexEntry* aEntry) { + nsresult rv; + + if (mBufPos + sizeof(CacheIndexRecord) > mBufSize) { + mHash->Update(mBuf, mBufPos); + + rv = FlushBuffer(); + NS_ENSURE_SUCCESS(rv, rv); + MOZ_ASSERT(mBufPos + sizeof(CacheIndexRecord) <= mBufSize); + } + + aEntry->WriteToBuf(mBuf + mBufPos); + mBufPos += sizeof(CacheIndexRecord); + + return NS_OK; +} + +nsresult WriteLogHelper::Finish() { + nsresult rv; + + mHash->Update(mBuf, mBufPos); + if (mBufPos + sizeof(CacheHash::Hash32_t) > mBufSize) { + rv = FlushBuffer(); + NS_ENSURE_SUCCESS(rv, rv); + MOZ_ASSERT(mBufPos + sizeof(CacheHash::Hash32_t) <= mBufSize); + } + + NetworkEndian::writeUint32(mBuf + mBufPos, mHash->GetHash()); + mBufPos += sizeof(CacheHash::Hash32_t); + + rv = FlushBuffer(); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult WriteLogHelper::FlushBuffer() { + if (CacheObserver::IsPastShutdownIOLag()) { + LOG(("WriteLogHelper::FlushBuffer() - Interrupting writing journal.")); + return NS_ERROR_FAILURE; + } + + int32_t bytesWritten = PR_Write(mFD, mBuf, mBufPos); + + if (bytesWritten != mBufPos) { + return NS_ERROR_FAILURE; + } + + mBufPos = 0; + return NS_OK; +} + +nsresult CacheIndex::WriteLogToDisk() { + LOG(("CacheIndex::WriteLogToDisk()")); + + nsresult rv; + + MOZ_ASSERT(mPendingUpdates.Count() == 0); + MOZ_ASSERT(mState == SHUTDOWN); + + if (CacheObserver::IsPastShutdownIOLag()) { + LOG(("CacheIndex::WriteLogToDisk() - Skipping writing journal.")); + return NS_ERROR_FAILURE; + } + + RemoveFile(nsLiteralCString(TEMP_INDEX_NAME)); + + nsCOMPtr<nsIFile> indexFile; + rv = GetFile(nsLiteralCString(INDEX_NAME), getter_AddRefs(indexFile)); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr<nsIFile> logFile; + rv = GetFile(nsLiteralCString(JOURNAL_NAME), getter_AddRefs(logFile)); + NS_ENSURE_SUCCESS(rv, rv); + + mIndexStats.Log(); + + PRFileDesc* fd = nullptr; + rv = logFile->OpenNSPRFileDesc(PR_RDWR | PR_CREATE_FILE | PR_TRUNCATE, 0600, + &fd); + NS_ENSURE_SUCCESS(rv, rv); + + WriteLogHelper wlh(fd); + for (auto iter = mIndex.Iter(); !iter.Done(); iter.Next()) { + CacheIndexEntry* entry = iter.Get(); + if (entry->IsRemoved() || entry->IsDirty()) { + rv = wlh.AddEntry(entry); + if (NS_WARN_IF(NS_FAILED(rv))) { + return rv; + } + } + } + + rv = wlh.Finish(); + PR_Close(fd); + NS_ENSURE_SUCCESS(rv, rv); + + rv = indexFile->OpenNSPRFileDesc(PR_RDWR, 0600, &fd); + NS_ENSURE_SUCCESS(rv, rv); + + // Seek to dirty flag in the index header and clear it. + static_assert(2 * sizeof(uint32_t) == offsetof(CacheIndexHeader, mIsDirty), + "Unexpected offset of CacheIndexHeader::mIsDirty"); + int64_t offset = PR_Seek64(fd, 2 * sizeof(uint32_t), PR_SEEK_SET); + if (offset == -1) { + PR_Close(fd); + return NS_ERROR_FAILURE; + } + + uint32_t isDirty = 0; + int32_t bytesWritten = PR_Write(fd, &isDirty, sizeof(isDirty)); + PR_Close(fd); + if (bytesWritten != sizeof(isDirty)) { + return NS_ERROR_FAILURE; + } + + return NS_OK; +} + +void CacheIndex::ReadIndexFromDisk(const StaticMutexAutoLock& aProofOfLock) { + sLock.AssertCurrentThreadOwns(); + LOG(("CacheIndex::ReadIndexFromDisk()")); + + nsresult rv; + + MOZ_ASSERT(mState == INITIAL); + + ChangeState(READING, aProofOfLock); + + mIndexFileOpener = new FileOpenHelper(this); + rv = CacheFileIOManager::OpenFile( + nsLiteralCString(INDEX_NAME), + CacheFileIOManager::SPECIAL_FILE | CacheFileIOManager::OPEN, + mIndexFileOpener); + if (NS_FAILED(rv)) { + LOG( + ("CacheIndex::ReadIndexFromDisk() - CacheFileIOManager::OpenFile() " + "failed [rv=0x%08" PRIx32 ", file=%s]", + static_cast<uint32_t>(rv), INDEX_NAME)); + FinishRead(false, aProofOfLock); + return; + } + + mJournalFileOpener = new FileOpenHelper(this); + rv = CacheFileIOManager::OpenFile( + nsLiteralCString(JOURNAL_NAME), + CacheFileIOManager::SPECIAL_FILE | CacheFileIOManager::OPEN, + mJournalFileOpener); + if (NS_FAILED(rv)) { + LOG( + ("CacheIndex::ReadIndexFromDisk() - CacheFileIOManager::OpenFile() " + "failed [rv=0x%08" PRIx32 ", file=%s]", + static_cast<uint32_t>(rv), JOURNAL_NAME)); + FinishRead(false, aProofOfLock); + } + + mTmpFileOpener = new FileOpenHelper(this); + rv = CacheFileIOManager::OpenFile( + nsLiteralCString(TEMP_INDEX_NAME), + CacheFileIOManager::SPECIAL_FILE | CacheFileIOManager::OPEN, + mTmpFileOpener); + if (NS_FAILED(rv)) { + LOG( + ("CacheIndex::ReadIndexFromDisk() - CacheFileIOManager::OpenFile() " + "failed [rv=0x%08" PRIx32 ", file=%s]", + static_cast<uint32_t>(rv), TEMP_INDEX_NAME)); + FinishRead(false, aProofOfLock); + } +} + +void CacheIndex::StartReadingIndex(const StaticMutexAutoLock& aProofOfLock) { + sLock.AssertCurrentThreadOwns(); + LOG(("CacheIndex::StartReadingIndex()")); + + nsresult rv; + + MOZ_ASSERT(mIndexHandle); + MOZ_ASSERT(mState == READING); + MOZ_ASSERT(!mIndexOnDiskIsValid); + MOZ_ASSERT(!mDontMarkIndexClean); + MOZ_ASSERT(!mJournalReadSuccessfully); + MOZ_ASSERT(mIndexHandle->FileSize() >= 0); + MOZ_ASSERT(!mRWPending); + + int64_t entriesSize = mIndexHandle->FileSize() - sizeof(CacheIndexHeader) - + sizeof(CacheHash::Hash32_t); + + if (entriesSize < 0 || entriesSize % sizeof(CacheIndexRecord)) { + LOG(("CacheIndex::StartReadingIndex() - Index is corrupted")); + FinishRead(false, aProofOfLock); + return; + } + + AllocBuffer(); + mSkipEntries = 0; + mRWHash = new CacheHash(); + + mRWBufPos = + std::min(mRWBufSize, static_cast<uint32_t>(mIndexHandle->FileSize())); + + rv = CacheFileIOManager::Read(mIndexHandle, 0, mRWBuf, mRWBufPos, this); + if (NS_FAILED(rv)) { + LOG( + ("CacheIndex::StartReadingIndex() - CacheFileIOManager::Read() failed " + "synchronously [rv=0x%08" PRIx32 "]", + static_cast<uint32_t>(rv))); + FinishRead(false, aProofOfLock); + } else { + mRWPending = true; + } +} + +void CacheIndex::ParseRecords(const StaticMutexAutoLock& aProofOfLock) { + sLock.AssertCurrentThreadOwns(); + LOG(("CacheIndex::ParseRecords()")); + + nsresult rv; + + MOZ_ASSERT(!mRWPending); + + uint32_t entryCnt = (mIndexHandle->FileSize() - sizeof(CacheIndexHeader) - + sizeof(CacheHash::Hash32_t)) / + sizeof(CacheIndexRecord); + uint32_t pos = 0; + + if (!mSkipEntries) { + if (NetworkEndian::readUint32(mRWBuf + pos) != kIndexVersion) { + FinishRead(false, aProofOfLock); + return; + } + pos += sizeof(uint32_t); + + mIndexTimeStamp = NetworkEndian::readUint32(mRWBuf + pos); + pos += sizeof(uint32_t); + + if (NetworkEndian::readUint32(mRWBuf + pos)) { + if (mJournalHandle) { + CacheFileIOManager::DoomFile(mJournalHandle, nullptr); + mJournalHandle = nullptr; + } + } else { + uint32_t* isDirty = + reinterpret_cast<uint32_t*>(moz_xmalloc(sizeof(uint32_t))); + NetworkEndian::writeUint32(isDirty, 1); + + // Mark index dirty. The buffer is freed by CacheFileIOManager when + // nullptr is passed as the listener and the call doesn't fail + // synchronously. + rv = CacheFileIOManager::Write(mIndexHandle, 2 * sizeof(uint32_t), + reinterpret_cast<char*>(isDirty), + sizeof(uint32_t), true, false, nullptr); + if (NS_FAILED(rv)) { + // This is not fatal, just free the memory + free(isDirty); + } + } + pos += sizeof(uint32_t); + + uint64_t dataWritten = NetworkEndian::readUint32(mRWBuf + pos); + pos += sizeof(uint32_t); + dataWritten <<= 10; + mTotalBytesWritten += dataWritten; + } + + uint32_t hashOffset = pos; + + while (pos + sizeof(CacheIndexRecord) <= mRWBufPos && + mSkipEntries != entryCnt) { + CacheIndexRecord* rec = reinterpret_cast<CacheIndexRecord*>(mRWBuf + pos); + CacheIndexEntry tmpEntry(&rec->mHash); + tmpEntry.ReadFromBuf(mRWBuf + pos); + + if (tmpEntry.IsDirty() || !tmpEntry.IsInitialized() || + tmpEntry.IsFileEmpty() || tmpEntry.IsFresh() || tmpEntry.IsRemoved()) { + LOG( + ("CacheIndex::ParseRecords() - Invalid entry found in index, removing" + " whole index [dirty=%d, initialized=%d, fileEmpty=%d, fresh=%d, " + "removed=%d]", + tmpEntry.IsDirty(), tmpEntry.IsInitialized(), tmpEntry.IsFileEmpty(), + tmpEntry.IsFresh(), tmpEntry.IsRemoved())); + FinishRead(false, aProofOfLock); + return; + } + + CacheIndexEntryAutoManage emng(tmpEntry.Hash(), this, aProofOfLock); + + CacheIndexEntry* entry = mIndex.PutEntry(*tmpEntry.Hash()); + *entry = tmpEntry; + + pos += sizeof(CacheIndexRecord); + mSkipEntries++; + } + + mRWHash->Update(mRWBuf + hashOffset, pos - hashOffset); + + if (pos != mRWBufPos) { + memmove(mRWBuf, mRWBuf + pos, mRWBufPos - pos); + } + + mRWBufPos -= pos; + pos = 0; + + int64_t fileOffset = sizeof(CacheIndexHeader) + + mSkipEntries * sizeof(CacheIndexRecord) + mRWBufPos; + + MOZ_ASSERT(fileOffset <= mIndexHandle->FileSize()); + if (fileOffset == mIndexHandle->FileSize()) { + uint32_t expectedHash = NetworkEndian::readUint32(mRWBuf); + if (mRWHash->GetHash() != expectedHash) { + LOG(("CacheIndex::ParseRecords() - Hash mismatch, [is %x, should be %x]", + mRWHash->GetHash(), expectedHash)); + FinishRead(false, aProofOfLock); + return; + } + + mIndexOnDiskIsValid = true; + mJournalReadSuccessfully = false; + + if (mJournalHandle) { + StartReadingJournal(aProofOfLock); + } else { + FinishRead(false, aProofOfLock); + } + + return; + } + + pos = mRWBufPos; + uint32_t toRead = + std::min(mRWBufSize - pos, + static_cast<uint32_t>(mIndexHandle->FileSize() - fileOffset)); + mRWBufPos = pos + toRead; + + rv = CacheFileIOManager::Read(mIndexHandle, fileOffset, mRWBuf + pos, toRead, + this); + if (NS_FAILED(rv)) { + LOG( + ("CacheIndex::ParseRecords() - CacheFileIOManager::Read() failed " + "synchronously [rv=0x%08" PRIx32 "]", + static_cast<uint32_t>(rv))); + FinishRead(false, aProofOfLock); + return; + } + mRWPending = true; +} + +void CacheIndex::StartReadingJournal(const StaticMutexAutoLock& aProofOfLock) { + sLock.AssertCurrentThreadOwns(); + LOG(("CacheIndex::StartReadingJournal()")); + + nsresult rv; + + MOZ_ASSERT(mJournalHandle); + MOZ_ASSERT(mIndexOnDiskIsValid); + MOZ_ASSERT(mTmpJournal.Count() == 0); + MOZ_ASSERT(mJournalHandle->FileSize() >= 0); + MOZ_ASSERT(!mRWPending); + + int64_t entriesSize = + mJournalHandle->FileSize() - sizeof(CacheHash::Hash32_t); + + if (entriesSize < 0 || entriesSize % sizeof(CacheIndexRecord)) { + LOG(("CacheIndex::StartReadingJournal() - Journal is corrupted")); + FinishRead(false, aProofOfLock); + return; + } + + mSkipEntries = 0; + mRWHash = new CacheHash(); + + mRWBufPos = + std::min(mRWBufSize, static_cast<uint32_t>(mJournalHandle->FileSize())); + + rv = CacheFileIOManager::Read(mJournalHandle, 0, mRWBuf, mRWBufPos, this); + if (NS_FAILED(rv)) { + LOG( + ("CacheIndex::StartReadingJournal() - CacheFileIOManager::Read() failed" + " synchronously [rv=0x%08" PRIx32 "]", + static_cast<uint32_t>(rv))); + FinishRead(false, aProofOfLock); + } else { + mRWPending = true; + } +} + +void CacheIndex::ParseJournal(const StaticMutexAutoLock& aProofOfLock) { + sLock.AssertCurrentThreadOwns(); + LOG(("CacheIndex::ParseJournal()")); + + nsresult rv; + + MOZ_ASSERT(!mRWPending); + + uint32_t entryCnt = + (mJournalHandle->FileSize() - sizeof(CacheHash::Hash32_t)) / + sizeof(CacheIndexRecord); + + uint32_t pos = 0; + + while (pos + sizeof(CacheIndexRecord) <= mRWBufPos && + mSkipEntries != entryCnt) { + CacheIndexEntry tmpEntry(reinterpret_cast<SHA1Sum::Hash*>(mRWBuf + pos)); + tmpEntry.ReadFromBuf(mRWBuf + pos); + + CacheIndexEntry* entry = mTmpJournal.PutEntry(*tmpEntry.Hash()); + *entry = tmpEntry; + + if (entry->IsDirty() || entry->IsFresh()) { + LOG( + ("CacheIndex::ParseJournal() - Invalid entry found in journal, " + "ignoring whole journal [dirty=%d, fresh=%d]", + entry->IsDirty(), entry->IsFresh())); + FinishRead(false, aProofOfLock); + return; + } + + pos += sizeof(CacheIndexRecord); + mSkipEntries++; + } + + mRWHash->Update(mRWBuf, pos); + + if (pos != mRWBufPos) { + memmove(mRWBuf, mRWBuf + pos, mRWBufPos - pos); + } + + mRWBufPos -= pos; + pos = 0; + + int64_t fileOffset = mSkipEntries * sizeof(CacheIndexRecord) + mRWBufPos; + + MOZ_ASSERT(fileOffset <= mJournalHandle->FileSize()); + if (fileOffset == mJournalHandle->FileSize()) { + uint32_t expectedHash = NetworkEndian::readUint32(mRWBuf); + if (mRWHash->GetHash() != expectedHash) { + LOG(("CacheIndex::ParseJournal() - Hash mismatch, [is %x, should be %x]", + mRWHash->GetHash(), expectedHash)); + FinishRead(false, aProofOfLock); + return; + } + + mJournalReadSuccessfully = true; + FinishRead(true, aProofOfLock); + return; + } + + pos = mRWBufPos; + uint32_t toRead = + std::min(mRWBufSize - pos, + static_cast<uint32_t>(mJournalHandle->FileSize() - fileOffset)); + mRWBufPos = pos + toRead; + + rv = CacheFileIOManager::Read(mJournalHandle, fileOffset, mRWBuf + pos, + toRead, this); + if (NS_FAILED(rv)) { + LOG( + ("CacheIndex::ParseJournal() - CacheFileIOManager::Read() failed " + "synchronously [rv=0x%08" PRIx32 "]", + static_cast<uint32_t>(rv))); + FinishRead(false, aProofOfLock); + return; + } + mRWPending = true; +} + +void CacheIndex::MergeJournal(const StaticMutexAutoLock& aProofOfLock) { + sLock.AssertCurrentThreadOwns(); + LOG(("CacheIndex::MergeJournal()")); + + for (auto iter = mTmpJournal.Iter(); !iter.Done(); iter.Next()) { + CacheIndexEntry* entry = iter.Get(); + + LOG(("CacheIndex::MergeJournal() [hash=%08x%08x%08x%08x%08x]", + LOGSHA1(entry->Hash()))); + + CacheIndexEntry* entry2 = mIndex.GetEntry(*entry->Hash()); + { + CacheIndexEntryAutoManage emng(entry->Hash(), this, aProofOfLock); + if (entry->IsRemoved()) { + if (entry2) { + entry2->MarkRemoved(); + entry2->MarkDirty(); + } + } else { + if (!entry2) { + entry2 = mIndex.PutEntry(*entry->Hash()); + } + + *entry2 = *entry; + entry2->MarkDirty(); + } + } + iter.Remove(); + } + + MOZ_ASSERT(mTmpJournal.Count() == 0); +} + +void CacheIndex::EnsureNoFreshEntry() { +#ifdef DEBUG_STATS + CacheIndexStats debugStats; + debugStats.DisableLogging(); + for (auto iter = mIndex.Iter(); !iter.Done(); iter.Next()) { + debugStats.BeforeChange(nullptr); + debugStats.AfterChange(iter.Get()); + } + MOZ_ASSERT(debugStats.Fresh() == 0); +#endif +} + +void CacheIndex::EnsureCorrectStats() { +#ifdef DEBUG_STATS + MOZ_ASSERT(mPendingUpdates.Count() == 0); + CacheIndexStats debugStats; + debugStats.DisableLogging(); + for (auto iter = mIndex.Iter(); !iter.Done(); iter.Next()) { + debugStats.BeforeChange(nullptr); + debugStats.AfterChange(iter.Get()); + } + MOZ_ASSERT(debugStats == mIndexStats); +#endif +} + +void CacheIndex::FinishRead(bool aSucceeded, + const StaticMutexAutoLock& aProofOfLock) { + sLock.AssertCurrentThreadOwns(); + LOG(("CacheIndex::FinishRead() [succeeded=%d]", aSucceeded)); + + MOZ_ASSERT((!aSucceeded && mState == SHUTDOWN) || mState == READING); + + MOZ_ASSERT( + // -> rebuild + (!aSucceeded && !mIndexOnDiskIsValid && !mJournalReadSuccessfully) || + // -> update + (!aSucceeded && mIndexOnDiskIsValid && !mJournalReadSuccessfully) || + // -> ready + (aSucceeded && mIndexOnDiskIsValid && mJournalReadSuccessfully)); + + // If there is read operation pending we must be cancelling reading of the + // index when shutting down or removing the whole index. + MOZ_ASSERT(!mRWPending || (!aSucceeded && (mShuttingDown || mRemovingAll))); + + if (mState == SHUTDOWN) { + RemoveFile(nsLiteralCString(TEMP_INDEX_NAME)); + RemoveFile(nsLiteralCString(JOURNAL_NAME)); + } else { + if (mIndexHandle && !mIndexOnDiskIsValid) { + CacheFileIOManager::DoomFile(mIndexHandle, nullptr); + } + + if (mJournalHandle) { + CacheFileIOManager::DoomFile(mJournalHandle, nullptr); + } + } + + if (mIndexFileOpener) { + mIndexFileOpener->Cancel(); + mIndexFileOpener = nullptr; + } + if (mJournalFileOpener) { + mJournalFileOpener->Cancel(); + mJournalFileOpener = nullptr; + } + if (mTmpFileOpener) { + mTmpFileOpener->Cancel(); + mTmpFileOpener = nullptr; + } + + mIndexHandle = nullptr; + mJournalHandle = nullptr; + mRWHash = nullptr; + ReleaseBuffer(); + + if (mState == SHUTDOWN) { + return; + } + + if (!mIndexOnDiskIsValid) { + MOZ_ASSERT(mTmpJournal.Count() == 0); + EnsureNoFreshEntry(); + ProcessPendingOperations(aProofOfLock); + // Remove all entries that we haven't seen during this session + RemoveNonFreshEntries(aProofOfLock); + StartUpdatingIndex(true, aProofOfLock); + return; + } + + if (!mJournalReadSuccessfully) { + mTmpJournal.Clear(); + EnsureNoFreshEntry(); + ProcessPendingOperations(aProofOfLock); + StartUpdatingIndex(false, aProofOfLock); + return; + } + + MergeJournal(aProofOfLock); + EnsureNoFreshEntry(); + ProcessPendingOperations(aProofOfLock); + mIndexStats.Log(); + + ChangeState(READY, aProofOfLock); + mLastDumpTime = TimeStamp::NowLoRes(); // Do not dump new index immediately +} + +// static +void CacheIndex::DelayedUpdate(nsITimer* aTimer, void* aClosure) { + LOG(("CacheIndex::DelayedUpdate()")); + + StaticMutexAutoLock lock(sLock); + RefPtr<CacheIndex> index = gInstance; + + if (!index) { + return; + } + + index->DelayedUpdateLocked(lock); +} + +// static +void CacheIndex::DelayedUpdateLocked(const StaticMutexAutoLock& aProofOfLock) { + sLock.AssertCurrentThreadOwns(); + LOG(("CacheIndex::DelayedUpdateLocked()")); + + nsresult rv; + + mUpdateTimer = nullptr; + + if (!IsIndexUsable()) { + return; + } + + if (mState == READY && mShuttingDown) { + return; + } + + // mUpdateEventPending must be false here since StartUpdatingIndex() won't + // schedule timer if it is true. + MOZ_ASSERT(!mUpdateEventPending); + if (mState != BUILDING && mState != UPDATING) { + LOG(("CacheIndex::DelayedUpdateLocked() - Update was canceled")); + return; + } + + // We need to redispatch to run with lower priority + RefPtr<CacheIOThread> ioThread = CacheFileIOManager::IOThread(); + MOZ_ASSERT(ioThread); + + mUpdateEventPending = true; + rv = ioThread->Dispatch(this, CacheIOThread::INDEX); + if (NS_FAILED(rv)) { + mUpdateEventPending = false; + NS_WARNING("CacheIndex::DelayedUpdateLocked() - Can't dispatch event"); + LOG(("CacheIndex::DelayedUpdate() - Can't dispatch event")); + FinishUpdate(false, aProofOfLock); + } +} + +nsresult CacheIndex::ScheduleUpdateTimer(uint32_t aDelay) { + LOG(("CacheIndex::ScheduleUpdateTimer() [delay=%u]", aDelay)); + + MOZ_ASSERT(!mUpdateTimer); + + nsCOMPtr<nsIEventTarget> ioTarget = CacheFileIOManager::IOTarget(); + MOZ_ASSERT(ioTarget); + + return NS_NewTimerWithFuncCallback( + getter_AddRefs(mUpdateTimer), CacheIndex::DelayedUpdate, nullptr, aDelay, + nsITimer::TYPE_ONE_SHOT, "net::CacheIndex::ScheduleUpdateTimer", + ioTarget); +} + +nsresult CacheIndex::SetupDirectoryEnumerator() { + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_ASSERT(!mDirEnumerator); + + nsresult rv; + nsCOMPtr<nsIFile> file; + + rv = mCacheDirectory->Clone(getter_AddRefs(file)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = file->AppendNative(nsLiteralCString(ENTRIES_DIR)); + NS_ENSURE_SUCCESS(rv, rv); + + bool exists; + rv = file->Exists(&exists); + NS_ENSURE_SUCCESS(rv, rv); + + if (!exists) { + NS_WARNING( + "CacheIndex::SetupDirectoryEnumerator() - Entries directory " + "doesn't exist!"); + LOG( + ("CacheIndex::SetupDirectoryEnumerator() - Entries directory doesn't " + "exist!")); + return NS_ERROR_UNEXPECTED; + } + + // Do not do IO under the lock. + nsCOMPtr<nsIDirectoryEnumerator> dirEnumerator; + { + StaticMutexAutoUnlock unlock(sLock); + rv = file->GetDirectoryEntries(getter_AddRefs(dirEnumerator)); + } + mDirEnumerator = dirEnumerator.forget(); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult CacheIndex::InitEntryFromDiskData(CacheIndexEntry* aEntry, + CacheFileMetadata* aMetaData, + int64_t aFileSize) { + nsresult rv; + + aEntry->InitNew(); + aEntry->MarkDirty(); + aEntry->MarkFresh(); + + aEntry->Init(GetOriginAttrsHash(aMetaData->OriginAttributes()), + aMetaData->IsAnonymous(), aMetaData->Pinned()); + + aEntry->SetFrecency(aMetaData->GetFrecency()); + + const char* altData = aMetaData->GetElement(CacheFileUtils::kAltDataKey); + bool hasAltData = altData != nullptr; + if (hasAltData && NS_FAILED(CacheFileUtils::ParseAlternativeDataInfo( + altData, nullptr, nullptr))) { + return NS_ERROR_FAILURE; + } + aEntry->SetHasAltData(hasAltData); + + static auto toUint16 = [](const char* aUint16String) -> uint16_t { + if (!aUint16String) { + return kIndexTimeNotAvailable; + } + nsresult rv; + uint64_t n64 = nsDependentCString(aUint16String).ToInteger64(&rv); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + return n64 <= kIndexTimeOutOfBound ? n64 : kIndexTimeOutOfBound; + }; + + aEntry->SetOnStartTime( + toUint16(aMetaData->GetElement("net-response-time-onstart"))); + aEntry->SetOnStopTime( + toUint16(aMetaData->GetElement("net-response-time-onstop"))); + + const char* contentTypeStr = aMetaData->GetElement("ctid"); + uint8_t contentType = nsICacheEntry::CONTENT_TYPE_UNKNOWN; + if (contentTypeStr) { + int64_t n64 = nsDependentCString(contentTypeStr).ToInteger64(&rv); + if (NS_FAILED(rv) || n64 < nsICacheEntry::CONTENT_TYPE_UNKNOWN || + n64 >= nsICacheEntry::CONTENT_TYPE_LAST) { + n64 = nsICacheEntry::CONTENT_TYPE_UNKNOWN; + } + contentType = n64; + } + aEntry->SetContentType(contentType); + + aEntry->SetFileSize(static_cast<uint32_t>(std::min( + static_cast<int64_t>(PR_UINT32_MAX), (aFileSize + 0x3FF) >> 10))); + return NS_OK; +} + +bool CacheIndex::IsUpdatePending() { + sLock.AssertCurrentThreadOwns(); + + return mUpdateTimer || mUpdateEventPending; +} + +void CacheIndex::BuildIndex(const StaticMutexAutoLock& aProofOfLock) { + sLock.AssertCurrentThreadOwns(); + LOG(("CacheIndex::BuildIndex()")); + + MOZ_ASSERT(mPendingUpdates.Count() == 0); + + nsresult rv; + + if (!mDirEnumerator) { + rv = SetupDirectoryEnumerator(); + if (mState == SHUTDOWN) { + // The index was shut down while we released the lock. FinishUpdate() was + // already called from Shutdown(), so just simply return here. + return; + } + + if (NS_FAILED(rv)) { + FinishUpdate(false, aProofOfLock); + return; + } + } + + while (true) { + if (CacheIOThread::YieldAndRerun()) { + LOG(( + "CacheIndex::BuildIndex() - Breaking loop for higher level events.")); + mUpdateEventPending = true; + return; + } + + bool fileExists = false; + nsCOMPtr<nsIFile> file; + { + // Do not do IO under the lock. + nsCOMPtr<nsIDirectoryEnumerator> dirEnumerator(mDirEnumerator); + sLock.AssertCurrentThreadOwns(); + StaticMutexAutoUnlock unlock(sLock); + rv = dirEnumerator->GetNextFile(getter_AddRefs(file)); + + if (file) { + file->Exists(&fileExists); + } + } + if (mState == SHUTDOWN) { + return; + } + if (!file) { + FinishUpdate(NS_SUCCEEDED(rv), aProofOfLock); + return; + } + + nsAutoCString leaf; + rv = file->GetNativeLeafName(leaf); + if (NS_FAILED(rv)) { + LOG( + ("CacheIndex::BuildIndex() - GetNativeLeafName() failed! Skipping " + "file.")); + mDontMarkIndexClean = true; + continue; + } + + if (!fileExists) { + LOG( + ("CacheIndex::BuildIndex() - File returned by the iterator was " + "removed in the meantime [name=%s]", + leaf.get())); + continue; + } + + SHA1Sum::Hash hash; + rv = CacheFileIOManager::StrToHash(leaf, &hash); + if (NS_FAILED(rv)) { + LOG( + ("CacheIndex::BuildIndex() - Filename is not a hash, removing file. " + "[name=%s]", + leaf.get())); + file->Remove(false); + continue; + } + + CacheIndexEntry* entry = mIndex.GetEntry(hash); + if (entry && entry->IsRemoved()) { + LOG( + ("CacheIndex::BuildIndex() - Found file that should not exist. " + "[name=%s]", + leaf.get())); + entry->Log(); + MOZ_ASSERT(entry->IsFresh()); + entry = nullptr; + } + +#ifdef DEBUG + RefPtr<CacheFileHandle> handle; + CacheFileIOManager::gInstance->mHandles.GetHandle(&hash, + getter_AddRefs(handle)); +#endif + + if (entry) { + // the entry is up to date + LOG( + ("CacheIndex::BuildIndex() - Skipping file because the entry is up to" + " date. [name=%s]", + leaf.get())); + entry->Log(); + MOZ_ASSERT(entry->IsFresh()); // The entry must be from this session + // there must be an active CacheFile if the entry is not initialized + MOZ_ASSERT(entry->IsInitialized() || handle); + continue; + } + + MOZ_ASSERT(!handle); + + RefPtr<CacheFileMetadata> meta = new CacheFileMetadata(); + int64_t size = 0; + + { + // Do not do IO under the lock. + StaticMutexAutoUnlock unlock(sLock); + rv = meta->SyncReadMetadata(file); + + if (NS_SUCCEEDED(rv)) { + rv = file->GetFileSize(&size); + if (NS_FAILED(rv)) { + LOG( + ("CacheIndex::BuildIndex() - Cannot get filesize of file that was" + " successfully parsed. [name=%s]", + leaf.get())); + } + } + } + if (mState == SHUTDOWN) { + return; + } + + // Nobody could add the entry while the lock was released since we modify + // the index only on IO thread and this loop is executed on IO thread too. + entry = mIndex.GetEntry(hash); + MOZ_ASSERT(!entry || entry->IsRemoved()); + + if (NS_FAILED(rv)) { + LOG( + ("CacheIndex::BuildIndex() - CacheFileMetadata::SyncReadMetadata() " + "failed, removing file. [name=%s]", + leaf.get())); + file->Remove(false); + } else { + CacheIndexEntryAutoManage entryMng(&hash, this, aProofOfLock); + entry = mIndex.PutEntry(hash); + if (NS_FAILED(InitEntryFromDiskData(entry, meta, size))) { + LOG( + ("CacheIndex::BuildIndex() - CacheFile::InitEntryFromDiskData() " + "failed, removing file. [name=%s]", + leaf.get())); + file->Remove(false); + entry->MarkRemoved(); + } else { + LOG(("CacheIndex::BuildIndex() - Added entry to index. [name=%s]", + leaf.get())); + entry->Log(); + } + } + } + + MOZ_ASSERT_UNREACHABLE("We should never get here"); +} + +bool CacheIndex::StartUpdatingIndexIfNeeded( + const StaticMutexAutoLock& aProofOfLock, bool aSwitchingToReadyState) { + sLock.AssertCurrentThreadOwns(); + // Start updating process when we are in or we are switching to READY state + // and index needs update, but not during shutdown or when removing all + // entries. + if ((mState == READY || aSwitchingToReadyState) && mIndexNeedsUpdate && + !mShuttingDown && !mRemovingAll) { + LOG(("CacheIndex::StartUpdatingIndexIfNeeded() - starting update process")); + mIndexNeedsUpdate = false; + StartUpdatingIndex(false, aProofOfLock); + return true; + } + + return false; +} + +void CacheIndex::StartUpdatingIndex(bool aRebuild, + const StaticMutexAutoLock& aProofOfLock) { + sLock.AssertCurrentThreadOwns(); + LOG(("CacheIndex::StartUpdatingIndex() [rebuild=%d]", aRebuild)); + + nsresult rv; + + mIndexStats.Log(); + + ChangeState(aRebuild ? BUILDING : UPDATING, aProofOfLock); + mDontMarkIndexClean = false; + + if (mShuttingDown || mRemovingAll) { + FinishUpdate(false, aProofOfLock); + return; + } + + if (IsUpdatePending()) { + LOG(("CacheIndex::StartUpdatingIndex() - Update is already pending")); + return; + } + + uint32_t elapsed = (TimeStamp::NowLoRes() - mStartTime).ToMilliseconds(); + if (elapsed < kUpdateIndexStartDelay) { + LOG( + ("CacheIndex::StartUpdatingIndex() - %u ms elapsed since startup, " + "scheduling timer to fire in %u ms.", + elapsed, kUpdateIndexStartDelay - elapsed)); + rv = ScheduleUpdateTimer(kUpdateIndexStartDelay - elapsed); + if (NS_SUCCEEDED(rv)) { + return; + } + + LOG( + ("CacheIndex::StartUpdatingIndex() - ScheduleUpdateTimer() failed. " + "Starting update immediately.")); + } else { + LOG( + ("CacheIndex::StartUpdatingIndex() - %u ms elapsed since startup, " + "starting update now.", + elapsed)); + } + + RefPtr<CacheIOThread> ioThread = CacheFileIOManager::IOThread(); + MOZ_ASSERT(ioThread); + + // We need to dispatch an event even if we are on IO thread since we need to + // update the index with the correct priority. + mUpdateEventPending = true; + rv = ioThread->Dispatch(this, CacheIOThread::INDEX); + if (NS_FAILED(rv)) { + mUpdateEventPending = false; + NS_WARNING("CacheIndex::StartUpdatingIndex() - Can't dispatch event"); + LOG(("CacheIndex::StartUpdatingIndex() - Can't dispatch event")); + FinishUpdate(false, aProofOfLock); + } +} + +void CacheIndex::UpdateIndex(const StaticMutexAutoLock& aProofOfLock) { + sLock.AssertCurrentThreadOwns(); + LOG(("CacheIndex::UpdateIndex()")); + + MOZ_ASSERT(mPendingUpdates.Count() == 0); + sLock.AssertCurrentThreadOwns(); + + nsresult rv; + + if (!mDirEnumerator) { + rv = SetupDirectoryEnumerator(); + if (mState == SHUTDOWN) { + // The index was shut down while we released the lock. FinishUpdate() was + // already called from Shutdown(), so just simply return here. + return; + } + + if (NS_FAILED(rv)) { + FinishUpdate(false, aProofOfLock); + return; + } + } + + while (true) { + if (CacheIOThread::YieldAndRerun()) { + LOG( + ("CacheIndex::UpdateIndex() - Breaking loop for higher level " + "events.")); + mUpdateEventPending = true; + return; + } + + bool fileExists = false; + nsCOMPtr<nsIFile> file; + { + // Do not do IO under the lock. + nsCOMPtr<nsIDirectoryEnumerator> dirEnumerator(mDirEnumerator); + StaticMutexAutoUnlock unlock(sLock); + rv = dirEnumerator->GetNextFile(getter_AddRefs(file)); + + if (file) { + file->Exists(&fileExists); + } + } + if (mState == SHUTDOWN) { + return; + } + if (!file) { + FinishUpdate(NS_SUCCEEDED(rv), aProofOfLock); + return; + } + + nsAutoCString leaf; + rv = file->GetNativeLeafName(leaf); + if (NS_FAILED(rv)) { + LOG( + ("CacheIndex::UpdateIndex() - GetNativeLeafName() failed! Skipping " + "file.")); + mDontMarkIndexClean = true; + continue; + } + + if (!fileExists) { + LOG( + ("CacheIndex::UpdateIndex() - File returned by the iterator was " + "removed in the meantime [name=%s]", + leaf.get())); + continue; + } + + SHA1Sum::Hash hash; + rv = CacheFileIOManager::StrToHash(leaf, &hash); + if (NS_FAILED(rv)) { + LOG( + ("CacheIndex::UpdateIndex() - Filename is not a hash, removing file. " + "[name=%s]", + leaf.get())); + file->Remove(false); + continue; + } + + CacheIndexEntry* entry = mIndex.GetEntry(hash); + if (entry && entry->IsRemoved()) { + if (entry->IsFresh()) { + LOG( + ("CacheIndex::UpdateIndex() - Found file that should not exist. " + "[name=%s]", + leaf.get())); + entry->Log(); + } + entry = nullptr; + } + +#ifdef DEBUG + RefPtr<CacheFileHandle> handle; + CacheFileIOManager::gInstance->mHandles.GetHandle(&hash, + getter_AddRefs(handle)); +#endif + + if (entry && entry->IsFresh()) { + // the entry is up to date + LOG( + ("CacheIndex::UpdateIndex() - Skipping file because the entry is up " + " to date. [name=%s]", + leaf.get())); + entry->Log(); + // there must be an active CacheFile if the entry is not initialized + MOZ_ASSERT(entry->IsInitialized() || handle); + continue; + } + + MOZ_ASSERT(!handle); + + if (entry) { + PRTime lastModifiedTime; + { + // Do not do IO under the lock. + StaticMutexAutoUnlock unlock(sLock); + rv = file->GetLastModifiedTime(&lastModifiedTime); + } + if (mState == SHUTDOWN) { + return; + } + if (NS_FAILED(rv)) { + LOG( + ("CacheIndex::UpdateIndex() - Cannot get lastModifiedTime. " + "[name=%s]", + leaf.get())); + // Assume the file is newer than index + } else { + if (mIndexTimeStamp > (lastModifiedTime / PR_MSEC_PER_SEC)) { + LOG( + ("CacheIndex::UpdateIndex() - Skipping file because of last " + "modified time. [name=%s, indexTimeStamp=%" PRIu32 ", " + "lastModifiedTime=%" PRId64 "]", + leaf.get(), mIndexTimeStamp, + lastModifiedTime / PR_MSEC_PER_SEC)); + + CacheIndexEntryAutoManage entryMng(&hash, this, aProofOfLock); + entry->MarkFresh(); + continue; + } + } + } + + RefPtr<CacheFileMetadata> meta = new CacheFileMetadata(); + int64_t size = 0; + + { + // Do not do IO under the lock. + StaticMutexAutoUnlock unlock(sLock); + rv = meta->SyncReadMetadata(file); + + if (NS_SUCCEEDED(rv)) { + rv = file->GetFileSize(&size); + if (NS_FAILED(rv)) { + LOG( + ("CacheIndex::UpdateIndex() - Cannot get filesize of file that " + "was successfully parsed. [name=%s]", + leaf.get())); + } + } + } + if (mState == SHUTDOWN) { + return; + } + + // Nobody could add the entry while the lock was released since we modify + // the index only on IO thread and this loop is executed on IO thread too. + entry = mIndex.GetEntry(hash); + MOZ_ASSERT(!entry || !entry->IsFresh()); + + CacheIndexEntryAutoManage entryMng(&hash, this, aProofOfLock); + + if (NS_FAILED(rv)) { + LOG( + ("CacheIndex::UpdateIndex() - CacheFileMetadata::SyncReadMetadata() " + "failed, removing file. [name=%s]", + leaf.get())); + } else { + entry = mIndex.PutEntry(hash); + rv = InitEntryFromDiskData(entry, meta, size); + if (NS_FAILED(rv)) { + LOG( + ("CacheIndex::UpdateIndex() - CacheIndex::InitEntryFromDiskData " + "failed, removing file. [name=%s]", + leaf.get())); + } + } + + if (NS_FAILED(rv)) { + file->Remove(false); + if (entry) { + entry->MarkRemoved(); + entry->MarkFresh(); + entry->MarkDirty(); + } + } else { + LOG( + ("CacheIndex::UpdateIndex() - Added/updated entry to/in index. " + "[name=%s]", + leaf.get())); + entry->Log(); + } + } + + MOZ_ASSERT_UNREACHABLE("We should never get here"); +} + +void CacheIndex::FinishUpdate(bool aSucceeded, + const StaticMutexAutoLock& aProofOfLock) { + LOG(("CacheIndex::FinishUpdate() [succeeded=%d]", aSucceeded)); + + MOZ_ASSERT(mState == UPDATING || mState == BUILDING || + (!aSucceeded && mState == SHUTDOWN)); + + if (mDirEnumerator) { + if (NS_IsMainThread()) { + LOG( + ("CacheIndex::FinishUpdate() - posting of PreShutdownInternal failed?" + " Cannot safely release mDirEnumerator, leaking it!")); + NS_WARNING(("CacheIndex::FinishUpdate() - Leaking mDirEnumerator!")); + // This can happen only in case dispatching event to IO thread failed in + // CacheIndex::PreShutdown(). + Unused << mDirEnumerator.forget(); // Leak it since dir enumerator is not + // threadsafe + } else { + mDirEnumerator->Close(); + mDirEnumerator = nullptr; + } + } + + if (!aSucceeded) { + mDontMarkIndexClean = true; + } + + if (mState == SHUTDOWN) { + return; + } + + if (mState == UPDATING && aSucceeded) { + // If we've iterated over all entries successfully then all entries that + // really exist on the disk are now marked as fresh. All non-fresh entries + // don't exist anymore and must be removed from the index. + RemoveNonFreshEntries(aProofOfLock); + } + + // Make sure we won't start update. If the build or update failed, there is no + // reason to believe that it will succeed next time. + mIndexNeedsUpdate = false; + + ChangeState(READY, aProofOfLock); + mLastDumpTime = TimeStamp::NowLoRes(); // Do not dump new index immediately +} + +void CacheIndex::RemoveNonFreshEntries( + const StaticMutexAutoLock& aProofOfLock) { + sLock.AssertCurrentThreadOwns(); + for (auto iter = mIndex.Iter(); !iter.Done(); iter.Next()) { + CacheIndexEntry* entry = iter.Get(); + if (entry->IsFresh()) { + continue; + } + + LOG( + ("CacheIndex::RemoveNonFreshEntries() - Removing entry. " + "[hash=%08x%08x%08x%08x%08x]", + LOGSHA1(entry->Hash()))); + + { + CacheIndexEntryAutoManage emng(entry->Hash(), this, aProofOfLock); + emng.DoNotSearchInIndex(); + } + + iter.Remove(); + } +} + +// static +char const* CacheIndex::StateString(EState aState) { + switch (aState) { + case INITIAL: + return "INITIAL"; + case READING: + return "READING"; + case WRITING: + return "WRITING"; + case BUILDING: + return "BUILDING"; + case UPDATING: + return "UPDATING"; + case READY: + return "READY"; + case SHUTDOWN: + return "SHUTDOWN"; + } + + MOZ_ASSERT(false, "Unexpected state!"); + return "?"; +} + +void CacheIndex::ChangeState(EState aNewState, + const StaticMutexAutoLock& aProofOfLock) { + sLock.AssertCurrentThreadOwns(); + LOG(("CacheIndex::ChangeState() changing state %s -> %s", StateString(mState), + StateString(aNewState))); + + // All pending updates should be processed before changing state + MOZ_ASSERT(mPendingUpdates.Count() == 0); + + // PreShutdownInternal() should change the state to READY from every state. It + // may go through different states, but once we are in READY state the only + // possible transition is to SHUTDOWN state. + MOZ_ASSERT(!mShuttingDown || mState != READY || aNewState == SHUTDOWN); + + // Start updating process when switching to READY state if needed + if (aNewState == READY && StartUpdatingIndexIfNeeded(aProofOfLock, true)) { + return; + } + + // Try to evict entries over limit everytime we're leaving state READING, + // BUILDING or UPDATING, but not during shutdown or when removing all + // entries. + if (!mShuttingDown && !mRemovingAll && aNewState != SHUTDOWN && + (mState == READING || mState == BUILDING || mState == UPDATING)) { + CacheFileIOManager::EvictIfOverLimit(); + } + + mState = aNewState; + + if (mState != SHUTDOWN) { + CacheFileIOManager::CacheIndexStateChanged(); + } + + NotifyAsyncGetDiskConsumptionCallbacks(); +} + +void CacheIndex::NotifyAsyncGetDiskConsumptionCallbacks() { + if ((mState == READY || mState == WRITING) && + !mAsyncGetDiskConsumptionBlocked && mDiskConsumptionObservers.Length()) { + for (uint32_t i = 0; i < mDiskConsumptionObservers.Length(); ++i) { + DiskConsumptionObserver* o = mDiskConsumptionObservers[i]; + // Safe to call under the lock. We always post to the main thread. + o->OnDiskConsumption(mIndexStats.Size() << 10); + } + + mDiskConsumptionObservers.Clear(); + } +} + +void CacheIndex::AllocBuffer() { + switch (mState) { + case WRITING: + mRWBufSize = sizeof(CacheIndexHeader) + sizeof(CacheHash::Hash32_t) + + mProcessEntries * sizeof(CacheIndexRecord); + if (mRWBufSize > kMaxBufSize) { + mRWBufSize = kMaxBufSize; + } + break; + case READING: + mRWBufSize = kMaxBufSize; + break; + default: + MOZ_ASSERT(false, "Unexpected state!"); + } + + mRWBuf = static_cast<char*>(moz_xmalloc(mRWBufSize)); +} + +void CacheIndex::ReleaseBuffer() { + sLock.AssertCurrentThreadOwns(); + + if (!mRWBuf || mRWPending) { + return; + } + + LOG(("CacheIndex::ReleaseBuffer() releasing buffer")); + + free(mRWBuf); + mRWBuf = nullptr; + mRWBufSize = 0; + mRWBufPos = 0; +} + +void CacheIndex::FrecencyArray::AppendRecord( + CacheIndexRecordWrapper* aRecord, const StaticMutexAutoLock& aProofOfLock) { + sLock.AssertCurrentThreadOwns(); + LOG( + ("CacheIndex::FrecencyArray::AppendRecord() [record=%p, hash=%08x%08x%08x" + "%08x%08x]", + aRecord, LOGSHA1(aRecord->Get()->mHash))); + + MOZ_DIAGNOSTIC_ASSERT(!mRecs.Contains(aRecord)); + mRecs.AppendElement(aRecord); + + // If the new frecency is 0, the element should be at the end of the array, + // i.e. this change doesn't affect order of the array + if (aRecord->Get()->mFrecency != 0) { + ++mUnsortedElements; + } +} + +void CacheIndex::FrecencyArray::RemoveRecord( + CacheIndexRecordWrapper* aRecord, const StaticMutexAutoLock& aProofOfLock) { + sLock.AssertCurrentThreadOwns(); + LOG(("CacheIndex::FrecencyArray::RemoveRecord() [record=%p]", aRecord)); + + decltype(mRecs)::index_type idx; + idx = mRecs.IndexOf(aRecord); + MOZ_RELEASE_ASSERT(idx != mRecs.NoIndex); + // sanity check to ensure correct record removal + MOZ_RELEASE_ASSERT(mRecs[idx] == aRecord); + mRecs[idx] = nullptr; + ++mRemovedElements; + + // Calling SortIfNeeded ensures that we get rid of removed elements in the + // array once we hit the limit. + SortIfNeeded(aProofOfLock); +} + +void CacheIndex::FrecencyArray::ReplaceRecord( + CacheIndexRecordWrapper* aOldRecord, CacheIndexRecordWrapper* aNewRecord, + const StaticMutexAutoLock& aProofOfLock) { + sLock.AssertCurrentThreadOwns(); + LOG( + ("CacheIndex::FrecencyArray::ReplaceRecord() [oldRecord=%p, " + "newRecord=%p]", + aOldRecord, aNewRecord)); + + decltype(mRecs)::index_type idx; + idx = mRecs.IndexOf(aOldRecord); + MOZ_RELEASE_ASSERT(idx != mRecs.NoIndex); + // sanity check to ensure correct record replaced + MOZ_RELEASE_ASSERT(mRecs[idx] == aOldRecord); + mRecs[idx] = aNewRecord; +} + +void CacheIndex::FrecencyArray::SortIfNeeded( + const StaticMutexAutoLock& aProofOfLock) { + sLock.AssertCurrentThreadOwns(); + const uint32_t kMaxUnsortedCount = 512; + const uint32_t kMaxUnsortedPercent = 10; + const uint32_t kMaxRemovedCount = 512; + + uint32_t unsortedLimit = std::min<uint32_t>( + kMaxUnsortedCount, Length() * kMaxUnsortedPercent / 100); + + if (mUnsortedElements > unsortedLimit || + mRemovedElements > kMaxRemovedCount) { + LOG( + ("CacheIndex::FrecencyArray::SortIfNeeded() - Sorting array " + "[unsortedElements=%u, unsortedLimit=%u, removedElements=%u, " + "maxRemovedCount=%u]", + mUnsortedElements, unsortedLimit, mRemovedElements, kMaxRemovedCount)); + + mRecs.Sort(FrecencyComparator()); + mUnsortedElements = 0; + if (mRemovedElements) { +#if defined(EARLY_BETA_OR_EARLIER) + // validate only null items are removed + for (uint32_t i = Length(); i < mRecs.Length(); ++i) { + MOZ_DIAGNOSTIC_ASSERT(!mRecs[i]); + } +#endif + // Removed elements are at the end after sorting. + mRecs.RemoveElementsAt(Length(), mRemovedElements); + mRemovedElements = 0; + } + } +} + +bool CacheIndex::FrecencyArray::RecordExistedUnlocked( + CacheIndexRecordWrapper* aRecord) { + return mRecs.Contains(aRecord); +} + +void CacheIndex::AddRecordToIterators(CacheIndexRecordWrapper* aRecord, + const StaticMutexAutoLock& aProofOfLock) { + sLock.AssertCurrentThreadOwns(); + for (uint32_t i = 0; i < mIterators.Length(); ++i) { + // Add a new record only when iterator is supposed to be updated. + if (mIterators[i]->ShouldBeNewAdded()) { + mIterators[i]->AddRecord(aRecord, aProofOfLock); + } + } +} + +void CacheIndex::RemoveRecordFromIterators( + CacheIndexRecordWrapper* aRecord, const StaticMutexAutoLock& aProofOfLock) { + sLock.AssertCurrentThreadOwns(); + for (uint32_t i = 0; i < mIterators.Length(); ++i) { + // Remove the record from iterator always, it makes no sence to return + // non-existing entries. Also the pointer to the record is no longer valid + // once the entry is removed from index. + mIterators[i]->RemoveRecord(aRecord, aProofOfLock); + } +} + +void CacheIndex::ReplaceRecordInIterators( + CacheIndexRecordWrapper* aOldRecord, CacheIndexRecordWrapper* aNewRecord, + const StaticMutexAutoLock& aProofOfLock) { + sLock.AssertCurrentThreadOwns(); + for (uint32_t i = 0; i < mIterators.Length(); ++i) { + // We have to replace the record always since the pointer is no longer + // valid after this point. NOTE: Replacing the record doesn't mean that + // a new entry was added, it just means that the data in the entry was + // changed (e.g. a file size) and we had to track this change in + // mPendingUpdates since mIndex was read-only. + mIterators[i]->ReplaceRecord(aOldRecord, aNewRecord, aProofOfLock); + } +} + +nsresult CacheIndex::Run() { + LOG(("CacheIndex::Run()")); + + StaticMutexAutoLock lock(sLock); + + if (!IsIndexUsable()) { + return NS_ERROR_NOT_AVAILABLE; + } + + if (mState == READY && mShuttingDown) { + return NS_OK; + } + + mUpdateEventPending = false; + + switch (mState) { + case BUILDING: + BuildIndex(lock); + break; + case UPDATING: + UpdateIndex(lock); + break; + default: + LOG(("CacheIndex::Run() - Update/Build was canceled")); + } + + return NS_OK; +} + +void CacheIndex::OnFileOpenedInternal(FileOpenHelper* aOpener, + CacheFileHandle* aHandle, + nsresult aResult, + const StaticMutexAutoLock& aProofOfLock) { + sLock.AssertCurrentThreadOwns(); + LOG( + ("CacheIndex::OnFileOpenedInternal() [opener=%p, handle=%p, " + "result=0x%08" PRIx32 "]", + aOpener, aHandle, static_cast<uint32_t>(aResult))); + MOZ_ASSERT(CacheFileIOManager::IsOnIOThread()); + + nsresult rv; + + MOZ_RELEASE_ASSERT(IsIndexUsable()); + + if (mState == READY && mShuttingDown) { + return; + } + + switch (mState) { + case WRITING: + MOZ_ASSERT(aOpener == mIndexFileOpener); + mIndexFileOpener = nullptr; + + if (NS_FAILED(aResult)) { + LOG( + ("CacheIndex::OnFileOpenedInternal() - Can't open index file for " + "writing [rv=0x%08" PRIx32 "]", + static_cast<uint32_t>(aResult))); + FinishWrite(false, aProofOfLock); + } else { + mIndexHandle = aHandle; + WriteRecords(aProofOfLock); + } + break; + case READING: + if (aOpener == mIndexFileOpener) { + mIndexFileOpener = nullptr; + + if (NS_SUCCEEDED(aResult)) { + if (aHandle->FileSize() == 0) { + FinishRead(false, aProofOfLock); + CacheFileIOManager::DoomFile(aHandle, nullptr); + break; + } + mIndexHandle = aHandle; + } else { + FinishRead(false, aProofOfLock); + break; + } + } else if (aOpener == mJournalFileOpener) { + mJournalFileOpener = nullptr; + mJournalHandle = aHandle; + } else if (aOpener == mTmpFileOpener) { + mTmpFileOpener = nullptr; + mTmpHandle = aHandle; + } else { + MOZ_ASSERT(false, "Unexpected state!"); + } + + if (mIndexFileOpener || mJournalFileOpener || mTmpFileOpener) { + // Some opener still didn't finish + break; + } + + // We fail and cancel all other openers when we opening index file fails. + MOZ_ASSERT(mIndexHandle); + + if (mTmpHandle) { + CacheFileIOManager::DoomFile(mTmpHandle, nullptr); + mTmpHandle = nullptr; + + if (mJournalHandle) { // this shouldn't normally happen + LOG( + ("CacheIndex::OnFileOpenedInternal() - Unexpected state, all " + "files [%s, %s, %s] should never exist. Removing whole index.", + INDEX_NAME, JOURNAL_NAME, TEMP_INDEX_NAME)); + FinishRead(false, aProofOfLock); + break; + } + } + + if (mJournalHandle) { + // Rename journal to make sure we update index on next start in case + // firefox crashes + rv = CacheFileIOManager::RenameFile( + mJournalHandle, nsLiteralCString(TEMP_INDEX_NAME), this); + if (NS_FAILED(rv)) { + LOG( + ("CacheIndex::OnFileOpenedInternal() - CacheFileIOManager::" + "RenameFile() failed synchronously [rv=0x%08" PRIx32 "]", + static_cast<uint32_t>(rv))); + FinishRead(false, aProofOfLock); + break; + } + } else { + StartReadingIndex(aProofOfLock); + } + + break; + default: + MOZ_ASSERT(false, "Unexpected state!"); + } +} + +nsresult CacheIndex::OnFileOpened(CacheFileHandle* aHandle, nsresult aResult) { + MOZ_CRASH("CacheIndex::OnFileOpened should not be called!"); + return NS_ERROR_UNEXPECTED; +} + +nsresult CacheIndex::OnDataWritten(CacheFileHandle* aHandle, const char* aBuf, + nsresult aResult) { + LOG(("CacheIndex::OnDataWritten() [handle=%p, result=0x%08" PRIx32 "]", + aHandle, static_cast<uint32_t>(aResult))); + + MOZ_ASSERT(CacheFileIOManager::IsOnIOThread()); + + nsresult rv; + + StaticMutexAutoLock lock(sLock); + + MOZ_RELEASE_ASSERT(IsIndexUsable()); + MOZ_RELEASE_ASSERT(mRWPending); + mRWPending = false; + + if (mState == READY && mShuttingDown) { + return NS_OK; + } + + switch (mState) { + case WRITING: + MOZ_ASSERT(mIndexHandle == aHandle); + + if (NS_FAILED(aResult)) { + FinishWrite(false, lock); + } else { + if (mSkipEntries == mProcessEntries) { + rv = CacheFileIOManager::RenameFile( + mIndexHandle, nsLiteralCString(INDEX_NAME), this); + if (NS_FAILED(rv)) { + LOG( + ("CacheIndex::OnDataWritten() - CacheFileIOManager::" + "RenameFile() failed synchronously [rv=0x%08" PRIx32 "]", + static_cast<uint32_t>(rv))); + FinishWrite(false, lock); + } + } else { + WriteRecords(lock); + } + } + break; + default: + // Writing was canceled. + LOG( + ("CacheIndex::OnDataWritten() - ignoring notification since the " + "operation was previously canceled [state=%d]", + mState)); + ReleaseBuffer(); + } + + return NS_OK; +} + +nsresult CacheIndex::OnDataRead(CacheFileHandle* aHandle, char* aBuf, + nsresult aResult) { + LOG(("CacheIndex::OnDataRead() [handle=%p, result=0x%08" PRIx32 "]", aHandle, + static_cast<uint32_t>(aResult))); + + MOZ_ASSERT(CacheFileIOManager::IsOnIOThread()); + + StaticMutexAutoLock lock(sLock); + + MOZ_RELEASE_ASSERT(IsIndexUsable()); + MOZ_RELEASE_ASSERT(mRWPending); + mRWPending = false; + + switch (mState) { + case READING: + MOZ_ASSERT(mIndexHandle == aHandle || mJournalHandle == aHandle); + + if (NS_FAILED(aResult)) { + FinishRead(false, lock); + } else { + if (!mIndexOnDiskIsValid) { + ParseRecords(lock); + } else { + ParseJournal(lock); + } + } + break; + default: + // Reading was canceled. + LOG( + ("CacheIndex::OnDataRead() - ignoring notification since the " + "operation was previously canceled [state=%d]", + mState)); + ReleaseBuffer(); + } + + return NS_OK; +} + +nsresult CacheIndex::OnFileDoomed(CacheFileHandle* aHandle, nsresult aResult) { + MOZ_CRASH("CacheIndex::OnFileDoomed should not be called!"); + return NS_ERROR_UNEXPECTED; +} + +nsresult CacheIndex::OnEOFSet(CacheFileHandle* aHandle, nsresult aResult) { + MOZ_CRASH("CacheIndex::OnEOFSet should not be called!"); + return NS_ERROR_UNEXPECTED; +} + +nsresult CacheIndex::OnFileRenamed(CacheFileHandle* aHandle, nsresult aResult) { + LOG(("CacheIndex::OnFileRenamed() [handle=%p, result=0x%08" PRIx32 "]", + aHandle, static_cast<uint32_t>(aResult))); + + MOZ_ASSERT(CacheFileIOManager::IsOnIOThread()); + + StaticMutexAutoLock lock(sLock); + + MOZ_RELEASE_ASSERT(IsIndexUsable()); + + if (mState == READY && mShuttingDown) { + return NS_OK; + } + + switch (mState) { + case WRITING: + // This is a result of renaming the new index written to tmpfile to index + // file. This is the last step when writing the index and the whole + // writing process is successful iff renaming was successful. + + if (mIndexHandle != aHandle) { + LOG( + ("CacheIndex::OnFileRenamed() - ignoring notification since it " + "belongs to previously canceled operation [state=%d]", + mState)); + break; + } + + FinishWrite(NS_SUCCEEDED(aResult), lock); + break; + case READING: + // This is a result of renaming journal file to tmpfile. It is renamed + // before we start reading index and journal file and it should normally + // succeed. If it fails give up reading of index. + + if (mJournalHandle != aHandle) { + LOG( + ("CacheIndex::OnFileRenamed() - ignoring notification since it " + "belongs to previously canceled operation [state=%d]", + mState)); + break; + } + + if (NS_FAILED(aResult)) { + FinishRead(false, lock); + } else { + StartReadingIndex(lock); + } + break; + default: + // Reading/writing was canceled. + LOG( + ("CacheIndex::OnFileRenamed() - ignoring notification since the " + "operation was previously canceled [state=%d]", + mState)); + } + + return NS_OK; +} + +// Memory reporting + +size_t CacheIndex::SizeOfExcludingThisInternal( + mozilla::MallocSizeOf mallocSizeOf) const { + sLock.AssertCurrentThreadOwns(); + + size_t n = 0; + nsCOMPtr<nsISizeOf> sizeOf; + + // mIndexHandle and mJournalHandle are reported via SizeOfHandlesRunnable + // in CacheFileIOManager::SizeOfExcludingThisInternal as part of special + // handles array. + + sizeOf = do_QueryInterface(mCacheDirectory); + if (sizeOf) { + n += sizeOf->SizeOfIncludingThis(mallocSizeOf); + } + + sizeOf = do_QueryInterface(mUpdateTimer); + if (sizeOf) { + n += sizeOf->SizeOfIncludingThis(mallocSizeOf); + } + + n += mallocSizeOf(mRWBuf); + n += mallocSizeOf(mRWHash); + + n += mIndex.SizeOfExcludingThis(mallocSizeOf); + n += mPendingUpdates.SizeOfExcludingThis(mallocSizeOf); + n += mTmpJournal.SizeOfExcludingThis(mallocSizeOf); + + // mFrecencyArray items are reported by mIndex/mPendingUpdates + n += mFrecencyArray.mRecs.ShallowSizeOfExcludingThis(mallocSizeOf); + n += mDiskConsumptionObservers.ShallowSizeOfExcludingThis(mallocSizeOf); + + return n; +} + +// static +size_t CacheIndex::SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) { + StaticMutexAutoLock lock(sLock); + + if (!gInstance) return 0; + + return gInstance->SizeOfExcludingThisInternal(mallocSizeOf); +} + +// static +size_t CacheIndex::SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) { + StaticMutexAutoLock lock(sLock); + + return mallocSizeOf(gInstance) + + (gInstance ? gInstance->SizeOfExcludingThisInternal(mallocSizeOf) : 0); +} + +// static +void CacheIndex::UpdateTotalBytesWritten(uint32_t aBytesWritten) { + StaticMutexAutoLock lock(sLock); + + RefPtr<CacheIndex> index = gInstance; + if (!index) { + return; + } + + index->mTotalBytesWritten += aBytesWritten; + + // Do telemetry report if enough data has been written and the index is + // in READY state. The data is available also in WRITING state, but we would + // need to deal with pending updates. + if (index->mTotalBytesWritten >= kTelemetryReportBytesLimit && + index->mState == READY && !index->mIndexNeedsUpdate && + !index->mShuttingDown) { + index->DoTelemetryReport(); + index->mTotalBytesWritten = 0; + return; + } +} + +void CacheIndex::DoTelemetryReport() { + static const nsLiteralCString + contentTypeNames[nsICacheEntry::CONTENT_TYPE_LAST] = { + "UNKNOWN"_ns, "OTHER"_ns, "JAVASCRIPT"_ns, "IMAGE"_ns, + "MEDIA"_ns, "STYLESHEET"_ns, "WASM"_ns}; + + for (uint32_t i = 0; i < nsICacheEntry::CONTENT_TYPE_LAST; ++i) { + if (mIndexStats.Size() > 0) { + Telemetry::Accumulate( + Telemetry::NETWORK_CACHE_SIZE_SHARE, contentTypeNames[i], + round(static_cast<double>(mIndexStats.SizeByType(i)) * 100.0 / + static_cast<double>(mIndexStats.Size()))); + } + + if (mIndexStats.Count() > 0) { + Telemetry::Accumulate( + Telemetry::NETWORK_CACHE_ENTRY_COUNT_SHARE, contentTypeNames[i], + round(static_cast<double>(mIndexStats.CountByType(i)) * 100.0 / + static_cast<double>(mIndexStats.Count()))); + } + } + + nsCString probeKey; + if (CacheObserver::SmartCacheSizeEnabled()) { + probeKey = "SMARTSIZE"_ns; + } else { + probeKey = "USERDEFINEDSIZE"_ns; + } + Telemetry::Accumulate(Telemetry::NETWORK_CACHE_ENTRY_COUNT, probeKey, + mIndexStats.Count()); + Telemetry::Accumulate(Telemetry::NETWORK_CACHE_SIZE, probeKey, + mIndexStats.Size() >> 10); +} + +// static +void CacheIndex::OnAsyncEviction(bool aEvicting) { + StaticMutexAutoLock lock(sLock); + + RefPtr<CacheIndex> index = gInstance; + if (!index) { + return; + } + + index->mAsyncGetDiskConsumptionBlocked = aEvicting; + if (!aEvicting) { + index->NotifyAsyncGetDiskConsumptionCallbacks(); + } +} + +} // namespace mozilla::net |