diff options
Diffstat (limited to 'dom/cache')
134 files changed, 22603 insertions, 0 deletions
diff --git a/dom/cache/Action.cpp b/dom/cache/Action.cpp new file mode 100644 index 0000000000..89690c099f --- /dev/null +++ b/dom/cache/Action.cpp @@ -0,0 +1,25 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/Action.h" + +namespace mozilla::dom::cache { + +void Action::CancelOnInitiatingThread() { + NS_ASSERT_OWNINGTHREAD(Action); + // It is possible for cancellation to be duplicated. For example, an + // individual Cache could have its Actions canceled and then shutdown + // could trigger a second action. + mCanceled = true; +} + +Action::Action() : mCanceled(false) {} + +Action::~Action() = default; + +bool Action::IsCanceled() const { return mCanceled; } + +} // namespace mozilla::dom::cache diff --git a/dom/cache/Action.h b/dom/cache/Action.h new file mode 100644 index 0000000000..1219c58e24 --- /dev/null +++ b/dom/cache/Action.h @@ -0,0 +1,102 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_Action_h +#define mozilla_dom_cache_Action_h + +#include "mozilla/Atomics.h" +#include "mozilla/dom/cache/Types.h" +#include "mozilla/dom/SafeRefPtr.h" +#include "nsISupportsImpl.h" + +class mozIStorageConnection; + +namespace mozilla::dom::cache { + +class Action : public SafeRefCounted<Action> { + public: + class Resolver { + public: + Resolver& operator=(const Resolver& aRHS) = delete; + + // Note: Action must drop Resolver ref after calling Resolve()! + // Note: Must be called on the same thread used to execute + // Action::RunOnTarget(). + virtual void Resolve(nsresult aRv) = 0; + + inline SafeRefPtr<Action::Resolver> SafeRefPtrFromThis() { + return SafeRefPtr<Action::Resolver>{this, AcquireStrongRefFromRawPtr{}}; + } + + NS_INLINE_DECL_PURE_VIRTUAL_REFCOUNTING + }; + + // Class containing data that can be opportunistically shared between + // multiple Actions running on the same thread/Context. In theory + // this could be abstracted to a generic key/value map, but for now + // just explicitly provide accessors for the data we need. + class Data { + public: + virtual mozIStorageConnection* GetConnection() const = 0; + + virtual void SetConnection(mozIStorageConnection* aConn) = 0; + }; + + // virtual because deleted through base class pointer + virtual ~Action(); + + // Execute operations on the target thread. Once complete call + // Resolver::Resolve(). This can be done sync or async. + // Note: Action should hold Resolver ref until its ready to call Resolve(). + // Note: The "target" thread is determined when the Action is scheduled on + // Context. The Action should not assume any particular thread is used. + virtual void RunOnTarget( + SafeRefPtr<Resolver> aResolver, + const Maybe<CacheDirectoryMetadata>& aDirectoryMetadata, + Data* aOptionalData) = 0; + + // Called on initiating thread when the Action is canceled. The Action is + // responsible for calling Resolver::Resolve() as normal; either with a + // normal error code or NS_ERROR_ABORT. If CancelOnInitiatingThread() is + // called after Resolve() has already occurred, then the cancel can be + // ignored. + // + // Cancellation is a best effort to stop processing as soon as possible, but + // does not guarantee the Action will not run. + // + // CancelOnInitiatingThread() may be called more than once. Subsequent + // calls should have no effect. + // + // Default implementation sets an internal cancellation flag that can be + // queried with IsCanceled(). + virtual void CancelOnInitiatingThread(); + + // Executed on the initiating thread and is passed the nsresult given to + // Resolver::Resolve(). + virtual void CompleteOnInitiatingThread(nsresult aRv) {} + + // Executed on the initiating thread. If this Action will operate on the + // given cache ID then override this to return true. + virtual bool MatchesCacheId(CacheId aCacheId) const { return false; } + + NS_DECL_OWNINGTHREAD + MOZ_DECLARE_REFCOUNTED_TYPENAME(cache::Action) + + protected: + Action(); + + // Check if this Action has been canceled. May be called from any thread, + // but typically used from the target thread. + bool IsCanceled() const; + + private: + // Accessible from any thread. + Atomic<bool> mCanceled; +}; + +} // namespace mozilla::dom::cache + +#endif // mozilla_dom_cache_Action_h diff --git a/dom/cache/ActorChild.cpp b/dom/cache/ActorChild.cpp new file mode 100644 index 0000000000..3d5eec8642 --- /dev/null +++ b/dom/cache/ActorChild.cpp @@ -0,0 +1,53 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/ActorChild.h" + +#include "mozilla/dom/cache/CacheWorkerRef.h" +#include "nsThreadUtils.h" + +namespace mozilla::dom::cache { + +void ActorChild::SetWorkerRef(SafeRefPtr<CacheWorkerRef> aWorkerRef) { + // Some of the Cache actors can have multiple DOM objects associated with + // them. In this case the workerRef will be added multiple times. This is + // permitted, but the workerRef should be the same each time. + if (mWorkerRef) { + // XXX Here, we don't use aWorkerRef, so we unnecessarily add-refed... This + // might be a case to show in the raw pointer guideline as a possible + // exception, if warranted by performance analyses. + + MOZ_DIAGNOSTIC_ASSERT(mWorkerRef == aWorkerRef); + return; + } + + mWorkerRef = std::move(aWorkerRef); + if (mWorkerRef) { + mWorkerRef->AddActor(*this); + } +} + +void ActorChild::RemoveWorkerRef() { + MOZ_ASSERT_IF(!NS_IsMainThread(), mWorkerRef); + if (mWorkerRef) { + mWorkerRef->RemoveActor(*this); + mWorkerRef = nullptr; + } +} + +const SafeRefPtr<CacheWorkerRef>& ActorChild::GetWorkerRefPtr() const { + return mWorkerRef; +} + +bool ActorChild::WorkerRefNotified() const { + return mWorkerRef && mWorkerRef->Notified(); +} + +ActorChild::ActorChild() = default; + +ActorChild::~ActorChild() { MOZ_DIAGNOSTIC_ASSERT(!mWorkerRef); } + +} // namespace mozilla::dom::cache diff --git a/dom/cache/ActorChild.h b/dom/cache/ActorChild.h new file mode 100644 index 0000000000..e7d375173d --- /dev/null +++ b/dom/cache/ActorChild.h @@ -0,0 +1,40 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_ActioChild_h +#define mozilla_dom_cache_ActioChild_h + +#include "mozilla/dom/SafeRefPtr.h" + +namespace mozilla::dom::cache { + +class CacheWorkerRef; + +class ActorChild { + public: + virtual void StartDestroy() = 0; + virtual void NoteDeletedActor() { /*no-op*/ + } + + void SetWorkerRef(SafeRefPtr<CacheWorkerRef> aWorkerRef); + + void RemoveWorkerRef(); + + const SafeRefPtr<CacheWorkerRef>& GetWorkerRefPtr() const; + + bool WorkerRefNotified() const; + + protected: + ActorChild(); + ~ActorChild(); + + private: + SafeRefPtr<CacheWorkerRef> mWorkerRef; +}; + +} // namespace mozilla::dom::cache + +#endif // mozilla_dom_cache_ActioChild_h diff --git a/dom/cache/ActorUtils.h b/dom/cache/ActorUtils.h new file mode 100644 index 0000000000..030e278bc2 --- /dev/null +++ b/dom/cache/ActorUtils.h @@ -0,0 +1,52 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_ActorUtils_h +#define mozilla_dom_cache_ActorUtils_h + +#include "mozilla/dom/cache/Types.h" + +namespace mozilla { + +namespace ipc { +class PBackgroundParent; +class PrincipalInfo; +} // namespace ipc + +namespace dom::cache { + +class PCacheChild; +class PCacheParent; +class PCacheStreamControlChild; +class PCacheStreamControlParent; +class PCacheStorageChild; +class PCacheStorageParent; + +// Factory methods for use in ipc/glue methods. Implemented in individual actor +// cpp files. + +already_AddRefed<PCacheChild> AllocPCacheChild(); + +void DeallocPCacheChild(PCacheChild* aActor); + +void DeallocPCacheParent(PCacheParent* aActor); + +already_AddRefed<PCacheStreamControlChild> AllocPCacheStreamControlChild(); + +void DeallocPCacheStreamControlParent(PCacheStreamControlParent* aActor); + +already_AddRefed<PCacheStorageParent> AllocPCacheStorageParent( + mozilla::ipc::PBackgroundParent* aManagingActor, Namespace aNamespace, + const mozilla::ipc::PrincipalInfo& aPrincipalInfo); + +void DeallocPCacheStorageChild(PCacheStorageChild* aActor); + +void DeallocPCacheStorageParent(PCacheStorageParent* aActor); + +} // namespace dom::cache +} // namespace mozilla + +#endif // mozilla_dom_cache_ActorUtils_h diff --git a/dom/cache/AutoUtils.cpp b/dom/cache/AutoUtils.cpp new file mode 100644 index 0000000000..2ea7deb9ad --- /dev/null +++ b/dom/cache/AutoUtils.cpp @@ -0,0 +1,472 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/AutoUtils.h" + +#include "mozilla/Unused.h" +#include "mozilla/dom/InternalHeaders.h" +#include "mozilla/dom/InternalRequest.h" +#include "mozilla/dom/cache/CacheParent.h" +#include "mozilla/dom/cache/CacheStreamControlParent.h" +#include "mozilla/dom/cache/ReadStream.h" +#include "mozilla/dom/cache/SavedTypes.h" +#include "mozilla/dom/cache/StreamList.h" +#include "mozilla/dom/cache/TypeUtils.h" +#include "mozilla/ipc/IPCStreamUtils.h" +#include "mozilla/ipc/PBackgroundParent.h" +#include "nsCharSeparatedTokenizer.h" +#include "nsHttp.h" + +using mozilla::Maybe; +using mozilla::Unused; +using mozilla::dom::cache::CacheReadStream; +using mozilla::ipc::PBackgroundParent; + +namespace { + +enum CleanupAction { Forget, Delete }; + +void CleanupChild(CacheReadStream& aReadStream, CleanupAction aAction) { + // fds cleaned up by mStreamCleanupList +} + +void CleanupChild(Maybe<CacheReadStream>& aMaybeReadStream, + CleanupAction aAction) { + if (aMaybeReadStream.isNothing()) { + return; + } + + CleanupChild(aMaybeReadStream.ref(), aAction); +} + +} // namespace + +namespace mozilla::dom::cache { + +// -------------------------------------------- + +AutoChildOpArgs::AutoChildOpArgs(TypeUtils* aTypeUtils, + const CacheOpArgs& aOpArgs, + uint32_t aEntryCount) + : mTypeUtils(aTypeUtils), mOpArgs(aOpArgs), mSent(false) { + MOZ_DIAGNOSTIC_ASSERT(mTypeUtils); + MOZ_RELEASE_ASSERT(aEntryCount != 0); + if (mOpArgs.type() == CacheOpArgs::TCachePutAllArgs) { + CachePutAllArgs& args = mOpArgs.get_CachePutAllArgs(); + args.requestResponseList().SetCapacity(aEntryCount); + } else { + MOZ_DIAGNOSTIC_ASSERT(aEntryCount == 1); + } +} + +AutoChildOpArgs::~AutoChildOpArgs() { + CleanupAction action = mSent ? Forget : Delete; + + switch (mOpArgs.type()) { + case CacheOpArgs::TCacheMatchArgs: { + CacheMatchArgs& args = mOpArgs.get_CacheMatchArgs(); + CleanupChild(args.request().body(), action); + break; + } + case CacheOpArgs::TCacheMatchAllArgs: { + CacheMatchAllArgs& args = mOpArgs.get_CacheMatchAllArgs(); + if (args.maybeRequest().isNothing()) { + break; + } + CleanupChild(args.maybeRequest().ref().body(), action); + break; + } + case CacheOpArgs::TCachePutAllArgs: { + CachePutAllArgs& args = mOpArgs.get_CachePutAllArgs(); + auto& list = args.requestResponseList(); + for (uint32_t i = 0; i < list.Length(); ++i) { + CleanupChild(list[i].request().body(), action); + CleanupChild(list[i].response().body(), action); + } + break; + } + case CacheOpArgs::TCacheDeleteArgs: { + CacheDeleteArgs& args = mOpArgs.get_CacheDeleteArgs(); + CleanupChild(args.request().body(), action); + break; + } + case CacheOpArgs::TCacheKeysArgs: { + CacheKeysArgs& args = mOpArgs.get_CacheKeysArgs(); + if (args.maybeRequest().isNothing()) { + break; + } + CleanupChild(args.maybeRequest().ref().body(), action); + break; + } + case CacheOpArgs::TStorageMatchArgs: { + StorageMatchArgs& args = mOpArgs.get_StorageMatchArgs(); + CleanupChild(args.request().body(), action); + break; + } + default: + // Other types do not need cleanup + break; + } +} + +void AutoChildOpArgs::Add(const InternalRequest& aRequest, + BodyAction aBodyAction, SchemeAction aSchemeAction, + ErrorResult& aRv) { + MOZ_DIAGNOSTIC_ASSERT(!mSent); + + switch (mOpArgs.type()) { + case CacheOpArgs::TCacheMatchArgs: { + CacheMatchArgs& args = mOpArgs.get_CacheMatchArgs(); + mTypeUtils->ToCacheRequest(args.request(), aRequest, aBodyAction, + aSchemeAction, aRv); + break; + } + case CacheOpArgs::TCacheMatchAllArgs: { + CacheMatchAllArgs& args = mOpArgs.get_CacheMatchAllArgs(); + MOZ_DIAGNOSTIC_ASSERT(args.maybeRequest().isNothing()); + args.maybeRequest().emplace(CacheRequest()); + mTypeUtils->ToCacheRequest(args.maybeRequest().ref(), aRequest, + aBodyAction, aSchemeAction, aRv); + break; + } + case CacheOpArgs::TCacheDeleteArgs: { + CacheDeleteArgs& args = mOpArgs.get_CacheDeleteArgs(); + mTypeUtils->ToCacheRequest(args.request(), aRequest, aBodyAction, + aSchemeAction, aRv); + break; + } + case CacheOpArgs::TCacheKeysArgs: { + CacheKeysArgs& args = mOpArgs.get_CacheKeysArgs(); + MOZ_DIAGNOSTIC_ASSERT(args.maybeRequest().isNothing()); + args.maybeRequest().emplace(CacheRequest()); + mTypeUtils->ToCacheRequest(args.maybeRequest().ref(), aRequest, + aBodyAction, aSchemeAction, aRv); + break; + } + case CacheOpArgs::TStorageMatchArgs: { + StorageMatchArgs& args = mOpArgs.get_StorageMatchArgs(); + mTypeUtils->ToCacheRequest(args.request(), aRequest, aBodyAction, + aSchemeAction, aRv); + break; + } + default: + MOZ_CRASH("Cache args type cannot send a Request!"); + } +} + +namespace { + +bool MatchInPutList(const InternalRequest& aRequest, + const nsTArray<CacheRequestResponse>& aPutList) { + // This method implements the SW spec QueryCache algorithm against an + // in memory array of Request/Response objects. This essentially the + // same algorithm that is implemented in DBSchema.cpp. Unfortunately + // we cannot unify them because when operating against the real database + // we don't want to load all request/response objects into memory. + + // Note, we can skip the check for a invalid request method because + // Cache should only call into here with a GET or HEAD. +#ifdef DEBUG + nsAutoCString method; + aRequest.GetMethod(method); + MOZ_ASSERT(method.LowerCaseEqualsLiteral("get") || + method.LowerCaseEqualsLiteral("head")); +#endif + + RefPtr<InternalHeaders> requestHeaders = aRequest.Headers(); + + for (uint32_t i = 0; i < aPutList.Length(); ++i) { + const CacheRequest& cachedRequest = aPutList[i].request(); + const CacheResponse& cachedResponse = aPutList[i].response(); + + nsAutoCString url; + aRequest.GetURL(url); + + nsAutoCString requestUrl(cachedRequest.urlWithoutQuery()); + requestUrl.Append(cachedRequest.urlQuery()); + + // If the URLs don't match, then just skip to the next entry. + if (url != requestUrl) { + continue; + } + + RefPtr<InternalHeaders> cachedRequestHeaders = + TypeUtils::ToInternalHeaders(cachedRequest.headers()); + + RefPtr<InternalHeaders> cachedResponseHeaders = + TypeUtils::ToInternalHeaders(cachedResponse.headers()); + + nsCString varyHeaders; + ErrorResult rv; + cachedResponseHeaders->Get("vary"_ns, varyHeaders, rv); + MOZ_ALWAYS_TRUE(!rv.Failed()); + + // Assume the vary headers match until we find a conflict + bool varyHeadersMatch = true; + + for (const nsACString& header : + nsCCharSeparatedTokenizer(varyHeaders, NS_HTTP_HEADER_SEP).ToRange()) { + MOZ_DIAGNOSTIC_ASSERT(!header.EqualsLiteral("*"), + "We should have already caught this in " + "TypeUtils::ToPCacheResponseWithoutBody()"); + + ErrorResult headerRv; + nsAutoCString value; + requestHeaders->Get(header, value, headerRv); + if (NS_WARN_IF(headerRv.Failed())) { + headerRv.SuppressException(); + MOZ_DIAGNOSTIC_ASSERT(value.IsEmpty()); + } + + nsAutoCString cachedValue; + cachedRequestHeaders->Get(header, cachedValue, headerRv); + if (NS_WARN_IF(headerRv.Failed())) { + headerRv.SuppressException(); + MOZ_DIAGNOSTIC_ASSERT(cachedValue.IsEmpty()); + } + + if (value != cachedValue) { + varyHeadersMatch = false; + break; + } + } + + // URL was equal and all vary headers match! + if (varyHeadersMatch) { + return true; + } + } + + return false; +} + +} // namespace + +void AutoChildOpArgs::Add(JSContext* aCx, const InternalRequest& aRequest, + BodyAction aBodyAction, SchemeAction aSchemeAction, + Response& aResponse, ErrorResult& aRv) { + MOZ_DIAGNOSTIC_ASSERT(!mSent); + + switch (mOpArgs.type()) { + case CacheOpArgs::TCachePutAllArgs: { + CachePutAllArgs& args = mOpArgs.get_CachePutAllArgs(); + + // Throw an error if a request/response pair would mask another + // request/response pair in the same PutAll operation. This is + // step 2.3.2.3 from the "Batch Cache Operations" spec algorithm. + if (MatchInPutList(aRequest, args.requestResponseList())) { + aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR); + return; + } + + MOZ_RELEASE_ASSERT(args.requestResponseList().Length() < + args.requestResponseList().Capacity()); + + // The FileDescriptorSetChild asserts in its destructor that all fds have + // been removed. The copy constructor, however, simply duplicates the + // fds without removing any. This means each temporary and copy must be + // explicitly cleaned up. + // + // Avoid a lot of this hassle by making sure we only create one here. On + // error we remove it. + CacheRequestResponse& pair = *args.requestResponseList().AppendElement(); + pair.request().body() = Nothing(); + pair.response().body() = Nothing(); + + mTypeUtils->ToCacheRequest(pair.request(), aRequest, aBodyAction, + aSchemeAction, aRv); + if (!aRv.Failed()) { + mTypeUtils->ToCacheResponse(aCx, pair.response(), aResponse, aRv); + } + + if (aRv.Failed()) { + CleanupChild(pair.request().body(), Delete); + args.requestResponseList().RemoveLastElement(); + } + + break; + } + default: + MOZ_CRASH("Cache args type cannot send a Request/Response pair!"); + } +} + +const CacheOpArgs& AutoChildOpArgs::SendAsOpArgs() { + MOZ_DIAGNOSTIC_ASSERT(!mSent); + mSent = true; + return mOpArgs; +} + +// -------------------------------------------- + +AutoParentOpResult::AutoParentOpResult( + mozilla::ipc::PBackgroundParent* aManager, const CacheOpResult& aOpResult, + uint32_t aEntryCount) + : mManager(aManager), + mOpResult(aOpResult), + mStreamControl(nullptr), + mSent(false) { + MOZ_DIAGNOSTIC_ASSERT(mManager); + MOZ_RELEASE_ASSERT(aEntryCount != 0); + if (mOpResult.type() == CacheOpResult::TCacheMatchAllResult) { + CacheMatchAllResult& result = mOpResult.get_CacheMatchAllResult(); + result.responseList().SetCapacity(aEntryCount); + } else if (mOpResult.type() == CacheOpResult::TCacheKeysResult) { + CacheKeysResult& result = mOpResult.get_CacheKeysResult(); + result.requestList().SetCapacity(aEntryCount); + } else { + MOZ_DIAGNOSTIC_ASSERT(aEntryCount == 1); + } +} + +AutoParentOpResult::~AutoParentOpResult() { + CleanupAction action = mSent ? Forget : Delete; + + switch (mOpResult.type()) { + case CacheOpResult::TStorageOpenResult: { + StorageOpenResult& result = mOpResult.get_StorageOpenResult(); + if (action == Forget || !result.actor()) { + break; + } + + QM_WARNONLY_TRY( + OkIf(PCacheParent::Send__delete__(result.actor().AsParent()))); + break; + } + default: + // other types do not need additional clean up + break; + } + + if (action == Delete && mStreamControl) { + mStreamControl->AssertWillDelete(); + QM_WARNONLY_TRY( + OkIf(PCacheStreamControlParent::Send__delete__(mStreamControl))); + } +} + +void AutoParentOpResult::Add(CacheId aOpenedCacheId, + SafeRefPtr<Manager> aManager) { + MOZ_DIAGNOSTIC_ASSERT(mOpResult.type() == CacheOpResult::TStorageOpenResult); + MOZ_DIAGNOSTIC_ASSERT(!mOpResult.get_StorageOpenResult().actor()); + mOpResult.get_StorageOpenResult().actor() = mManager->SendPCacheConstructor( + new CacheParent(std::move(aManager), aOpenedCacheId)); +} + +void AutoParentOpResult::Add(const SavedResponse& aSavedResponse, + StreamList& aStreamList) { + MOZ_DIAGNOSTIC_ASSERT(!mSent); + + switch (mOpResult.type()) { + case CacheOpResult::TCacheMatchResult: { + CacheMatchResult& result = mOpResult.get_CacheMatchResult(); + MOZ_DIAGNOSTIC_ASSERT(result.maybeResponse().isNothing()); + result.maybeResponse().emplace(aSavedResponse.mValue); + SerializeResponseBody(aSavedResponse, aStreamList, + &result.maybeResponse().ref()); + break; + } + case CacheOpResult::TCacheMatchAllResult: { + CacheMatchAllResult& result = mOpResult.get_CacheMatchAllResult(); + MOZ_RELEASE_ASSERT(result.responseList().Length() < + result.responseList().Capacity()); + result.responseList().AppendElement(aSavedResponse.mValue); + SerializeResponseBody(aSavedResponse, aStreamList, + &result.responseList().LastElement()); + break; + } + case CacheOpResult::TStorageMatchResult: { + StorageMatchResult& result = mOpResult.get_StorageMatchResult(); + MOZ_DIAGNOSTIC_ASSERT(result.maybeResponse().isNothing()); + result.maybeResponse().emplace(aSavedResponse.mValue); + SerializeResponseBody(aSavedResponse, aStreamList, + &result.maybeResponse().ref()); + break; + } + default: + MOZ_CRASH("Cache result type cannot handle returning a Response!"); + } +} + +void AutoParentOpResult::Add(const SavedRequest& aSavedRequest, + StreamList& aStreamList) { + MOZ_DIAGNOSTIC_ASSERT(!mSent); + + switch (mOpResult.type()) { + case CacheOpResult::TCacheKeysResult: { + CacheKeysResult& result = mOpResult.get_CacheKeysResult(); + MOZ_RELEASE_ASSERT(result.requestList().Length() < + result.requestList().Capacity()); + result.requestList().AppendElement(aSavedRequest.mValue); + CacheRequest& request = result.requestList().LastElement(); + + if (!aSavedRequest.mHasBodyId) { + request.body() = Nothing(); + break; + } + + request.body().emplace(CacheReadStream()); + SerializeReadStream(aSavedRequest.mBodyId, aStreamList, + &request.body().ref()); + break; + } + default: + MOZ_CRASH("Cache result type cannot handle returning a Request!"); + } +} + +const CacheOpResult& AutoParentOpResult::SendAsOpResult() { + MOZ_DIAGNOSTIC_ASSERT(!mSent); + mSent = true; + return mOpResult; +} + +void AutoParentOpResult::SerializeResponseBody( + const SavedResponse& aSavedResponse, StreamList& aStreamList, + CacheResponse* aResponseOut) { + MOZ_DIAGNOSTIC_ASSERT(aResponseOut); + + if (!aSavedResponse.mHasBodyId) { + aResponseOut->body() = Nothing(); + return; + } + + aResponseOut->body().emplace(CacheReadStream()); + SerializeReadStream(aSavedResponse.mBodyId, aStreamList, + &aResponseOut->body().ref()); +} + +void AutoParentOpResult::SerializeReadStream(const nsID& aId, + StreamList& aStreamList, + CacheReadStream* aReadStreamOut) { + MOZ_DIAGNOSTIC_ASSERT(aReadStreamOut); + MOZ_DIAGNOSTIC_ASSERT(!mSent); + + nsCOMPtr<nsIInputStream> stream = aStreamList.Extract(aId); + + if (!mStreamControl) { + mStreamControl = static_cast<CacheStreamControlParent*>( + mManager->SendPCacheStreamControlConstructor( + new CacheStreamControlParent())); + + // If this failed, then the child process is gone. Warn and allow actor + // cleanup to proceed as normal. + if (!mStreamControl) { + NS_WARNING("Cache failed to create stream control actor."); + return; + } + } + + aStreamList.SetStreamControl(mStreamControl); + + RefPtr<ReadStream> readStream = + ReadStream::Create(mStreamControl, aId, stream); + ErrorResult rv; + readStream->Serialize(aReadStreamOut, rv); + MOZ_DIAGNOSTIC_ASSERT(!rv.Failed()); +} + +} // namespace mozilla::dom::cache diff --git a/dom/cache/AutoUtils.h b/dom/cache/AutoUtils.h new file mode 100644 index 0000000000..b24c07a37c --- /dev/null +++ b/dom/cache/AutoUtils.h @@ -0,0 +1,100 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_AutoUtils_h +#define mozilla_dom_cache_AutoUtils_h + +#include "mozilla/Attributes.h" +#include "mozilla/dom/SafeRefPtr.h" +#include "mozilla/dom/cache/CacheTypes.h" +#include "mozilla/dom/cache/Types.h" +#include "mozilla/dom/cache/TypeUtils.h" +#include "nsTArray.h" + +struct nsID; + +namespace mozilla { + +class ErrorResult; + +namespace ipc { +class PBackgroundParent; +} // namespace ipc + +namespace dom { + +class InternalRequest; + +namespace cache { + +class CacheStreamControlParent; +class Manager; +struct SavedRequest; +struct SavedResponse; +class StreamList; + +// A collection of RAII-style helper classes to ensure that IPC +// FileDescriptorSet actors are properly cleaned up. The user of these actors +// must manually either Forget() the Fds or Send__delete__() the actor +// depending on if the descriptors were actually sent. +// +// Note, these should only be used when *sending* streams across IPC. The +// deserialization case is handled by creating a ReadStream object. + +class MOZ_STACK_CLASS AutoChildOpArgs final { + public: + using BodyAction = TypeUtils::BodyAction; + using SchemeAction = TypeUtils::SchemeAction; + + AutoChildOpArgs(TypeUtils* aTypeUtils, const CacheOpArgs& aOpArgs, + uint32_t aEntryCount); + ~AutoChildOpArgs(); + + void Add(const InternalRequest& aRequest, BodyAction aBodyAction, + SchemeAction aSchemeAction, ErrorResult& aRv); + void Add(JSContext* aCx, const InternalRequest& aRequest, + BodyAction aBodyAction, SchemeAction aSchemeAction, + Response& aResponse, ErrorResult& aRv); + + const CacheOpArgs& SendAsOpArgs(); + + private: + TypeUtils* mTypeUtils; + CacheOpArgs mOpArgs; + bool mSent; +}; + +class MOZ_STACK_CLASS AutoParentOpResult final { + public: + AutoParentOpResult(mozilla::ipc::PBackgroundParent* aManager, + const CacheOpResult& aOpResult, uint32_t aEntryCount); + ~AutoParentOpResult(); + + void Add(CacheId aOpenedCacheId, SafeRefPtr<Manager> aManager); + void Add(const SavedResponse& aSavedResponse, StreamList& aStreamList); + void Add(const SavedRequest& aSavedRequest, StreamList& aStreamList); + + const CacheOpResult& SendAsOpResult(); + + private: + void SerializeResponseBody(const SavedResponse& aSavedResponse, + StreamList& aStreamList, + CacheResponse* aResponseOut); + + void SerializeReadStream(const nsID& aId, StreamList& aStreamList, + CacheReadStream* aReadStreamOut); + + mozilla::ipc::PBackgroundParent* mManager; + CacheOpResult mOpResult; + CacheStreamControlParent* mStreamControl; + bool mSent; +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_cache_AutoUtils_h diff --git a/dom/cache/Cache.cpp b/dom/cache/Cache.cpp new file mode 100644 index 0000000000..d9b94f6b14 --- /dev/null +++ b/dom/cache/Cache.cpp @@ -0,0 +1,617 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/Cache.h" + +#include "js/Array.h" // JS::GetArrayLength, JS::IsArrayObject +#include "js/PropertyAndElement.h" // JS_GetElement +#include "mozilla/dom/Headers.h" +#include "mozilla/dom/InternalResponse.h" +#include "mozilla/dom/Promise.h" +#include "mozilla/dom/PromiseNativeHandler.h" +#include "mozilla/dom/Response.h" +#include "mozilla/dom/RootedDictionary.h" +#include "mozilla/dom/WorkerPrivate.h" +#include "mozilla/dom/CacheBinding.h" +#include "mozilla/dom/cache/AutoUtils.h" +#include "mozilla/dom/cache/CacheChild.h" +#include "mozilla/dom/cache/CacheCommon.h" +#include "mozilla/dom/cache/CacheWorkerRef.h" +#include "mozilla/dom/quota/ResultExtensions.h" +#include "mozilla/ErrorResult.h" +#include "mozilla/Preferences.h" +#include "mozilla/Unused.h" +#include "nsIGlobalObject.h" + +namespace mozilla::dom::cache { + +using mozilla::ipc::PBackgroundChild; + +namespace { + +enum class PutStatusPolicy { Default, RequireOK }; + +bool IsValidPutRequestURL(const nsAString& aUrl, ErrorResult& aRv) { + bool validScheme = false; + + // make a copy because ProcessURL strips the fragmet + NS_ConvertUTF16toUTF8 url(aUrl); + + TypeUtils::ProcessURL(url, &validScheme, nullptr, nullptr, aRv); + if (aRv.Failed()) { + return false; + } + + if (!validScheme) { + // `url` has been modified, so don't use it here. + aRv.ThrowTypeError<MSG_INVALID_URL_SCHEME>("Request", + NS_ConvertUTF16toUTF8(aUrl)); + return false; + } + + return true; +} + +static bool IsValidPutRequestMethod(const Request& aRequest, ErrorResult& aRv) { + nsAutoCString method; + aRequest.GetMethod(method); + if (!method.LowerCaseEqualsLiteral("get")) { + aRv.ThrowTypeError<MSG_INVALID_REQUEST_METHOD>(method); + return false; + } + + return true; +} + +static bool IsValidPutRequestMethod(const RequestOrUSVString& aRequest, + ErrorResult& aRv) { + // If the provided request is a string URL, then it will default to + // a valid http method automatically. + if (!aRequest.IsRequest()) { + return true; + } + return IsValidPutRequestMethod(aRequest.GetAsRequest(), aRv); +} + +static bool IsValidPutResponseStatus(Response& aResponse, + PutStatusPolicy aPolicy, + ErrorResult& aRv) { + if ((aPolicy == PutStatusPolicy::RequireOK && !aResponse.Ok()) || + aResponse.Status() == 206) { + nsCString type(ResponseTypeValues::GetString(aResponse.Type())); + + nsAutoString url; + aResponse.GetUrl(url); + + aRv.ThrowTypeError<MSG_CACHE_ADD_FAILED_RESPONSE>( + type, IntToCString(aResponse.Status()), NS_ConvertUTF16toUTF8(url)); + return false; + } + + return true; +} + +} // namespace + +// Helper class to wait for Add()/AddAll() fetch requests to complete and +// then perform a PutAll() with the responses. This class holds a WorkerRef +// to keep the Worker thread alive. This is mainly to ensure that Add/AddAll +// act the same as other Cache operations that directly create a CacheOpChild +// actor. +class Cache::FetchHandler final : public PromiseNativeHandler { + public: + FetchHandler(SafeRefPtr<CacheWorkerRef> aWorkerRef, Cache* aCache, + nsTArray<SafeRefPtr<Request>>&& aRequestList, Promise* aPromise) + : mWorkerRef(std::move(aWorkerRef)), + mCache(aCache), + mRequestList(std::move(aRequestList)), + mPromise(aPromise) { + MOZ_ASSERT_IF(!NS_IsMainThread(), mWorkerRef); + MOZ_DIAGNOSTIC_ASSERT(mCache); + MOZ_DIAGNOSTIC_ASSERT(mPromise); + } + + virtual void ResolvedCallback(JSContext* aCx, JS::Handle<JS::Value> aValue, + ErrorResult& aRv) override { + NS_ASSERT_OWNINGTHREAD(FetchHandler); + + // Stop holding the worker alive when we leave this method. + const SafeRefPtr<CacheWorkerRef> workerRef = std::move(mWorkerRef); + + // Promise::All() passed an array of fetch() Promises should give us + // an Array of Response objects. The following code unwraps these + // JS values back to an nsTArray<RefPtr<Response>>. + + AutoTArray<RefPtr<Response>, 256> responseList; + responseList.SetCapacity(mRequestList.Length()); + + const auto failOnErr = [this](const auto) { Fail(); }; + + bool isArray; + QM_TRY(OkIf(JS::IsArrayObject(aCx, aValue, &isArray)), QM_VOID, failOnErr); + QM_TRY(OkIf(isArray), QM_VOID, failOnErr); + + JS::Rooted<JSObject*> obj(aCx, &aValue.toObject()); + + uint32_t length; + QM_TRY(OkIf(JS::GetArrayLength(aCx, obj, &length)), QM_VOID, failOnErr); + + for (uint32_t i = 0; i < length; ++i) { + JS::Rooted<JS::Value> value(aCx); + + QM_TRY(OkIf(JS_GetElement(aCx, obj, i, &value)), QM_VOID, failOnErr); + + QM_TRY(OkIf(value.isObject()), QM_VOID, failOnErr); + + JS::Rooted<JSObject*> responseObj(aCx, &value.toObject()); + + RefPtr<Response> response; + QM_TRY(MOZ_TO_RESULT(UNWRAP_OBJECT(Response, responseObj, response)), + QM_VOID, failOnErr); + + QM_TRY(OkIf(response->Type() != ResponseType::Error), QM_VOID, failOnErr); + + // Do not allow the convenience methods .add()/.addAll() to store failed + // or invalid responses. A consequence of this is that these methods + // cannot be used to store opaque or opaqueredirect responses since they + // always expose a 0 status value. + ErrorResult errorResult; + if (!IsValidPutResponseStatus(*response, PutStatusPolicy::RequireOK, + errorResult)) { + // TODO: abort the fetch requests we have running (bug 1157434) + mPromise->MaybeReject(std::move(errorResult)); + return; + } + + responseList.AppendElement(std::move(response)); + } + + MOZ_DIAGNOSTIC_ASSERT(mRequestList.Length() == responseList.Length()); + + // Now store the unwrapped Response list in the Cache. + ErrorResult result; + // TODO: Here we use the JSContext as received by the ResolvedCallback, and + // its state could be the wrong one. The spec doesn't say anything + // about it, yet (bug 1384006) + RefPtr<Promise> put = + mCache->PutAll(aCx, mRequestList, responseList, result); + result.WouldReportJSException(); + if (NS_WARN_IF(result.Failed())) { + // TODO: abort the fetch requests we have running (bug 1157434) + mPromise->MaybeReject(std::move(result)); + return; + } + + // Chain the Cache::Put() promise to the original promise returned to + // the content script. + mPromise->MaybeResolve(put); + } + + virtual void RejectedCallback(JSContext* aCx, JS::Handle<JS::Value> aValue, + ErrorResult& aRv) override { + NS_ASSERT_OWNINGTHREAD(FetchHandler); + Fail(); + } + + private: + ~FetchHandler() = default; + + void Fail() { mPromise->MaybeRejectWithTypeError<MSG_FETCH_FAILED>(); } + + SafeRefPtr<CacheWorkerRef> mWorkerRef; + RefPtr<Cache> mCache; + nsTArray<SafeRefPtr<Request>> mRequestList; + RefPtr<Promise> mPromise; + + NS_DECL_ISUPPORTS +}; + +NS_IMPL_ISUPPORTS0(Cache::FetchHandler) + +NS_IMPL_CYCLE_COLLECTING_ADDREF(mozilla::dom::cache::Cache); +NS_IMPL_CYCLE_COLLECTING_RELEASE(mozilla::dom::cache::Cache); +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(mozilla::dom::cache::Cache, mGlobal); + +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(Cache) + NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY + NS_INTERFACE_MAP_ENTRY(nsISupports) +NS_INTERFACE_MAP_END + +Cache::Cache(nsIGlobalObject* aGlobal, CacheChild* aActor, Namespace aNamespace) + : mGlobal(aGlobal), mActor(aActor), mNamespace(aNamespace) { + MOZ_DIAGNOSTIC_ASSERT(mGlobal); + MOZ_DIAGNOSTIC_ASSERT(mActor); + MOZ_DIAGNOSTIC_ASSERT(mNamespace != INVALID_NAMESPACE); + mActor->SetListener(this); +} + +already_AddRefed<Promise> Cache::Match(JSContext* aCx, + const RequestOrUSVString& aRequest, + const CacheQueryOptions& aOptions, + ErrorResult& aRv) { + if (NS_WARN_IF(!mActor)) { + aRv.Throw(NS_ERROR_UNEXPECTED); + return nullptr; + } + + CacheChild::AutoLock actorLock(*mActor); + + SafeRefPtr<InternalRequest> ir = + ToInternalRequest(aCx, aRequest, IgnoreBody, aRv); + if (NS_WARN_IF(aRv.Failed())) { + return nullptr; + } + + CacheQueryParams params; + ToCacheQueryParams(params, aOptions); + + AutoChildOpArgs args( + this, CacheMatchArgs(CacheRequest(), params, GetOpenMode()), 1); + + args.Add(*ir, IgnoreBody, IgnoreInvalidScheme, aRv); + if (NS_WARN_IF(aRv.Failed())) { + return nullptr; + } + + return ExecuteOp(args, aRv); +} + +already_AddRefed<Promise> Cache::MatchAll( + JSContext* aCx, const Optional<RequestOrUSVString>& aRequest, + const CacheQueryOptions& aOptions, ErrorResult& aRv) { + if (NS_WARN_IF(!mActor)) { + aRv.Throw(NS_ERROR_UNEXPECTED); + return nullptr; + } + + CacheChild::AutoLock actorLock(*mActor); + + CacheQueryParams params; + ToCacheQueryParams(params, aOptions); + + AutoChildOpArgs args(this, + CacheMatchAllArgs(Nothing(), params, GetOpenMode()), 1); + + if (aRequest.WasPassed()) { + SafeRefPtr<InternalRequest> ir = + ToInternalRequest(aCx, aRequest.Value(), IgnoreBody, aRv); + if (aRv.Failed()) { + return nullptr; + } + + args.Add(*ir, IgnoreBody, IgnoreInvalidScheme, aRv); + if (aRv.Failed()) { + return nullptr; + } + } + + return ExecuteOp(args, aRv); +} + +already_AddRefed<Promise> Cache::Add(JSContext* aContext, + const RequestOrUSVString& aRequest, + CallerType aCallerType, ErrorResult& aRv) { + if (NS_WARN_IF(!mActor)) { + aRv.Throw(NS_ERROR_UNEXPECTED); + return nullptr; + } + + CacheChild::AutoLock actorLock(*mActor); + + if (!IsValidPutRequestMethod(aRequest, aRv)) { + return nullptr; + } + + GlobalObject global(aContext, mGlobal->GetGlobalJSObject()); + MOZ_DIAGNOSTIC_ASSERT(!global.Failed()); + + nsTArray<SafeRefPtr<Request>> requestList(1); + RootedDictionary<RequestInit> requestInit(aContext); + SafeRefPtr<Request> request = + Request::Constructor(global, aRequest, requestInit, aRv); + if (NS_WARN_IF(aRv.Failed())) { + return nullptr; + } + + nsAutoString url; + request->GetUrl(url); + if (NS_WARN_IF(!IsValidPutRequestURL(url, aRv))) { + return nullptr; + } + + requestList.AppendElement(std::move(request)); + return AddAll(global, std::move(requestList), aCallerType, aRv); +} + +already_AddRefed<Promise> Cache::AddAll( + JSContext* aContext, const Sequence<OwningRequestOrUSVString>& aRequestList, + CallerType aCallerType, ErrorResult& aRv) { + if (NS_WARN_IF(!mActor)) { + aRv.Throw(NS_ERROR_UNEXPECTED); + return nullptr; + } + + CacheChild::AutoLock actorLock(*mActor); + + GlobalObject global(aContext, mGlobal->GetGlobalJSObject()); + MOZ_DIAGNOSTIC_ASSERT(!global.Failed()); + + nsTArray<SafeRefPtr<Request>> requestList(aRequestList.Length()); + for (uint32_t i = 0; i < aRequestList.Length(); ++i) { + RequestOrUSVString requestOrString; + + if (aRequestList[i].IsRequest()) { + requestOrString.SetAsRequest() = aRequestList[i].GetAsRequest(); + if (NS_WARN_IF( + !IsValidPutRequestMethod(requestOrString.GetAsRequest(), aRv))) { + return nullptr; + } + } else { + requestOrString.SetAsUSVString().ShareOrDependUpon( + aRequestList[i].GetAsUSVString()); + } + + RootedDictionary<RequestInit> requestInit(aContext); + SafeRefPtr<Request> request = + Request::Constructor(global, requestOrString, requestInit, aRv); + if (NS_WARN_IF(aRv.Failed())) { + return nullptr; + } + + nsAutoString url; + request->GetUrl(url); + if (NS_WARN_IF(!IsValidPutRequestURL(url, aRv))) { + return nullptr; + } + + requestList.AppendElement(std::move(request)); + } + + return AddAll(global, std::move(requestList), aCallerType, aRv); +} + +already_AddRefed<Promise> Cache::Put(JSContext* aCx, + const RequestOrUSVString& aRequest, + Response& aResponse, ErrorResult& aRv) { + if (NS_WARN_IF(!mActor)) { + aRv.Throw(NS_ERROR_UNEXPECTED); + return nullptr; + } + + CacheChild::AutoLock actorLock(*mActor); + + if (NS_WARN_IF(!IsValidPutRequestMethod(aRequest, aRv))) { + return nullptr; + } + + if (!IsValidPutResponseStatus(aResponse, PutStatusPolicy::Default, aRv)) { + return nullptr; + } + + if (NS_WARN_IF(aResponse.GetPrincipalInfo() && + aResponse.GetPrincipalInfo()->type() == + mozilla::ipc::PrincipalInfo::TExpandedPrincipalInfo)) { + // WebExtensions Content Scripts can currently run fetch from their global + // which will end up to have an expanded principal, but we require that the + // contents of Cache storage for the content origin to be same-origin, and + // never an expanded principal (See Bug 1753810). + aRv.ThrowSecurityError("Disallowed on WebExtension ContentScript Request"); + return nullptr; + } + + SafeRefPtr<InternalRequest> ir = + ToInternalRequest(aCx, aRequest, ReadBody, aRv); + if (NS_WARN_IF(aRv.Failed())) { + return nullptr; + } + + AutoChildOpArgs args(this, CachePutAllArgs(), 1); + + args.Add(aCx, *ir, ReadBody, TypeErrorOnInvalidScheme, aResponse, aRv); + if (NS_WARN_IF(aRv.Failed())) { + return nullptr; + } + + return ExecuteOp(args, aRv); +} + +already_AddRefed<Promise> Cache::Delete(JSContext* aCx, + const RequestOrUSVString& aRequest, + const CacheQueryOptions& aOptions, + ErrorResult& aRv) { + if (NS_WARN_IF(!mActor)) { + aRv.Throw(NS_ERROR_UNEXPECTED); + return nullptr; + } + + CacheChild::AutoLock actorLock(*mActor); + + SafeRefPtr<InternalRequest> ir = + ToInternalRequest(aCx, aRequest, IgnoreBody, aRv); + if (NS_WARN_IF(aRv.Failed())) { + return nullptr; + } + + CacheQueryParams params; + ToCacheQueryParams(params, aOptions); + + AutoChildOpArgs args(this, CacheDeleteArgs(CacheRequest(), params), 1); + + args.Add(*ir, IgnoreBody, IgnoreInvalidScheme, aRv); + if (NS_WARN_IF(aRv.Failed())) { + return nullptr; + } + + return ExecuteOp(args, aRv); +} + +already_AddRefed<Promise> Cache::Keys( + JSContext* aCx, const Optional<RequestOrUSVString>& aRequest, + const CacheQueryOptions& aOptions, ErrorResult& aRv) { + if (NS_WARN_IF(!mActor)) { + aRv.Throw(NS_ERROR_UNEXPECTED); + return nullptr; + } + + CacheChild::AutoLock actorLock(*mActor); + + CacheQueryParams params; + ToCacheQueryParams(params, aOptions); + + AutoChildOpArgs args(this, CacheKeysArgs(Nothing(), params, GetOpenMode()), + 1); + + if (aRequest.WasPassed()) { + SafeRefPtr<InternalRequest> ir = + ToInternalRequest(aCx, aRequest.Value(), IgnoreBody, aRv); + if (NS_WARN_IF(aRv.Failed())) { + return nullptr; + } + + args.Add(*ir, IgnoreBody, IgnoreInvalidScheme, aRv); + if (NS_WARN_IF(aRv.Failed())) { + return nullptr; + } + } + + return ExecuteOp(args, aRv); +} + +nsISupports* Cache::GetParentObject() const { return mGlobal; } + +JSObject* Cache::WrapObject(JSContext* aContext, + JS::Handle<JSObject*> aGivenProto) { + return Cache_Binding::Wrap(aContext, this, aGivenProto); +} + +void Cache::DestroyInternal(CacheChild* aActor) { + MOZ_DIAGNOSTIC_ASSERT(mActor); + MOZ_DIAGNOSTIC_ASSERT(mActor == aActor); + mActor->ClearListener(); + mActor = nullptr; +} + +nsIGlobalObject* Cache::GetGlobalObject() const { return mGlobal; } + +#ifdef DEBUG +void Cache::AssertOwningThread() const { NS_ASSERT_OWNINGTHREAD(Cache); } +#endif + +PBackgroundChild* Cache::GetIPCManager() { + NS_ASSERT_OWNINGTHREAD(Cache); + MOZ_DIAGNOSTIC_ASSERT(mActor); + return mActor->Manager(); +} + +Cache::~Cache() { + NS_ASSERT_OWNINGTHREAD(Cache); + if (mActor) { + mActor->StartDestroyFromListener(); + // DestroyInternal() is called synchronously by StartDestroyFromListener(). + // So we should have already cleared the mActor. + MOZ_DIAGNOSTIC_ASSERT(!mActor); + } +} + +already_AddRefed<Promise> Cache::ExecuteOp(AutoChildOpArgs& aOpArgs, + ErrorResult& aRv) { + MOZ_DIAGNOSTIC_ASSERT(mActor); + + RefPtr<Promise> promise = Promise::Create(mGlobal, aRv); + if (NS_WARN_IF(!promise)) { + return nullptr; + } + + mActor->ExecuteOp(mGlobal, promise, this, aOpArgs.SendAsOpArgs()); + return promise.forget(); +} + +already_AddRefed<Promise> Cache::AddAll( + const GlobalObject& aGlobal, nsTArray<SafeRefPtr<Request>>&& aRequestList, + CallerType aCallerType, ErrorResult& aRv) { + MOZ_DIAGNOSTIC_ASSERT(mActor); + + // If there is no work to do, then resolve immediately + if (aRequestList.IsEmpty()) { + RefPtr<Promise> promise = Promise::Create(mGlobal, aRv); + if (NS_WARN_IF(!promise)) { + return nullptr; + } + + promise->MaybeResolveWithUndefined(); + return promise.forget(); + } + + AutoTArray<RefPtr<Promise>, 256> fetchList; + fetchList.SetCapacity(aRequestList.Length()); + + // Begin fetching each request in parallel. For now, if an error occurs just + // abandon our previous fetch calls. In theory we could cancel them in the + // future once fetch supports it. + + for (uint32_t i = 0; i < aRequestList.Length(); ++i) { + RequestOrUSVString requestOrString; + requestOrString.SetAsRequest() = aRequestList[i].unsafeGetRawPtr(); + RootedDictionary<RequestInit> requestInit(aGlobal.Context()); + RefPtr<Promise> fetch = + FetchRequest(mGlobal, requestOrString, requestInit, aCallerType, aRv); + if (NS_WARN_IF(aRv.Failed())) { + return nullptr; + } + + fetchList.AppendElement(std::move(fetch)); + } + + RefPtr<Promise> promise = Promise::Create(mGlobal, aRv); + if (NS_WARN_IF(aRv.Failed())) { + return nullptr; + } + + RefPtr<FetchHandler> handler = + new FetchHandler(mActor->GetWorkerRefPtr().clonePtr(), this, + std::move(aRequestList), promise); + + RefPtr<Promise> fetchPromise = + Promise::All(aGlobal.Context(), fetchList, aRv); + if (NS_WARN_IF(aRv.Failed())) { + return nullptr; + } + fetchPromise->AppendNativeHandler(handler); + + return promise.forget(); +} + +already_AddRefed<Promise> Cache::PutAll( + JSContext* aCx, const nsTArray<SafeRefPtr<Request>>& aRequestList, + const nsTArray<RefPtr<Response>>& aResponseList, ErrorResult& aRv) { + MOZ_DIAGNOSTIC_ASSERT(aRequestList.Length() == aResponseList.Length()); + + if (NS_WARN_IF(!mActor)) { + aRv.Throw(NS_ERROR_UNEXPECTED); + return nullptr; + } + + CacheChild::AutoLock actorLock(*mActor); + + AutoChildOpArgs args(this, CachePutAllArgs(), aRequestList.Length()); + + for (uint32_t i = 0; i < aRequestList.Length(); ++i) { + SafeRefPtr<InternalRequest> ir = aRequestList[i]->GetInternalRequest(); + args.Add(aCx, *ir, ReadBody, TypeErrorOnInvalidScheme, *aResponseList[i], + aRv); + if (NS_WARN_IF(aRv.Failed())) { + return nullptr; + } + } + + return ExecuteOp(args, aRv); +} + +OpenMode Cache::GetOpenMode() const { + return mNamespace == CHROME_ONLY_NAMESPACE ? OpenMode::Eager : OpenMode::Lazy; +} + +} // namespace mozilla::dom::cache diff --git a/dom/cache/Cache.h b/dom/cache/Cache.h new file mode 100644 index 0000000000..02ccb557f7 --- /dev/null +++ b/dom/cache/Cache.h @@ -0,0 +1,124 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_Cache_h +#define mozilla_dom_cache_Cache_h + +#include "mozilla/dom/cache/Types.h" +#include "mozilla/dom/cache/TypeUtils.h" +#include "nsCOMPtr.h" +#include "nsISupportsImpl.h" +#include "nsString.h" +#include "nsWrapperCache.h" + +class nsIGlobalObject; + +namespace mozilla { + +class ErrorResult; + +namespace dom { + +class OwningRequestOrUSVString; +class Promise; +struct CacheQueryOptions; +class RequestOrUSVString; +class Response; +template <typename T> +class Optional; +template <typename T> +class Sequence; +enum class CallerType : uint32_t; + +namespace cache { + +class AutoChildOpArgs; +class CacheChild; + +class Cache final : public nsISupports, + public nsWrapperCache, + public TypeUtils { + public: + Cache(nsIGlobalObject* aGlobal, CacheChild* aActor, Namespace aNamespace); + + // webidl interface methods + already_AddRefed<Promise> Match(JSContext* aCx, + const RequestOrUSVString& aRequest, + const CacheQueryOptions& aOptions, + ErrorResult& aRv); + already_AddRefed<Promise> MatchAll( + JSContext* aCx, const Optional<RequestOrUSVString>& aRequest, + const CacheQueryOptions& aOptions, ErrorResult& aRv); + already_AddRefed<Promise> Add(JSContext* aContext, + const RequestOrUSVString& aRequest, + CallerType aCallerType, ErrorResult& aRv); + already_AddRefed<Promise> AddAll( + JSContext* aContext, const Sequence<OwningRequestOrUSVString>& aRequests, + CallerType aCallerType, ErrorResult& aRv); + already_AddRefed<Promise> Put(JSContext* aCx, + const RequestOrUSVString& aRequest, + Response& aResponse, ErrorResult& aRv); + already_AddRefed<Promise> Delete(JSContext* aCx, + const RequestOrUSVString& aRequest, + const CacheQueryOptions& aOptions, + ErrorResult& aRv); + already_AddRefed<Promise> Keys(JSContext* aCx, + const Optional<RequestOrUSVString>& aRequest, + const CacheQueryOptions& aParams, + ErrorResult& aRv); + + // binding methods + nsISupports* GetParentObject() const; + virtual JSObject* WrapObject(JSContext* aContext, + JS::Handle<JSObject*> aGivenProto) override; + + // Called when CacheChild actor is being destroyed + void DestroyInternal(CacheChild* aActor); + + // TypeUtils methods + virtual nsIGlobalObject* GetGlobalObject() const override; + +#ifdef DEBUG + virtual void AssertOwningThread() const override; +#endif + + virtual mozilla::ipc::PBackgroundChild* GetIPCManager() override; + + private: + class FetchHandler; + + ~Cache(); + + // Called when we're destroyed or CCed. + void DisconnectFromActor(); + + already_AddRefed<Promise> ExecuteOp(AutoChildOpArgs& aOpArgs, + ErrorResult& aRv); + + already_AddRefed<Promise> AddAll(const GlobalObject& aGlobal, + nsTArray<SafeRefPtr<Request>>&& aRequestList, + CallerType aCallerType, ErrorResult& aRv); + + already_AddRefed<Promise> PutAll( + JSContext* aCx, const nsTArray<SafeRefPtr<Request>>& aRequestList, + const nsTArray<RefPtr<Response>>& aResponseList, ErrorResult& aRv); + + OpenMode GetOpenMode() const; + + nsCOMPtr<nsIGlobalObject> mGlobal; + CacheChild* mActor; + const Namespace mNamespace; + + public: + NS_DECL_CYCLE_COLLECTING_ISUPPORTS + NS_DECL_CYCLE_COLLECTION_WRAPPERCACHE_CLASS(Cache) +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_Cache_h diff --git a/dom/cache/CacheChild.cpp b/dom/cache/CacheChild.cpp new file mode 100644 index 0000000000..c822653fea --- /dev/null +++ b/dom/cache/CacheChild.cpp @@ -0,0 +1,136 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/CacheChild.h" + +#include "mozilla/Unused.h" +#include "mozilla/dom/cache/ActorUtils.h" +#include "mozilla/dom/cache/Cache.h" +#include "mozilla/dom/cache/CacheOpChild.h" +#include "CacheWorkerRef.h" + +namespace mozilla::dom::cache { + +// Declared in ActorUtils.h +already_AddRefed<PCacheChild> AllocPCacheChild() { + return MakeAndAddRef<CacheChild>(); +} + +// Declared in ActorUtils.h +void DeallocPCacheChild(PCacheChild* aActor) { delete aActor; } + +CacheChild::CacheChild() + : mListener(nullptr), mLocked(false), mDelayedDestroy(false) { + MOZ_COUNT_CTOR(cache::CacheChild); +} + +CacheChild::~CacheChild() { + MOZ_COUNT_DTOR(cache::CacheChild); + NS_ASSERT_OWNINGTHREAD(CacheChild); + MOZ_DIAGNOSTIC_ASSERT(!mListener); + MOZ_DIAGNOSTIC_ASSERT(!mLocked); +} + +void CacheChild::SetListener(Cache* aListener) { + NS_ASSERT_OWNINGTHREAD(CacheChild); + MOZ_DIAGNOSTIC_ASSERT(!mListener); + mListener = aListener; + MOZ_DIAGNOSTIC_ASSERT(mListener); +} + +void CacheChild::ClearListener() { + NS_ASSERT_OWNINGTHREAD(CacheChild); + MOZ_DIAGNOSTIC_ASSERT(mListener); + mListener = nullptr; +} + +void CacheChild::ExecuteOp(nsIGlobalObject* aGlobal, Promise* aPromise, + nsISupports* aParent, const CacheOpArgs& aArgs) { + MOZ_ALWAYS_TRUE(SendPCacheOpConstructor( + new CacheOpChild(GetWorkerRefPtr().clonePtr(), aGlobal, aParent, aPromise, + this), + aArgs)); +} + +void CacheChild::StartDestroyFromListener() { + NS_ASSERT_OWNINGTHREAD(CacheChild); + + // The listener should be held alive by any async operations, so if it + // is going away then there must not be any child actors. This in turn + // ensures that StartDestroy() will not trigger the delayed path. + MOZ_DIAGNOSTIC_ASSERT(NumChildActors() == 0); + StartDestroy(); +} + +void CacheChild::DestroyInternal() { + RefPtr<Cache> listener = mListener; + + // StartDestroy() can get called from either Cache or the WorkerRef. + // Theoretically we can get double called if the right race happens. Handle + // that by just ignoring the second StartDestroy() call. + if (!listener) { + return; + } + + listener->DestroyInternal(this); + + // Cache listener should call ClearListener() in DestroyInternal() + MOZ_DIAGNOSTIC_ASSERT(!mListener); + + // Start actor destruction from parent process + QM_WARNONLY_TRY(OkIf(SendTeardown())); +} + +void CacheChild::StartDestroy() { + NS_ASSERT_OWNINGTHREAD(CacheChild); + + if (NumChildActors() != 0 || mLocked) { + mDelayedDestroy = true; + return; + } + + DestroyInternal(); +} + +void CacheChild::ActorDestroy(ActorDestroyReason aReason) { + NS_ASSERT_OWNINGTHREAD(CacheChild); + RefPtr<Cache> listener = mListener; + if (listener) { + listener->DestroyInternal(this); + // Cache listener should call ClearListener() in DestroyInternal() + MOZ_DIAGNOSTIC_ASSERT(!mListener); + } + + RemoveWorkerRef(); +} + +void CacheChild::NoteDeletedActor() { + // Check to see if DestroyInternal was delayed because there were still active + // CacheOpChilds when StartDestroy was called from WorkerRef notification. If + // the last CacheOpChild is getting destructed; it's the time for us to + // SendTearDown to the other side. + if (NumChildActors() == 1 && mDelayedDestroy && !mLocked) DestroyInternal(); +} + +already_AddRefed<PCacheOpChild> CacheChild::AllocPCacheOpChild( + const CacheOpArgs& aOpArgs) { + MOZ_CRASH("CacheOpChild should be manually constructed."); + return nullptr; +} + +void CacheChild::Lock() { + NS_ASSERT_OWNINGTHREAD(CacheChild); + MOZ_DIAGNOSTIC_ASSERT(!mLocked); + mLocked = true; +} + +void CacheChild::Unlock() { + NS_ASSERT_OWNINGTHREAD(CacheChild); + MOZ_DIAGNOSTIC_ASSERT(mLocked); + mLocked = false; +} + +} // namespace mozilla::dom::cache diff --git a/dom/cache/CacheChild.h b/dom/cache/CacheChild.h new file mode 100644 index 0000000000..7f37c79064 --- /dev/null +++ b/dom/cache/CacheChild.h @@ -0,0 +1,84 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_CacheChild_h +#define mozilla_dom_cache_CacheChild_h + +#include "mozilla/dom/cache/ActorChild.h" +#include "mozilla/dom/cache/PCacheChild.h" + +class nsIAsyncInputStream; +class nsIGlobalObject; + +namespace mozilla::dom { + +class Promise; + +namespace cache { + +class Cache; +class CacheOpArgs; + +class CacheChild final : public PCacheChild, public ActorChild { + friend class PCacheChild; + + public: + friend class mozilla::detail::BaseAutoLock<CacheChild&>; + using AutoLock = mozilla::detail::BaseAutoLock<CacheChild&>; + + CacheChild(); + + void SetListener(Cache* aListener); + + // Must be called by the associated Cache listener in its DestroyInternal() + // method. Also, Cache must call StartDestroyFromListener() on the actor in + // its destructor to trigger ActorDestroy() if it has not been called yet. + void ClearListener(); + + void ExecuteOp(nsIGlobalObject* aGlobal, Promise* aPromise, + nsISupports* aParent, const CacheOpArgs& aArgs); + + // Our parent Listener object has gone out of scope and is being destroyed. + void StartDestroyFromListener(); + void NoteDeletedActor() override; + + NS_INLINE_DECL_REFCOUNTING(CacheChild, override); + + private: + ~CacheChild(); + + void DestroyInternal(); + // ActorChild methods + // WorkerRef is trying to destroy due to worker shutdown. + virtual void StartDestroy() override; + + // PCacheChild methods + virtual void ActorDestroy(ActorDestroyReason aReason) override; + + already_AddRefed<PCacheOpChild> AllocPCacheOpChild( + const CacheOpArgs& aOpArgs); + + // utility methods + inline uint32_t NumChildActors() { return ManagedPCacheOpChild().Count(); } + + // Methods used to temporarily force the actor alive. Only called from + // AutoLock. + void Lock(); + + void Unlock(); + + // Use a weak ref so actor does not hold DOM object alive past content use. + // The Cache object must call ClearListener() to null this before its + // destroyed. + Cache* MOZ_NON_OWNING_REF mListener; + bool mLocked; + bool mDelayedDestroy; +}; + +} // namespace cache +} // namespace mozilla::dom + +#endif // mozilla_dom_cache_CacheChild_h diff --git a/dom/cache/CacheCommon.cpp b/dom/cache/CacheCommon.cpp new file mode 100644 index 0000000000..b259fe4a1e --- /dev/null +++ b/dom/cache/CacheCommon.cpp @@ -0,0 +1,7 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "CacheCommon.h" diff --git a/dom/cache/CacheCommon.h b/dom/cache/CacheCommon.h new file mode 100644 index 0000000000..82bb9ff829 --- /dev/null +++ b/dom/cache/CacheCommon.h @@ -0,0 +1,12 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_CacheCommon_h +#define mozilla_dom_cache_CacheCommon_h + +#include "mozilla/dom/quota/QuotaCommon.h" + +#endif // mozilla_dom_cache_CacheCommon_h diff --git a/dom/cache/CacheOpChild.cpp b/dom/cache/CacheOpChild.cpp new file mode 100644 index 0000000000..b0c5c78bfc --- /dev/null +++ b/dom/cache/CacheOpChild.cpp @@ -0,0 +1,236 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/CacheOpChild.h" + +#include "mozilla/dom/Promise.h" +#include "mozilla/dom/Request.h" +#include "mozilla/dom/Response.h" +#include "mozilla/dom/cache/Cache.h" +#include "mozilla/dom/cache/CacheChild.h" +#include "mozilla/dom/cache/CacheStreamControlChild.h" +#include "mozilla/dom/cache/CacheWorkerRef.h" + +namespace mozilla::dom { +// XXX Move this to ToJSValue.h +template <typename T> +[[nodiscard]] bool ToJSValue(JSContext* aCx, const SafeRefPtr<T>& aArgument, + JS::MutableHandle<JS::Value> aValue) { + return ToJSValue(aCx, *aArgument.unsafeGetRawPtr(), aValue); +} + +namespace cache { + +using mozilla::ipc::PBackgroundChild; + +namespace { + +void AddWorkerRefToStreamChild(const CacheReadStream& aReadStream, + const SafeRefPtr<CacheWorkerRef>& aWorkerRef) { + MOZ_ASSERT_IF(!NS_IsMainThread(), aWorkerRef); + CacheStreamControlChild* cacheControl = + static_cast<CacheStreamControlChild*>(aReadStream.control().AsChild()); + if (cacheControl) { + cacheControl->SetWorkerRef(aWorkerRef.clonePtr()); + } +} + +void AddWorkerRefToStreamChild(const CacheResponse& aResponse, + const SafeRefPtr<CacheWorkerRef>& aWorkerRef) { + MOZ_ASSERT_IF(!NS_IsMainThread(), aWorkerRef); + + if (aResponse.body().isNothing()) { + return; + } + + AddWorkerRefToStreamChild(aResponse.body().ref(), aWorkerRef); +} + +void AddWorkerRefToStreamChild(const CacheRequest& aRequest, + const SafeRefPtr<CacheWorkerRef>& aWorkerRef) { + MOZ_ASSERT_IF(!NS_IsMainThread(), aWorkerRef); + + if (aRequest.body().isNothing()) { + return; + } + + AddWorkerRefToStreamChild(aRequest.body().ref(), aWorkerRef); +} + +} // namespace + +CacheOpChild::CacheOpChild(SafeRefPtr<CacheWorkerRef> aWorkerRef, + nsIGlobalObject* aGlobal, nsISupports* aParent, + Promise* aPromise, ActorChild* aParentActor) + : mGlobal(aGlobal), + mParent(aParent), + mPromise(aPromise), + mParentActor(aParentActor) { + MOZ_DIAGNOSTIC_ASSERT(mGlobal); + MOZ_DIAGNOSTIC_ASSERT(mParent); + MOZ_DIAGNOSTIC_ASSERT(mPromise); + + MOZ_ASSERT_IF(!NS_IsMainThread(), aWorkerRef); + + SetWorkerRef(CacheWorkerRef::PreferBehavior( + std::move(aWorkerRef), CacheWorkerRef::eStrongWorkerRef)); +} + +CacheOpChild::~CacheOpChild() { + NS_ASSERT_OWNINGTHREAD(CacheOpChild); + MOZ_DIAGNOSTIC_ASSERT(!mPromise); +} + +void CacheOpChild::ActorDestroy(ActorDestroyReason aReason) { + NS_ASSERT_OWNINGTHREAD(CacheOpChild); + + // If the actor was terminated for some unknown reason, then indicate the + // operation is dead. + if (mPromise) { + mPromise->MaybeReject(NS_ERROR_FAILURE); + mPromise = nullptr; + } + mParentActor->NoteDeletedActor(); + RemoveWorkerRef(); +} + +mozilla::ipc::IPCResult CacheOpChild::Recv__delete__( + ErrorResult&& aRv, const CacheOpResult& aResult) { + NS_ASSERT_OWNINGTHREAD(CacheOpChild); + + if (NS_WARN_IF(aRv.Failed())) { + MOZ_DIAGNOSTIC_ASSERT(aResult.type() == CacheOpResult::Tvoid_t); + mPromise->MaybeReject(std::move(aRv)); + mPromise = nullptr; + return IPC_OK(); + } + + switch (aResult.type()) { + case CacheOpResult::TCacheMatchResult: { + HandleResponse(aResult.get_CacheMatchResult().maybeResponse()); + break; + } + case CacheOpResult::TCacheMatchAllResult: { + HandleResponseList(aResult.get_CacheMatchAllResult().responseList()); + break; + } + case CacheOpResult::TCachePutAllResult: { + mPromise->MaybeResolveWithUndefined(); + break; + } + case CacheOpResult::TCacheDeleteResult: { + mPromise->MaybeResolve(aResult.get_CacheDeleteResult().success()); + break; + } + case CacheOpResult::TCacheKeysResult: { + HandleRequestList(aResult.get_CacheKeysResult().requestList()); + break; + } + case CacheOpResult::TStorageMatchResult: { + HandleResponse(aResult.get_StorageMatchResult().maybeResponse()); + break; + } + case CacheOpResult::TStorageHasResult: { + mPromise->MaybeResolve(aResult.get_StorageHasResult().success()); + break; + } + case CacheOpResult::TStorageOpenResult: { + auto result = aResult.get_StorageOpenResult(); + auto actor = static_cast<CacheChild*>(result.actor().AsChild()); + + // If we have a success status then we should have an actor. Gracefully + // reject instead of crashing, though, if we get a nullptr here. + MOZ_DIAGNOSTIC_ASSERT(actor); + if (!actor) { + mPromise->MaybeRejectWithTypeError( + "CacheStorage.open() failed to access the storage system."); + break; + } + + actor->SetWorkerRef(CacheWorkerRef::PreferBehavior( + GetWorkerRefPtr().clonePtr(), CacheWorkerRef::eIPCWorkerRef)); + RefPtr<Cache> cache = new Cache(mGlobal, actor, result.ns()); + mPromise->MaybeResolve(cache); + break; + } + case CacheOpResult::TStorageDeleteResult: { + mPromise->MaybeResolve(aResult.get_StorageDeleteResult().success()); + break; + } + case CacheOpResult::TStorageKeysResult: { + mPromise->MaybeResolve(aResult.get_StorageKeysResult().keyList()); + break; + } + default: + MOZ_CRASH("Unknown Cache op result type!"); + } + + mPromise = nullptr; + + return IPC_OK(); +} + +void CacheOpChild::StartDestroy() { + NS_ASSERT_OWNINGTHREAD(CacheOpChild); + + // Do not cancel on-going operations when WorkerRef calls this. Instead, + // keep the Worker alive until we are done. +} + +nsIGlobalObject* CacheOpChild::GetGlobalObject() const { return mGlobal; } + +#ifdef DEBUG +void CacheOpChild::AssertOwningThread() const { + NS_ASSERT_OWNINGTHREAD(CacheOpChild); +} +#endif + +PBackgroundChild* CacheOpChild::GetIPCManager() { + MOZ_CRASH("CacheOpChild does not implement TypeUtils::GetIPCManager()"); +} + +void CacheOpChild::HandleResponse(const Maybe<CacheResponse>& aMaybeResponse) { + if (aMaybeResponse.isNothing()) { + mPromise->MaybeResolveWithUndefined(); + return; + } + + const CacheResponse& cacheResponse = aMaybeResponse.ref(); + + AddWorkerRefToStreamChild(cacheResponse, GetWorkerRefPtr()); + RefPtr<Response> response = ToResponse(cacheResponse); + + mPromise->MaybeResolve(response); +} + +void CacheOpChild::HandleResponseList( + const nsTArray<CacheResponse>& aResponseList) { + AutoTArray<RefPtr<Response>, 256> responses; + responses.SetCapacity(aResponseList.Length()); + + for (uint32_t i = 0; i < aResponseList.Length(); ++i) { + AddWorkerRefToStreamChild(aResponseList[i], GetWorkerRefPtr()); + responses.AppendElement(ToResponse(aResponseList[i])); + } + + mPromise->MaybeResolve(responses); +} + +void CacheOpChild::HandleRequestList( + const nsTArray<CacheRequest>& aRequestList) { + AutoTArray<SafeRefPtr<Request>, 256> requests; + requests.SetCapacity(aRequestList.Length()); + + for (uint32_t i = 0; i < aRequestList.Length(); ++i) { + AddWorkerRefToStreamChild(aRequestList[i], GetWorkerRefPtr()); + requests.AppendElement(ToRequest(aRequestList[i])); + } + + mPromise->MaybeResolve(requests); +} + +} // namespace cache +} // namespace mozilla::dom diff --git a/dom/cache/CacheOpChild.h b/dom/cache/CacheOpChild.h new file mode 100644 index 0000000000..b41a8edb1a --- /dev/null +++ b/dom/cache/CacheOpChild.h @@ -0,0 +1,77 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_CacheOpChild_h +#define mozilla_dom_cache_CacheOpChild_h + +#include "mozilla/dom/cache/ActorChild.h" +#include "mozilla/dom/cache/PCacheOpChild.h" +#include "mozilla/dom/cache/TypeUtils.h" +#include "mozilla/RefPtr.h" + +class nsIGlobalObject; + +namespace mozilla::dom { + +class Promise; + +namespace cache { + +class CacheOpChild final : public PCacheOpChild, + public ActorChild, + public TypeUtils { + friend class CacheChild; + friend class CacheStorageChild; + friend class PCacheOpChild; + + public: + NS_INLINE_DECL_REFCOUNTING(CacheOpChild, override) + + private: + // This class must be constructed by CacheChild or CacheStorageChild using + // their ExecuteOp() factory method. + CacheOpChild(SafeRefPtr<CacheWorkerRef> aWorkerRef, nsIGlobalObject* aGlobal, + nsISupports* aParent, Promise* aPromise, + ActorChild* aParentActor); + ~CacheOpChild(); + + // PCacheOpChild methods + virtual void ActorDestroy(ActorDestroyReason aReason) override; + + mozilla::ipc::IPCResult Recv__delete__(ErrorResult&& aRv, + const CacheOpResult& aResult); + + // ActorChild methods + virtual void StartDestroy() override; + + // TypeUtils methods + virtual nsIGlobalObject* GetGlobalObject() const override; + +#ifdef DEBUG + virtual void AssertOwningThread() const override; +#endif + + virtual mozilla::ipc::PBackgroundChild* GetIPCManager() override; + + // Utility methods + void HandleResponse(const Maybe<CacheResponse>& aMaybeResponse); + + void HandleResponseList(const nsTArray<CacheResponse>& aResponseList); + + void HandleRequestList(const nsTArray<CacheRequest>& aRequestList); + + nsCOMPtr<nsIGlobalObject> mGlobal; + // Hold the parent Cache or CacheStorage object alive until this async + // operation completes. + nsCOMPtr<nsISupports> mParent; + RefPtr<Promise> mPromise; + ActorChild* mParentActor; +}; + +} // namespace cache +} // namespace mozilla::dom + +#endif // mozilla_dom_cache_CacheOpChild_h diff --git a/dom/cache/CacheOpParent.cpp b/dom/cache/CacheOpParent.cpp new file mode 100644 index 0000000000..288eb5618b --- /dev/null +++ b/dom/cache/CacheOpParent.cpp @@ -0,0 +1,323 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/CacheOpParent.h" + +#include "mozilla/StaticPrefs_browser.h" +#include "mozilla/Unused.h" +#include "mozilla/dom/cache/AutoUtils.h" +#include "mozilla/dom/cache/ManagerId.h" +#include "mozilla/dom/cache/ReadStream.h" +#include "mozilla/dom/cache/SavedTypes.h" +#include "mozilla/ipc/InputStreamUtils.h" +#include "mozilla/ipc/IPCStreamUtils.h" + +namespace mozilla::dom::cache { + +using mozilla::ipc::PBackgroundParent; + +CacheOpParent::CacheOpParent(PBackgroundParent* aIpcManager, CacheId aCacheId, + const CacheOpArgs& aOpArgs) + : mIpcManager(aIpcManager), + mCacheId(aCacheId), + mNamespace(INVALID_NAMESPACE), + mOpArgs(aOpArgs) { + MOZ_DIAGNOSTIC_ASSERT(mIpcManager); +} + +CacheOpParent::CacheOpParent(PBackgroundParent* aIpcManager, + Namespace aNamespace, const CacheOpArgs& aOpArgs) + : mIpcManager(aIpcManager), + mCacheId(INVALID_CACHE_ID), + mNamespace(aNamespace), + mOpArgs(aOpArgs) { + MOZ_DIAGNOSTIC_ASSERT(mIpcManager); +} + +CacheOpParent::~CacheOpParent() { NS_ASSERT_OWNINGTHREAD(CacheOpParent); } + +void CacheOpParent::Execute(const SafeRefPtr<ManagerId>& aManagerId) { + NS_ASSERT_OWNINGTHREAD(CacheOpParent); + MOZ_DIAGNOSTIC_ASSERT(!mManager); + MOZ_DIAGNOSTIC_ASSERT(!mVerifier); + + auto managerOrErr = cache::Manager::AcquireCreateIfNonExistent(aManagerId); + if (NS_WARN_IF(managerOrErr.isErr())) { + ErrorResult result(managerOrErr.unwrapErr()); + Unused << Send__delete__(this, std::move(result), void_t()); + return; + } + + Execute(managerOrErr.unwrap()); +} + +void CacheOpParent::Execute(SafeRefPtr<cache::Manager> aManager) { + NS_ASSERT_OWNINGTHREAD(CacheOpParent); + MOZ_DIAGNOSTIC_ASSERT(!mManager); + MOZ_DIAGNOSTIC_ASSERT(!mVerifier); + + mManager = std::move(aManager); + + // Handle put op + if (mOpArgs.type() == CacheOpArgs::TCachePutAllArgs) { + MOZ_DIAGNOSTIC_ASSERT(mCacheId != INVALID_CACHE_ID); + + const CachePutAllArgs& args = mOpArgs.get_CachePutAllArgs(); + const nsTArray<CacheRequestResponse>& list = args.requestResponseList(); + + AutoTArray<nsCOMPtr<nsIInputStream>, 256> requestStreamList; + AutoTArray<nsCOMPtr<nsIInputStream>, 256> responseStreamList; + + for (uint32_t i = 0; i < list.Length(); ++i) { + requestStreamList.AppendElement( + DeserializeCacheStream(list[i].request().body())); + responseStreamList.AppendElement( + DeserializeCacheStream(list[i].response().body())); + } + + mManager->ExecutePutAll(this, mCacheId, args.requestResponseList(), + requestStreamList, responseStreamList); + return; + } + + // Handle all other cache ops + if (mCacheId != INVALID_CACHE_ID) { + MOZ_DIAGNOSTIC_ASSERT(mNamespace == INVALID_NAMESPACE); + mManager->ExecuteCacheOp(this, mCacheId, mOpArgs); + return; + } + + // Handle all storage ops + MOZ_DIAGNOSTIC_ASSERT(mNamespace != INVALID_NAMESPACE); + mManager->ExecuteStorageOp(this, mNamespace, mOpArgs); +} + +void CacheOpParent::WaitForVerification(PrincipalVerifier* aVerifier) { + NS_ASSERT_OWNINGTHREAD(CacheOpParent); + MOZ_DIAGNOSTIC_ASSERT(!mManager); + MOZ_DIAGNOSTIC_ASSERT(!mVerifier); + + mVerifier = aVerifier; + mVerifier->AddListener(*this); +} + +void CacheOpParent::ActorDestroy(ActorDestroyReason aReason) { + NS_ASSERT_OWNINGTHREAD(CacheOpParent); + + if (mVerifier) { + mVerifier->RemoveListener(*this); + mVerifier = nullptr; + } + + if (mManager) { + mManager->RemoveListener(this); + mManager = nullptr; + } + + mIpcManager = nullptr; +} + +void CacheOpParent::OnPrincipalVerified( + nsresult aRv, const SafeRefPtr<ManagerId>& aManagerId) { + NS_ASSERT_OWNINGTHREAD(CacheOpParent); + + mVerifier->RemoveListener(*this); + mVerifier = nullptr; + + if (NS_WARN_IF(NS_FAILED(aRv))) { + ErrorResult result(aRv); + Unused << Send__delete__(this, std::move(result), void_t()); + return; + } + + Execute(aManagerId); +} + +void CacheOpParent::OnOpComplete(ErrorResult&& aRv, + const CacheOpResult& aResult, + CacheId aOpenedCacheId, + const Maybe<StreamInfo>& aStreamInfo) { + NS_ASSERT_OWNINGTHREAD(CacheOpParent); + MOZ_DIAGNOSTIC_ASSERT(mIpcManager); + MOZ_DIAGNOSTIC_ASSERT(mManager); + + // Never send an op-specific result if we have an error. Instead, send + // void_t() to ensure that we don't leak actors on the child side. + if (NS_WARN_IF(aRv.Failed())) { + Unused << Send__delete__(this, std::move(aRv), void_t()); + return; + } + + if (aStreamInfo.isSome()) { + ProcessCrossOriginResourcePolicyHeader(aRv, + aStreamInfo->mSavedResponseList); + if (NS_WARN_IF(aRv.Failed())) { + Unused << Send__delete__(this, std::move(aRv), void_t()); + return; + } + } + + uint32_t entryCount = + std::max(1lu, aStreamInfo ? static_cast<unsigned long>(std::max( + aStreamInfo->mSavedResponseList.Length(), + aStreamInfo->mSavedRequestList.Length())) + : 0lu); + + // The result must contain the appropriate type at this point. It may + // or may not contain the additional result data yet. For types that + // do not need special processing, it should already be set. If the + // result requires actor-specific operations, then we do that below. + // If the type and data types don't match, then we will trigger an + // assertion in AutoParentOpResult::Add(). + AutoParentOpResult result(mIpcManager, aResult, entryCount); + + if (aOpenedCacheId != INVALID_CACHE_ID) { + result.Add(aOpenedCacheId, mManager.clonePtr()); + } + + if (aStreamInfo) { + const auto& streamInfo = *aStreamInfo; + + for (const auto& savedResponse : streamInfo.mSavedResponseList) { + result.Add(savedResponse, streamInfo.mStreamList); + } + + for (const auto& savedRequest : streamInfo.mSavedRequestList) { + result.Add(savedRequest, streamInfo.mStreamList); + } + } + + Unused << Send__delete__(this, std::move(aRv), result.SendAsOpResult()); +} + +already_AddRefed<nsIInputStream> CacheOpParent::DeserializeCacheStream( + const Maybe<CacheReadStream>& aMaybeStream) { + if (aMaybeStream.isNothing()) { + return nullptr; + } + + nsCOMPtr<nsIInputStream> stream; + const CacheReadStream& readStream = aMaybeStream.ref(); + + // Option 1: One of our own ReadStreams was passed back to us with a stream + // control actor. + stream = ReadStream::Create(readStream); + if (stream) { + return stream.forget(); + } + + // Option 2: A stream was serialized using normal methods or passed + // as a DataPipe. Use the standard method for extracting the + // resulting stream. + return DeserializeIPCStream(readStream.stream()); +} + +void CacheOpParent::ProcessCrossOriginResourcePolicyHeader( + ErrorResult& aRv, const nsTArray<SavedResponse>& aResponses) { + if (!StaticPrefs::browser_tabs_remote_useCrossOriginEmbedderPolicy()) { + return; + } + // Only checking for match/matchAll. + nsILoadInfo::CrossOriginEmbedderPolicy loadingCOEP = + nsILoadInfo::EMBEDDER_POLICY_NULL; + Maybe<mozilla::ipc::PrincipalInfo> principalInfo; + switch (mOpArgs.type()) { + case CacheOpArgs::TCacheMatchArgs: { + const auto& request = mOpArgs.get_CacheMatchArgs().request(); + loadingCOEP = request.loadingEmbedderPolicy(); + principalInfo = request.principalInfo(); + break; + } + case CacheOpArgs::TCacheMatchAllArgs: { + if (mOpArgs.get_CacheMatchAllArgs().maybeRequest().isSome()) { + const auto& request = + mOpArgs.get_CacheMatchAllArgs().maybeRequest().ref(); + loadingCOEP = request.loadingEmbedderPolicy(); + principalInfo = request.principalInfo(); + } + break; + } + default: { + return; + } + } + + // skip checking if the request has no principal for same-origin/same-site + // checking. + if (principalInfo.isNothing() || + principalInfo.ref().type() != + mozilla::ipc::PrincipalInfo::TContentPrincipalInfo) { + return; + } + const mozilla::ipc::ContentPrincipalInfo& contentPrincipalInfo = + principalInfo.ref().get_ContentPrincipalInfo(); + + for (auto it = aResponses.cbegin(); it != aResponses.cend(); ++it) { + if (it->mValue.type() != ResponseType::Opaque && + it->mValue.type() != ResponseType::Opaqueredirect) { + continue; + } + + const auto& headers = it->mValue.headers(); + const RequestCredentials credentials = it->mValue.credentials(); + const auto corpHeaderIt = + std::find_if(headers.cbegin(), headers.cend(), [](const auto& header) { + return header.name().EqualsLiteral("Cross-Origin-Resource-Policy"); + }); + + // According to https://github.com/w3c/ServiceWorker/issues/1490, the cache + // response is expected with CORP header, otherwise, throw the type error. + // Note that this is different with the CORP checking for fetch metioned in + // https://wicg.github.io/cross-origin-embedder-policy/#corp-check. + // For fetch, if the response has no CORP header, "same-origin" checking + // will be performed. + if (corpHeaderIt == headers.cend() && + loadingCOEP == nsILoadInfo::EMBEDDER_POLICY_REQUIRE_CORP) { + aRv.ThrowTypeError("Response is expected with CORP header."); + return; + } + + // Skip the case if the response has no principal for same-origin/same-site + // checking. + if (it->mValue.principalInfo().isNothing() || + it->mValue.principalInfo().ref().type() != + mozilla::ipc::PrincipalInfo::TContentPrincipalInfo) { + continue; + } + + const mozilla::ipc::ContentPrincipalInfo& responseContentPrincipalInfo = + it->mValue.principalInfo().ref().get_ContentPrincipalInfo(); + + nsCString corp = + corpHeaderIt == headers.cend() ? EmptyCString() : corpHeaderIt->value(); + + if (corp.IsEmpty()) { + if (loadingCOEP == nsILoadInfo::EMBEDDER_POLICY_CREDENTIALLESS) { + // This means the request of this request doesn't have + // credentials, so it's safe for us to return. + if (credentials == RequestCredentials::Omit) { + return; + } + corp = "same-origin"; + } + } + + if (corp.EqualsLiteral("same-origin")) { + if (responseContentPrincipalInfo != contentPrincipalInfo) { + aRv.ThrowTypeError("Response is expected from same origin."); + return; + } + } else if (corp.EqualsLiteral("same-site")) { + if (!responseContentPrincipalInfo.baseDomain().Equals( + contentPrincipalInfo.baseDomain())) { + aRv.ThrowTypeError("Response is expected from same site."); + return; + } + } + } +} + +} // namespace mozilla::dom::cache diff --git a/dom/cache/CacheOpParent.h b/dom/cache/CacheOpParent.h new file mode 100644 index 0000000000..a1a6308243 --- /dev/null +++ b/dom/cache/CacheOpParent.h @@ -0,0 +1,74 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_CacheOpParent_h +#define mozilla_dom_cache_CacheOpParent_h + +#include "mozilla/dom/cache/Manager.h" +#include "mozilla/dom/cache/PCacheOpParent.h" +#include "mozilla/dom/cache/PrincipalVerifier.h" +#include "nsTArray.h" + +namespace mozilla { +namespace ipc { +class PBackgroundParent; +} // namespace ipc +namespace dom::cache { + +class CacheOpParent final : public PCacheOpParent, + public PrincipalVerifier::Listener, + public Manager::Listener { + // to allow use of convenience overrides + using Manager::Listener::OnOpComplete; + + public: + CacheOpParent(mozilla::ipc::PBackgroundParent* aIpcManager, CacheId aCacheId, + const CacheOpArgs& aOpArgs); + CacheOpParent(mozilla::ipc::PBackgroundParent* aIpcManager, + Namespace aNamespace, const CacheOpArgs& aOpArgs); + + void Execute(const SafeRefPtr<ManagerId>& aManagerId); + + void Execute(SafeRefPtr<cache::Manager> aManager); + + void WaitForVerification(PrincipalVerifier* aVerifier); + + NS_INLINE_DECL_REFCOUNTING(CacheOpParent, override) + + private: + ~CacheOpParent(); + + // PCacheOpParent methods + virtual void ActorDestroy(ActorDestroyReason aReason) override; + + // PrincipalVerifier::Listener methods + virtual void OnPrincipalVerified( + nsresult aRv, const SafeRefPtr<ManagerId>& aManagerId) override; + + // Manager::Listener methods + virtual void OnOpComplete(ErrorResult&& aRv, const CacheOpResult& aResult, + CacheId aOpenedCacheId, + const Maybe<StreamInfo>& aStreamInfo) override; + + // utility methods + already_AddRefed<nsIInputStream> DeserializeCacheStream( + const Maybe<CacheReadStream>& aMaybeStream); + + void ProcessCrossOriginResourcePolicyHeader( + ErrorResult& aRv, const nsTArray<SavedResponse>& aResponses); + + mozilla::ipc::PBackgroundParent* mIpcManager; + const CacheId mCacheId; + const Namespace mNamespace; + const CacheOpArgs mOpArgs; + SafeRefPtr<cache::Manager> mManager; + RefPtr<PrincipalVerifier> mVerifier; +}; + +} // namespace dom::cache +} // namespace mozilla + +#endif // mozilla_dom_cache_CacheOpParent_h diff --git a/dom/cache/CacheParent.cpp b/dom/cache/CacheParent.cpp new file mode 100644 index 0000000000..c286256622 --- /dev/null +++ b/dom/cache/CacheParent.cpp @@ -0,0 +1,61 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/CacheParent.h" + +#include "mozilla/dom/cache/CacheOpParent.h" +#include "nsCOMPtr.h" + +namespace mozilla::dom::cache { + +// Declared in ActorUtils.h +void DeallocPCacheParent(PCacheParent* aActor) { delete aActor; } + +CacheParent::CacheParent(SafeRefPtr<cache::Manager> aManager, CacheId aCacheId) + : mManager(std::move(aManager)), mCacheId(aCacheId) { + MOZ_COUNT_CTOR(cache::CacheParent); + MOZ_DIAGNOSTIC_ASSERT(mManager); + mManager->AddRefCacheId(mCacheId); +} + +CacheParent::~CacheParent() { + MOZ_COUNT_DTOR(cache::CacheParent); + MOZ_DIAGNOSTIC_ASSERT(!mManager); +} + +void CacheParent::ActorDestroy(ActorDestroyReason aReason) { + MOZ_DIAGNOSTIC_ASSERT(mManager); + mManager->ReleaseCacheId(mCacheId); + mManager = nullptr; +} + +already_AddRefed<PCacheOpParent> CacheParent::AllocPCacheOpParent( + const CacheOpArgs& aOpArgs) { + if (aOpArgs.type() != CacheOpArgs::TCacheMatchArgs && + aOpArgs.type() != CacheOpArgs::TCacheMatchAllArgs && + aOpArgs.type() != CacheOpArgs::TCachePutAllArgs && + aOpArgs.type() != CacheOpArgs::TCacheDeleteArgs && + aOpArgs.type() != CacheOpArgs::TCacheKeysArgs) { + MOZ_CRASH("Invalid operation sent to Cache actor!"); + } + + return MakeAndAddRef<CacheOpParent>(Manager(), mCacheId, aOpArgs); +} + +mozilla::ipc::IPCResult CacheParent::RecvPCacheOpConstructor( + PCacheOpParent* aActor, const CacheOpArgs& aOpArgs) { + auto actor = static_cast<CacheOpParent*>(aActor); + actor->Execute(mManager.clonePtr()); + return IPC_OK(); +} + +mozilla::ipc::IPCResult CacheParent::RecvTeardown() { + // If child process is gone, warn and allow actor to clean up normally + QM_WARNONLY_TRY(OkIf(Send__delete__(this))); + return IPC_OK(); +} + +} // namespace mozilla::dom::cache diff --git a/dom/cache/CacheParent.h b/dom/cache/CacheParent.h new file mode 100644 index 0000000000..391c59cbb2 --- /dev/null +++ b/dom/cache/CacheParent.h @@ -0,0 +1,46 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_CacheParent_h +#define mozilla_dom_cache_CacheParent_h + +#include "mozilla/dom/SafeRefPtr.h" +#include "mozilla/dom/cache/PCacheParent.h" +#include "mozilla/dom/cache/Types.h" + +namespace mozilla::dom::cache { + +class Manager; + +class CacheParent final : public PCacheParent { + friend class PCacheParent; + + public: + CacheParent(SafeRefPtr<cache::Manager> aManager, CacheId aCacheId); + + private: + virtual ~CacheParent(); + + // PCacheParent methods + virtual void ActorDestroy(ActorDestroyReason aReason) override; + + already_AddRefed<PCacheOpParent> AllocPCacheOpParent( + const CacheOpArgs& aOpArgs); + + virtual mozilla::ipc::IPCResult RecvPCacheOpConstructor( + PCacheOpParent* actor, const CacheOpArgs& aOpArgs) override; + + mozilla::ipc::IPCResult RecvTeardown(); + + SafeRefPtr<cache::Manager> mManager; + const CacheId mCacheId; + + NS_INLINE_DECL_REFCOUNTING(CacheParent, override) +}; + +} // namespace mozilla::dom::cache + +#endif // mozilla_dom_cache_CacheParent_h diff --git a/dom/cache/CacheStorage.cpp b/dom/cache/CacheStorage.cpp new file mode 100644 index 0000000000..c1a1ead48a --- /dev/null +++ b/dom/cache/CacheStorage.cpp @@ -0,0 +1,592 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/CacheStorage.h" + +#include "mozilla/Preferences.h" +#include "mozilla/Unused.h" +#include "mozilla/dom/CacheBinding.h" +#include "mozilla/dom/CacheStorageBinding.h" +#include "mozilla/dom/InternalRequest.h" +#include "mozilla/dom/Promise.h" +#include "mozilla/dom/Response.h" +#include "mozilla/dom/cache/AutoUtils.h" +#include "mozilla/dom/cache/Cache.h" +#include "mozilla/dom/cache/CacheChild.h" +#include "mozilla/dom/cache/CacheCommon.h" +#include "mozilla/dom/cache/CacheStorageChild.h" +#include "mozilla/dom/cache/CacheWorkerRef.h" +#include "mozilla/dom/cache/PCacheChild.h" +#include "mozilla/dom/cache/ReadStream.h" +#include "mozilla/dom/cache/TypeUtils.h" +#include "mozilla/dom/quota/QuotaManager.h" +#include "mozilla/dom/quota/ResultExtensions.h" +#include "mozilla/dom/WorkerPrivate.h" +#include "mozilla/ipc/BackgroundChild.h" +#include "mozilla/ipc/BackgroundUtils.h" +#include "mozilla/ipc/PBackgroundChild.h" +#include "mozilla/ipc/PBackgroundSharedTypes.h" +#include "mozilla/StaticPrefs_dom.h" +#include "mozilla/StaticPrefs_extensions.h" +#include "nsContentUtils.h" +#include "mozilla/dom/Document.h" +#include "nsIGlobalObject.h" +#include "nsMixedContentBlocker.h" +#include "nsURLParsers.h" +#include "js/Object.h" // JS::GetClass +#include "js/PropertyAndElement.h" // JS_DefineProperty + +namespace mozilla::dom::cache { + +using mozilla::ErrorResult; +using mozilla::dom::quota::QuotaManager; +using mozilla::ipc::BackgroundChild; +using mozilla::ipc::PBackgroundChild; +using mozilla::ipc::PrincipalInfo; +using mozilla::ipc::PrincipalToPrincipalInfo; + +NS_IMPL_CYCLE_COLLECTING_ADDREF(mozilla::dom::cache::CacheStorage); +NS_IMPL_CYCLE_COLLECTING_RELEASE(mozilla::dom::cache::CacheStorage); +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(mozilla::dom::cache::CacheStorage, + mGlobal); + +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(CacheStorage) + NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY + NS_INTERFACE_MAP_ENTRY(nsISupports) +NS_INTERFACE_MAP_END + +// We cannot reference IPC types in a webidl binding implementation header. So +// define this in the .cpp. +struct CacheStorage::Entry final { + RefPtr<Promise> mPromise; + CacheOpArgs mArgs; + // We cannot add the requests until after the actor is present. So store + // the request data separately for now. + SafeRefPtr<InternalRequest> mRequest; +}; + +namespace { + +bool IsTrusted(const PrincipalInfo& aPrincipalInfo, bool aTestingPrefEnabled) { + // Can happen on main thread or worker thread + + if (aPrincipalInfo.type() == PrincipalInfo::TSystemPrincipalInfo) { + return true; + } + + // Require a ContentPrincipal to avoid null principal, etc. + QM_TRY(OkIf(aPrincipalInfo.type() == PrincipalInfo::TContentPrincipalInfo), + false); + + // If we're in testing mode, then don't do any more work to determine if + // the origin is trusted. We have to run some tests as http. + if (aTestingPrefEnabled) { + return true; + } + + // Now parse the scheme of the principal's origin. This is a short term + // method for determining "trust". In the long term we need to implement + // the full algorithm here: + // + // https://w3c.github.io/webappsec/specs/powerfulfeatures/#settings-secure + // + // TODO: Implement full secure setting algorithm. (bug 1177856) + + const nsCString& flatURL = aPrincipalInfo.get_ContentPrincipalInfo().spec(); + const char* const url = flatURL.get(); + + // off the main thread URL parsing using nsStdURLParser. + const nsCOMPtr<nsIURLParser> urlParser = new nsStdURLParser(); + + uint32_t schemePos; + int32_t schemeLen; + uint32_t authPos; + int32_t authLen; + QM_TRY(MOZ_TO_RESULT(urlParser->ParseURL(url, flatURL.Length(), &schemePos, + &schemeLen, &authPos, &authLen, + nullptr, nullptr)), // ignore path + false); + + const nsAutoCString scheme(Substring(flatURL, schemePos, schemeLen)); + if (scheme.LowerCaseEqualsLiteral("https") || + scheme.LowerCaseEqualsLiteral("file") || + scheme.LowerCaseEqualsLiteral("moz-extension")) { + return true; + } + + uint32_t hostPos; + int32_t hostLen; + QM_TRY(MOZ_TO_RESULT( + urlParser->ParseAuthority(url + authPos, authLen, nullptr, + nullptr, // ignore username + nullptr, nullptr, // ignore password + &hostPos, &hostLen, + nullptr)), // ignore port + false); + + return nsMixedContentBlocker::IsPotentiallyTrustworthyLoopbackHost( + nsDependentCSubstring(url + authPos + hostPos, hostLen)); +} + +} // namespace + +// static +already_AddRefed<CacheStorage> CacheStorage::CreateOnMainThread( + Namespace aNamespace, nsIGlobalObject* aGlobal, nsIPrincipal* aPrincipal, + bool aForceTrustedOrigin, ErrorResult& aRv) { + MOZ_DIAGNOSTIC_ASSERT(aGlobal); + MOZ_DIAGNOSTIC_ASSERT(aPrincipal); + MOZ_ASSERT(NS_IsMainThread()); + + PrincipalInfo principalInfo; + QM_TRY(MOZ_TO_RESULT(PrincipalToPrincipalInfo(aPrincipal, &principalInfo)), + nullptr, [&aRv](const nsresult rv) { aRv.Throw(rv); }); + + QM_TRY(OkIf(QuotaManager::IsPrincipalInfoValid(principalInfo)), + RefPtr{new CacheStorage(NS_ERROR_DOM_SECURITY_ERR)}.forget(), + [](const auto) { + NS_WARNING("CacheStorage not supported on invalid origins."); + }); + + const bool testingEnabled = + aForceTrustedOrigin || + Preferences::GetBool("dom.caches.testing.enabled", false) || + StaticPrefs::dom_serviceWorkers_testing_enabled(); + + if (!IsTrusted(principalInfo, testingEnabled)) { + NS_WARNING("CacheStorage not supported on untrusted origins."); + RefPtr<CacheStorage> ref = new CacheStorage(NS_ERROR_DOM_SECURITY_ERR); + return ref.forget(); + } + + RefPtr<CacheStorage> ref = + new CacheStorage(aNamespace, aGlobal, principalInfo, nullptr); + return ref.forget(); +} + +// static +already_AddRefed<CacheStorage> CacheStorage::CreateOnWorker( + Namespace aNamespace, nsIGlobalObject* aGlobal, + WorkerPrivate* aWorkerPrivate, ErrorResult& aRv) { + MOZ_DIAGNOSTIC_ASSERT(aGlobal); + MOZ_DIAGNOSTIC_ASSERT(aWorkerPrivate); + aWorkerPrivate->AssertIsOnWorkerThread(); + + if (aWorkerPrivate->GetOriginAttributes().mPrivateBrowsingId > 0) { + NS_WARNING("CacheStorage not supported during private browsing."); + RefPtr<CacheStorage> ref = new CacheStorage(NS_ERROR_DOM_SECURITY_ERR); + return ref.forget(); + } + + SafeRefPtr<CacheWorkerRef> workerRef = + CacheWorkerRef::Create(aWorkerPrivate, CacheWorkerRef::eIPCWorkerRef); + if (!workerRef) { + NS_WARNING("Worker thread is shutting down."); + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + + const PrincipalInfo& principalInfo = + aWorkerPrivate->GetEffectiveStoragePrincipalInfo(); + + QM_TRY(OkIf(QuotaManager::IsPrincipalInfoValid(principalInfo)), nullptr, + [&aRv](const auto) { aRv.Throw(NS_ERROR_FAILURE); }); + + // We have a number of cases where we want to skip the https scheme + // validation: + // + // 1) Any worker when dom.caches.testing.enabled pref is true. + // 2) Any worker when dom.serviceWorkers.testing.enabled pref is true. This + // is mainly because most sites using SWs will expect Cache to work if + // SWs are enabled. + // 3) If the window that created this worker has the devtools SW testing + // option enabled. Same reasoning as (2). + // 4) If the worker itself is a ServiceWorker, then we always skip the + // origin checks. The ServiceWorker has its own trusted origin checks + // that are better than ours. In addition, we don't have information + // about the window any more, so we can't do our own checks. + bool testingEnabled = StaticPrefs::dom_caches_testing_enabled() || + StaticPrefs::dom_serviceWorkers_testing_enabled() || + aWorkerPrivate->ServiceWorkersTestingInWindow() || + aWorkerPrivate->IsServiceWorker(); + + if (!IsTrusted(principalInfo, testingEnabled)) { + NS_WARNING("CacheStorage not supported on untrusted origins."); + RefPtr<CacheStorage> ref = new CacheStorage(NS_ERROR_DOM_SECURITY_ERR); + return ref.forget(); + } + + RefPtr<CacheStorage> ref = new CacheStorage( + aNamespace, aGlobal, principalInfo, std::move(workerRef)); + return ref.forget(); +} + +// static +bool CacheStorage::DefineCaches(JSContext* aCx, JS::Handle<JSObject*> aGlobal) { + MOZ_ASSERT(NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(JS::GetClass(aGlobal)->flags & JSCLASS_DOM_GLOBAL, + "Passed object is not a global object!"); + js::AssertSameCompartment(aCx, aGlobal); + + if (NS_WARN_IF(!CacheStorage_Binding::GetConstructorObject(aCx) || + !Cache_Binding::GetConstructorObject(aCx))) { + return false; + } + + nsIPrincipal* principal = nsContentUtils::ObjectPrincipal(aGlobal); + MOZ_DIAGNOSTIC_ASSERT(principal); + + ErrorResult rv; + RefPtr<CacheStorage> storage = + CreateOnMainThread(DEFAULT_NAMESPACE, xpc::NativeGlobal(aGlobal), + principal, true, /* force trusted */ + rv); + if (NS_WARN_IF(rv.MaybeSetPendingException(aCx))) { + return false; + } + + JS::Rooted<JS::Value> caches(aCx); + if (NS_WARN_IF(!ToJSValue(aCx, storage, &caches))) { + return false; + } + + return JS_DefineProperty(aCx, aGlobal, "caches", caches, JSPROP_ENUMERATE); +} + +CacheStorage::CacheStorage(Namespace aNamespace, nsIGlobalObject* aGlobal, + const PrincipalInfo& aPrincipalInfo, + SafeRefPtr<CacheWorkerRef> aWorkerRef) + : mNamespace(aNamespace), + mGlobal(aGlobal), + mPrincipalInfo(MakeUnique<PrincipalInfo>(aPrincipalInfo)), + mActor(nullptr), + mStatus(NS_OK) { + MOZ_DIAGNOSTIC_ASSERT(mGlobal); + + // If the PBackground actor is already initialized then we can + // immediately use it + PBackgroundChild* actor = BackgroundChild::GetOrCreateForCurrentThread(); + if (NS_WARN_IF(!actor)) { + mStatus = NS_ERROR_UNEXPECTED; + return; + } + + // WorkerRef ownership is passed to the CacheStorageChild actor and any + // actors it may create. The WorkerRef will keep the worker thread alive + // until the actors can gracefully shutdown. + CacheStorageChild* newActor = + new CacheStorageChild(this, std::move(aWorkerRef)); + PCacheStorageChild* constructedActor = actor->SendPCacheStorageConstructor( + newActor, mNamespace, *mPrincipalInfo); + + if (NS_WARN_IF(!constructedActor)) { + mStatus = NS_ERROR_UNEXPECTED; + return; + } + + MOZ_DIAGNOSTIC_ASSERT(constructedActor == newActor); + mActor = newActor; +} + +CacheStorage::CacheStorage(nsresult aFailureResult) + : mNamespace(INVALID_NAMESPACE), mActor(nullptr), mStatus(aFailureResult) { + MOZ_DIAGNOSTIC_ASSERT(NS_FAILED(mStatus)); +} + +already_AddRefed<Promise> CacheStorage::Match( + JSContext* aCx, const RequestOrUSVString& aRequest, + const MultiCacheQueryOptions& aOptions, ErrorResult& aRv) { + NS_ASSERT_OWNINGTHREAD(CacheStorage); + + if (!HasStorageAccess(eUseCounter_custom_PrivateBrowsingCachesMatch, + UseCounterWorker::Custom_PrivateBrowsingCachesMatch)) { + aRv.Throw(NS_ERROR_DOM_SECURITY_ERR); + return nullptr; + } + + if (NS_WARN_IF(NS_FAILED(mStatus))) { + aRv.Throw(mStatus); + return nullptr; + } + + SafeRefPtr<InternalRequest> request = + ToInternalRequest(aCx, aRequest, IgnoreBody, aRv); + if (NS_WARN_IF(aRv.Failed())) { + return nullptr; + } + + RefPtr<Promise> promise = Promise::Create(mGlobal, aRv); + if (NS_WARN_IF(!promise)) { + return nullptr; + } + + CacheQueryParams params; + ToCacheQueryParams(params, aOptions); + + auto entry = MakeUnique<Entry>(); + entry->mPromise = promise; + entry->mArgs = StorageMatchArgs(CacheRequest(), params, GetOpenMode()); + entry->mRequest = std::move(request); + + RunRequest(std::move(entry)); + + return promise.forget(); +} + +already_AddRefed<Promise> CacheStorage::Has(const nsAString& aKey, + ErrorResult& aRv) { + NS_ASSERT_OWNINGTHREAD(CacheStorage); + + if (!HasStorageAccess(eUseCounter_custom_PrivateBrowsingCachesHas, + UseCounterWorker::Custom_PrivateBrowsingCachesHas)) { + aRv.Throw(NS_ERROR_DOM_SECURITY_ERR); + return nullptr; + } + + if (NS_WARN_IF(NS_FAILED(mStatus))) { + aRv.Throw(mStatus); + return nullptr; + } + + RefPtr<Promise> promise = Promise::Create(mGlobal, aRv); + if (NS_WARN_IF(!promise)) { + return nullptr; + } + + auto entry = MakeUnique<Entry>(); + entry->mPromise = promise; + entry->mArgs = StorageHasArgs(nsString(aKey)); + + RunRequest(std::move(entry)); + + return promise.forget(); +} + +already_AddRefed<Promise> CacheStorage::Open(const nsAString& aKey, + ErrorResult& aRv) { + NS_ASSERT_OWNINGTHREAD(CacheStorage); + + if (!HasStorageAccess(eUseCounter_custom_PrivateBrowsingCachesOpen, + UseCounterWorker::Custom_PrivateBrowsingCachesOpen)) { + aRv.Throw(NS_ERROR_DOM_SECURITY_ERR); + return nullptr; + } + + if (NS_WARN_IF(NS_FAILED(mStatus))) { + aRv.Throw(mStatus); + return nullptr; + } + + RefPtr<Promise> promise = Promise::Create(mGlobal, aRv); + if (NS_WARN_IF(!promise)) { + return nullptr; + } + + auto entry = MakeUnique<Entry>(); + entry->mPromise = promise; + entry->mArgs = StorageOpenArgs(nsString(aKey)); + + RunRequest(std::move(entry)); + + return promise.forget(); +} + +already_AddRefed<Promise> CacheStorage::Delete(const nsAString& aKey, + ErrorResult& aRv) { + NS_ASSERT_OWNINGTHREAD(CacheStorage); + + if (!HasStorageAccess(eUseCounter_custom_PrivateBrowsingCachesDelete, + UseCounterWorker::Custom_PrivateBrowsingCachesDelete)) { + aRv.Throw(NS_ERROR_DOM_SECURITY_ERR); + return nullptr; + } + + if (NS_WARN_IF(NS_FAILED(mStatus))) { + aRv.Throw(mStatus); + return nullptr; + } + + RefPtr<Promise> promise = Promise::Create(mGlobal, aRv); + if (NS_WARN_IF(!promise)) { + return nullptr; + } + + auto entry = MakeUnique<Entry>(); + entry->mPromise = promise; + entry->mArgs = StorageDeleteArgs(nsString(aKey)); + + RunRequest(std::move(entry)); + + return promise.forget(); +} + +already_AddRefed<Promise> CacheStorage::Keys(ErrorResult& aRv) { + NS_ASSERT_OWNINGTHREAD(CacheStorage); + + if (!HasStorageAccess(eUseCounter_custom_PrivateBrowsingCachesKeys, + UseCounterWorker::Custom_PrivateBrowsingCachesKeys)) { + aRv.Throw(NS_ERROR_DOM_SECURITY_ERR); + return nullptr; + } + + if (NS_WARN_IF(NS_FAILED(mStatus))) { + aRv.Throw(mStatus); + return nullptr; + } + + RefPtr<Promise> promise = Promise::Create(mGlobal, aRv); + if (NS_WARN_IF(!promise)) { + return nullptr; + } + + auto entry = MakeUnique<Entry>(); + entry->mPromise = promise; + entry->mArgs = StorageKeysArgs(); + + RunRequest(std::move(entry)); + + return promise.forget(); +} + +// static +already_AddRefed<CacheStorage> CacheStorage::Constructor( + const GlobalObject& aGlobal, CacheStorageNamespace aNamespace, + nsIPrincipal* aPrincipal, ErrorResult& aRv) { + if (NS_WARN_IF(!NS_IsMainThread())) { + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + + // TODO: remove Namespace in favor of CacheStorageNamespace + static_assert(DEFAULT_NAMESPACE == (uint32_t)CacheStorageNamespace::Content, + "Default namespace should match webidl Content enum"); + static_assert( + CHROME_ONLY_NAMESPACE == (uint32_t)CacheStorageNamespace::Chrome, + "Chrome namespace should match webidl Chrome enum"); + static_assert(NUMBER_OF_NAMESPACES == CacheStorageNamespaceValues::Count, + "Number of namespace should match webidl count"); + + Namespace ns = static_cast<Namespace>(aNamespace); + nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports()); + + bool privateBrowsing = false; + if (nsCOMPtr<nsPIDOMWindowInner> window = do_QueryInterface(global)) { + RefPtr<Document> doc = window->GetExtantDoc(); + if (doc) { + nsCOMPtr<nsILoadContext> loadContext = doc->GetLoadContext(); + privateBrowsing = loadContext && loadContext->UsePrivateBrowsing(); + } + } + + if (privateBrowsing) { + RefPtr<CacheStorage> ref = new CacheStorage(NS_ERROR_DOM_SECURITY_ERR); + return ref.forget(); + } + + // Create a CacheStorage object bypassing the trusted origin checks + // since this is a chrome-only constructor. + return CreateOnMainThread(ns, global, aPrincipal, + true /* force trusted origin */, aRv); +} + +nsISupports* CacheStorage::GetParentObject() const { return mGlobal; } + +JSObject* CacheStorage::WrapObject(JSContext* aContext, + JS::Handle<JSObject*> aGivenProto) { + return mozilla::dom::CacheStorage_Binding::Wrap(aContext, this, aGivenProto); +} + +void CacheStorage::DestroyInternal(CacheStorageChild* aActor) { + NS_ASSERT_OWNINGTHREAD(CacheStorage); + MOZ_DIAGNOSTIC_ASSERT(mActor); + MOZ_DIAGNOSTIC_ASSERT(mActor == aActor); + MOZ_DIAGNOSTIC_ASSERT(!NS_FAILED(mStatus)); + mActor->ClearListener(); + mActor = nullptr; + mStatus = NS_ERROR_UNEXPECTED; + + // Note that we will never get an actor again in case another request is + // made before this object is destructed. +} + +nsIGlobalObject* CacheStorage::GetGlobalObject() const { return mGlobal; } + +#ifdef DEBUG +void CacheStorage::AssertOwningThread() const { + NS_ASSERT_OWNINGTHREAD(CacheStorage); +} +#endif + +PBackgroundChild* CacheStorage::GetIPCManager() { + // This is true because CacheStorage always uses IgnoreBody for requests. + // So we should never need to get the IPC manager during Request or + // Response serialization. + MOZ_CRASH("CacheStorage does not implement TypeUtils::GetIPCManager()"); +} + +CacheStorage::~CacheStorage() { + NS_ASSERT_OWNINGTHREAD(CacheStorage); + if (mActor) { + mActor->StartDestroyFromListener(); + // DestroyInternal() is called synchronously by StartDestroyFromListener(). + // So we should have already cleared the mActor. + MOZ_DIAGNOSTIC_ASSERT(!mActor); + } +} + +void CacheStorage::RunRequest(UniquePtr<Entry> aEntry) { + MOZ_ASSERT(mActor); + + AutoChildOpArgs args(this, aEntry->mArgs, 1); + + if (aEntry->mRequest) { + ErrorResult rv; + args.Add(*aEntry->mRequest, IgnoreBody, IgnoreInvalidScheme, rv); + if (NS_WARN_IF(rv.Failed())) { + aEntry->mPromise->MaybeReject(std::move(rv)); + return; + } + } + + mActor->ExecuteOp(mGlobal, aEntry->mPromise, this, args.SendAsOpArgs()); +} + +OpenMode CacheStorage::GetOpenMode() const { + return mNamespace == CHROME_ONLY_NAMESPACE ? OpenMode::Eager : OpenMode::Lazy; +} + +bool CacheStorage::HasStorageAccess(UseCounter aLabel, + UseCounterWorker aLabelWorker) const { + NS_ASSERT_OWNINGTHREAD(CacheStorage); + if (NS_WARN_IF(!mGlobal)) { + return false; + } + + StorageAccess access = mGlobal->GetStorageAccess(); + if (access == StorageAccess::ePrivateBrowsing) { + if (NS_IsMainThread()) { + SetUseCounter(mGlobal->GetGlobalJSObject(), aLabel); + } else { + SetUseCounter(aLabelWorker); + } + } + + // Deny storage access for private browsing. + if (nsIPrincipal* principal = mGlobal->PrincipalOrNull()) { + if (!principal->IsSystemPrincipal() && + principal->GetPrivateBrowsingId() != + nsIScriptSecurityManager::DEFAULT_PRIVATE_BROWSING_ID) { + return false; + } + } + + return access > StorageAccess::ePrivateBrowsing || + (StaticPrefs:: + privacy_partition_always_partition_third_party_non_cookie_storage() && + ShouldPartitionStorage(access)); +} + +} // namespace mozilla::dom::cache diff --git a/dom/cache/CacheStorage.h b/dom/cache/CacheStorage.h new file mode 100644 index 0000000000..306c7cccd4 --- /dev/null +++ b/dom/cache/CacheStorage.h @@ -0,0 +1,120 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_CacheStorage_h +#define mozilla_dom_cache_CacheStorage_h + +#include "mozilla/UniquePtr.h" +#include "mozilla/dom/cache/Types.h" +#include "mozilla/dom/cache/TypeUtils.h" +#include "nsCOMPtr.h" +#include "nsISupportsImpl.h" +#include "nsTArray.h" +#include "nsWrapperCache.h" + +class nsIGlobalObject; + +namespace mozilla { + +class ErrorResult; +enum UseCounter : int16_t; +enum class UseCounterWorker : int16_t; + +namespace ipc { +class PrincipalInfo; +} // namespace ipc + +namespace dom { + +enum class CacheStorageNamespace : uint8_t; +class Promise; +class WorkerPrivate; + +namespace cache { + +class CacheStorageChild; +class CacheWorkerRef; + +class CacheStorage final : public nsISupports, + public nsWrapperCache, + public TypeUtils { + using PBackgroundChild = mozilla::ipc::PBackgroundChild; + + public: + static already_AddRefed<CacheStorage> CreateOnMainThread( + Namespace aNamespace, nsIGlobalObject* aGlobal, nsIPrincipal* aPrincipal, + bool aForceTrustedOrigin, ErrorResult& aRv); + + static already_AddRefed<CacheStorage> CreateOnWorker( + Namespace aNamespace, nsIGlobalObject* aGlobal, + WorkerPrivate* aWorkerPrivate, ErrorResult& aRv); + + static bool DefineCaches(JSContext* aCx, JS::Handle<JSObject*> aGlobal); + + // webidl interface methods + already_AddRefed<Promise> Match(JSContext* aCx, + const RequestOrUSVString& aRequest, + const MultiCacheQueryOptions& aOptions, + ErrorResult& aRv); + already_AddRefed<Promise> Has(const nsAString& aKey, ErrorResult& aRv); + already_AddRefed<Promise> Open(const nsAString& aKey, ErrorResult& aRv); + already_AddRefed<Promise> Delete(const nsAString& aKey, ErrorResult& aRv); + already_AddRefed<Promise> Keys(ErrorResult& aRv); + + // chrome-only webidl interface methods + static already_AddRefed<CacheStorage> Constructor( + const GlobalObject& aGlobal, CacheStorageNamespace aNamespace, + nsIPrincipal* aPrincipal, ErrorResult& aRv); + + // binding methods + nsISupports* GetParentObject() const; + virtual JSObject* WrapObject(JSContext* aContext, + JS::Handle<JSObject*> aGivenProto) override; + + // Called when CacheStorageChild actor is being destroyed + void DestroyInternal(CacheStorageChild* aActor); + + // TypeUtils methods + virtual nsIGlobalObject* GetGlobalObject() const override; +#ifdef DEBUG + virtual void AssertOwningThread() const override; +#endif + + virtual mozilla::ipc::PBackgroundChild* GetIPCManager() override; + + private: + CacheStorage(Namespace aNamespace, nsIGlobalObject* aGlobal, + const mozilla::ipc::PrincipalInfo& aPrincipalInfo, + SafeRefPtr<CacheWorkerRef> aWorkerRef); + explicit CacheStorage(nsresult aFailureResult); + ~CacheStorage(); + + struct Entry; + void RunRequest(UniquePtr<Entry> aEntry); + + OpenMode GetOpenMode() const; + + bool HasStorageAccess(UseCounter aLabel, UseCounterWorker aLabelWorker) const; + + const Namespace mNamespace; + nsCOMPtr<nsIGlobalObject> mGlobal; + const UniquePtr<mozilla::ipc::PrincipalInfo> mPrincipalInfo; + + // weak ref cleared in DestroyInternal + CacheStorageChild* mActor; + + nsresult mStatus; + + public: + NS_DECL_CYCLE_COLLECTING_ISUPPORTS + NS_DECL_CYCLE_COLLECTION_WRAPPERCACHE_CLASS(CacheStorage) +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_cache_CacheStorage_h diff --git a/dom/cache/CacheStorageChild.cpp b/dom/cache/CacheStorageChild.cpp new file mode 100644 index 0000000000..e7918bbf88 --- /dev/null +++ b/dom/cache/CacheStorageChild.cpp @@ -0,0 +1,111 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/CacheStorageChild.h" + +#include "mozilla/Unused.h" +#include "mozilla/dom/cache/CacheChild.h" +#include "mozilla/dom/cache/CacheOpChild.h" +#include "mozilla/dom/cache/CacheStorage.h" +#include "mozilla/dom/cache/CacheWorkerRef.h" + +namespace mozilla::dom::cache { + +// declared in ActorUtils.h +void DeallocPCacheStorageChild(PCacheStorageChild* aActor) { delete aActor; } + +CacheStorageChild::CacheStorageChild(CacheStorage* aListener, + SafeRefPtr<CacheWorkerRef> aWorkerRef) + : mListener(aListener), mDelayedDestroy(false) { + MOZ_COUNT_CTOR(cache::CacheStorageChild); + MOZ_DIAGNOSTIC_ASSERT(mListener); + + SetWorkerRef(std::move(aWorkerRef)); +} + +CacheStorageChild::~CacheStorageChild() { + MOZ_COUNT_DTOR(cache::CacheStorageChild); + NS_ASSERT_OWNINGTHREAD(CacheStorageChild); + MOZ_DIAGNOSTIC_ASSERT(!mListener); +} + +void CacheStorageChild::ClearListener() { + NS_ASSERT_OWNINGTHREAD(CacheStorageChild); + MOZ_DIAGNOSTIC_ASSERT(mListener); + mListener = nullptr; +} + +void CacheStorageChild::ExecuteOp(nsIGlobalObject* aGlobal, Promise* aPromise, + nsISupports* aParent, + const CacheOpArgs& aArgs) { + Unused << SendPCacheOpConstructor( + new CacheOpChild(GetWorkerRefPtr().clonePtr(), aGlobal, aParent, aPromise, + this), + aArgs); +} + +void CacheStorageChild::StartDestroyFromListener() { + NS_ASSERT_OWNINGTHREAD(CacheStorageChild); + + StartDestroy(); +} + +void CacheStorageChild::DestroyInternal() { + RefPtr<CacheStorage> listener = mListener; + + // StartDestroy() can get called from either CacheStorage or the + // CacheWorkerRef. + // Theoretically we can get double called if the right race happens. Handle + // that by just ignoring the second StartDestroy() call. + if (!listener) { + return; + } + + listener->DestroyInternal(this); + + // CacheStorage listener should call ClearListener() in DestroyInternal() + MOZ_DIAGNOSTIC_ASSERT(!mListener); + + // Start actor destruction from parent process + QM_WARNONLY_TRY(OkIf(SendTeardown())); +} + +void CacheStorageChild::StartDestroy() { + NS_ASSERT_OWNINGTHREAD(CacheStorageChild); + + if (NumChildActors() != 0) { + mDelayedDestroy = true; + return; + } + DestroyInternal(); +} + +void CacheStorageChild::NoteDeletedActor() { + // Check to see if DestroyInternal was delayed because of active CacheOpChilds + // when StartDestroy was called from WorkerRef notification. If the last + // CacheOpChild is getting destructed; it's the time for us to SendTearDown to + // the other side. + if (NumChildActors() == 1 && mDelayedDestroy) DestroyInternal(); +} + +void CacheStorageChild::ActorDestroy(ActorDestroyReason aReason) { + NS_ASSERT_OWNINGTHREAD(CacheStorageChild); + RefPtr<CacheStorage> listener = mListener; + if (listener) { + listener->DestroyInternal(this); + // CacheStorage listener should call ClearListener() in DestroyInternal() + MOZ_DIAGNOSTIC_ASSERT(!mListener); + } + + RemoveWorkerRef(); +} + +PCacheOpChild* CacheStorageChild::AllocPCacheOpChild( + const CacheOpArgs& aOpArgs) { + MOZ_CRASH("CacheOpChild should be manually constructed."); + return nullptr; +} +} // namespace mozilla::dom::cache diff --git a/dom/cache/CacheStorageChild.h b/dom/cache/CacheStorageChild.h new file mode 100644 index 0000000000..3e653b7812 --- /dev/null +++ b/dom/cache/CacheStorageChild.h @@ -0,0 +1,78 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_CacheStorageChild_h +#define mozilla_dom_cache_CacheStorageChild_h + +#include "mozilla/dom/cache/ActorChild.h" +#include "mozilla/dom/cache/Types.h" +#include "mozilla/dom/cache/PCacheStorageChild.h" + +class nsIGlobalObject; + +namespace mozilla::dom { + +class Promise; + +namespace cache { + +class CacheOpArgs; +class CacheStorage; +class CacheWorkerRef; +class PCacheChild; + +class CacheStorageChild final : public PCacheStorageChild, public ActorChild { + friend class PCacheStorageChild; + + public: + CacheStorageChild(CacheStorage* aListener, + SafeRefPtr<CacheWorkerRef> aWorkerRef); + + // Must be called by the associated CacheStorage listener in its + // DestroyInternal() method. Also, CacheStorage must call + // SendDestroyFromListener() on the actor in its destructor to trigger + // ActorDestroy() if it has not been called yet. + void ClearListener(); + + void ExecuteOp(nsIGlobalObject* aGlobal, Promise* aPromise, + nsISupports* aParent, const CacheOpArgs& aArgs); + + // Our parent Listener object has gone out of scope and is being destroyed. + void StartDestroyFromListener(); + + NS_INLINE_DECL_REFCOUNTING(CacheStorageChild, override) + void NoteDeletedActor() override; + + private: + ~CacheStorageChild(); + + void DestroyInternal(); + + // ActorChild methods + // CacheWorkerRef is trying to destroy due to worker shutdown. + virtual void StartDestroy() override; + + // PCacheStorageChild methods + virtual void ActorDestroy(ActorDestroyReason aReason) override; + + PCacheOpChild* AllocPCacheOpChild(const CacheOpArgs& aOpArgs); + + bool DeallocPCacheOpChild(PCacheOpChild* aActor); + + // utility methods + inline uint32_t NumChildActors() { return ManagedPCacheOpChild().Count(); } + + // Use a weak ref so actor does not hold DOM object alive past content use. + // The CacheStorage object must call ClearListener() to null this before its + // destroyed. + CacheStorage* MOZ_NON_OWNING_REF mListener; + bool mDelayedDestroy; +}; + +} // namespace cache +} // namespace mozilla::dom + +#endif // mozilla_dom_cache_CacheStorageChild_h diff --git a/dom/cache/CacheStorageParent.cpp b/dom/cache/CacheStorageParent.cpp new file mode 100644 index 0000000000..5a7c5774cb --- /dev/null +++ b/dom/cache/CacheStorageParent.cpp @@ -0,0 +1,125 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/CacheStorageParent.h" + +#include "mozilla/Unused.h" +#include "mozilla/dom/cache/ActorUtils.h" +#include "mozilla/dom/cache/CacheOpParent.h" +#include "mozilla/dom/cache/ManagerId.h" +#include "mozilla/dom/quota/QuotaManager.h" +#include "mozilla/ipc/PBackgroundParent.h" + +namespace mozilla::dom::cache { + +using mozilla::dom::quota::QuotaManager; +using mozilla::ipc::PBackgroundParent; +using mozilla::ipc::PrincipalInfo; + +// declared in ActorUtils.h +already_AddRefed<PCacheStorageParent> AllocPCacheStorageParent( + PBackgroundParent* aManagingActor, Namespace aNamespace, + const mozilla::ipc::PrincipalInfo& aPrincipalInfo) { + if (NS_WARN_IF(!QuotaManager::IsPrincipalInfoValid(aPrincipalInfo))) { + MOZ_ASSERT(false); + return nullptr; + } + + return MakeAndAddRef<CacheStorageParent>(aManagingActor, aNamespace, + aPrincipalInfo); +} + +// declared in ActorUtils.h +void DeallocPCacheStorageParent(PCacheStorageParent* aActor) { delete aActor; } + +CacheStorageParent::CacheStorageParent(PBackgroundParent* aManagingActor, + Namespace aNamespace, + const PrincipalInfo& aPrincipalInfo) + : mNamespace(aNamespace), mVerifiedStatus(NS_OK) { + MOZ_COUNT_CTOR(cache::CacheStorageParent); + MOZ_DIAGNOSTIC_ASSERT(aManagingActor); + + // Start the async principal verification process immediately. + mVerifier = PrincipalVerifier::CreateAndDispatch(*this, aManagingActor, + aPrincipalInfo); + MOZ_DIAGNOSTIC_ASSERT(mVerifier); +} + +CacheStorageParent::~CacheStorageParent() { + MOZ_COUNT_DTOR(cache::CacheStorageParent); + MOZ_DIAGNOSTIC_ASSERT(!mVerifier); +} + +void CacheStorageParent::ActorDestroy(ActorDestroyReason aReason) { + if (mVerifier) { + mVerifier->RemoveListener(*this); + mVerifier = nullptr; + } +} + +PCacheOpParent* CacheStorageParent::AllocPCacheOpParent( + const CacheOpArgs& aOpArgs) { + if (aOpArgs.type() != CacheOpArgs::TStorageMatchArgs && + aOpArgs.type() != CacheOpArgs::TStorageHasArgs && + aOpArgs.type() != CacheOpArgs::TStorageOpenArgs && + aOpArgs.type() != CacheOpArgs::TStorageDeleteArgs && + aOpArgs.type() != CacheOpArgs::TStorageKeysArgs) { + MOZ_CRASH("Invalid operation sent to CacheStorage actor!"); + } + + return new CacheOpParent(Manager(), mNamespace, aOpArgs); +} + +bool CacheStorageParent::DeallocPCacheOpParent(PCacheOpParent* aActor) { + delete aActor; + return true; +} + +mozilla::ipc::IPCResult CacheStorageParent::RecvPCacheOpConstructor( + PCacheOpParent* aActor, const CacheOpArgs& aOpArgs) { + auto actor = static_cast<CacheOpParent*>(aActor); + + if (mVerifier) { + MOZ_DIAGNOSTIC_ASSERT(!mManagerId); + actor->WaitForVerification(mVerifier); + return IPC_OK(); + } + + if (NS_WARN_IF(NS_FAILED(mVerifiedStatus))) { + ErrorResult result(mVerifiedStatus); + + QM_WARNONLY_TRY(OkIf( + CacheOpParent::Send__delete__(actor, std::move(result), void_t()))); + return IPC_OK(); + } + + MOZ_DIAGNOSTIC_ASSERT(mManagerId); + actor->Execute(mManagerId); + return IPC_OK(); +} + +mozilla::ipc::IPCResult CacheStorageParent::RecvTeardown() { + // If child process is gone, warn and allow actor to clean up normally + QM_WARNONLY_TRY(OkIf(Send__delete__(this))); + return IPC_OK(); +} + +void CacheStorageParent::OnPrincipalVerified( + nsresult aRv, const SafeRefPtr<ManagerId>& aManagerId) { + MOZ_DIAGNOSTIC_ASSERT(mVerifier); + MOZ_DIAGNOSTIC_ASSERT(!mManagerId); + MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(mVerifiedStatus)); + + if (NS_WARN_IF(NS_FAILED(aRv))) { + mVerifiedStatus = aRv; + } + + mManagerId = aManagerId.clonePtr(); + mVerifier->RemoveListener(*this); + mVerifier = nullptr; +} + +} // namespace mozilla::dom::cache diff --git a/dom/cache/CacheStorageParent.h b/dom/cache/CacheStorageParent.h new file mode 100644 index 0000000000..40d069f223 --- /dev/null +++ b/dom/cache/CacheStorageParent.h @@ -0,0 +1,55 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_CacheStorageParent_h +#define mozilla_dom_cache_CacheStorageParent_h + +#include "mozilla/dom/cache/PCacheStorageParent.h" +#include "mozilla/dom/cache/PrincipalVerifier.h" +#include "mozilla/dom/cache/Types.h" + +namespace mozilla::dom::cache { + +class ManagerId; + +class CacheStorageParent final : public PCacheStorageParent, + public PrincipalVerifier::Listener { + friend class PCacheStorageParent; + + public: + CacheStorageParent(PBackgroundParent* aManagingActor, Namespace aNamespace, + const mozilla::ipc::PrincipalInfo& aPrincipalInfo); + + NS_INLINE_DECL_REFCOUNTING(CacheStorageParent, override) + + private: + virtual ~CacheStorageParent(); + + // PCacheStorageParent methods + virtual void ActorDestroy(ActorDestroyReason aReason) override; + + PCacheOpParent* AllocPCacheOpParent(const CacheOpArgs& aOpArgs); + + bool DeallocPCacheOpParent(PCacheOpParent* aActor); + + virtual mozilla::ipc::IPCResult RecvPCacheOpConstructor( + PCacheOpParent* actor, const CacheOpArgs& aOpArgs) override; + + mozilla::ipc::IPCResult RecvTeardown(); + + // PrincipalVerifier::Listener methods + virtual void OnPrincipalVerified( + nsresult aRv, const SafeRefPtr<ManagerId>& aManagerId) override; + + const Namespace mNamespace; + RefPtr<PrincipalVerifier> mVerifier; + nsresult mVerifiedStatus; + SafeRefPtr<ManagerId> mManagerId; +}; + +} // namespace mozilla::dom::cache + +#endif // mozilla_dom_cache_CacheStorageParent_h diff --git a/dom/cache/CacheStreamControlChild.cpp b/dom/cache/CacheStreamControlChild.cpp new file mode 100644 index 0000000000..26b11e7f46 --- /dev/null +++ b/dom/cache/CacheStreamControlChild.cpp @@ -0,0 +1,142 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "CacheStreamControlChild.h" + +#include "mozilla/Unused.h" +#include "mozilla/dom/cache/ActorUtils.h" +#include "mozilla/dom/cache/CacheTypes.h" +#include "mozilla/dom/cache/CacheWorkerRef.h" +#include "mozilla/ipc/IPCStreamUtils.h" +#include "mozilla/ipc/PBackgroundChild.h" +#include "nsISupportsImpl.h" + +namespace mozilla::dom::cache { + +using mozilla::ipc::FileDescriptor; + +// declared in ActorUtils.h +already_AddRefed<PCacheStreamControlChild> AllocPCacheStreamControlChild() { + return MakeAndAddRef<CacheStreamControlChild>(); +} + +CacheStreamControlChild::CacheStreamControlChild() + : mDestroyStarted(false), mDestroyDelayed(false) { + MOZ_COUNT_CTOR(cache::CacheStreamControlChild); +} + +CacheStreamControlChild::~CacheStreamControlChild() { + NS_ASSERT_OWNINGTHREAD(CacheStreamControlChild); + MOZ_COUNT_DTOR(cache::CacheStreamControlChild); +} + +void CacheStreamControlChild::StartDestroy() { + NS_ASSERT_OWNINGTHREAD(CacheStreamControlChild); + // This can get called twice under some circumstances. For example, if the + // actor is added to a CacheWorkerRef that has already been notified and + // the Cache actor has no mListener. + if (mDestroyStarted) { + return; + } + mDestroyStarted = true; + + // If any of the streams have started to be read, then wait for them to close + // naturally. + if (HasEverBeenRead()) { + // Note that we are delaying so that we can re-check for active streams + // in NoteClosedAfterForget(). + mDestroyDelayed = true; + return; + } + + // Otherwise, if the streams have not been touched then just pre-emptively + // close them now. This handles the case where someone retrieves a Response + // from the Cache, but never accesses the body. We should not keep the + // Worker alive until that Response is GC'd just because of its ignored + // body stream. + + // Begin shutting down all streams. This is the same as if the parent had + // asked us to shutdown. So simulate the CloseAll IPC message. + RecvCloseAll(); +} + +void CacheStreamControlChild::SerializeControl( + CacheReadStream* aReadStreamOut) { + NS_ASSERT_OWNINGTHREAD(CacheStreamControlChild); + MOZ_DIAGNOSTIC_ASSERT(aReadStreamOut); + aReadStreamOut->control() = this; +} + +void CacheStreamControlChild::SerializeStream(CacheReadStream* aReadStreamOut, + nsIInputStream* aStream) { + NS_ASSERT_OWNINGTHREAD(CacheStreamControlChild); + MOZ_DIAGNOSTIC_ASSERT(aReadStreamOut); + MOZ_ALWAYS_TRUE(mozilla::ipc::SerializeIPCStream( + do_AddRef(aStream), aReadStreamOut->stream(), /* aAllowLazy */ false)); +} + +void CacheStreamControlChild::OpenStream(const nsID& aId, + InputStreamResolver&& aResolver) { + NS_ASSERT_OWNINGTHREAD(CacheStreamControlChild); + + if (mDestroyStarted) { + aResolver(nullptr); + return; + } + + // If we are on a worker, then we need to hold it alive until the async + // IPC operation below completes. While the IPC layer will trigger a + // rejection here in many cases, we must handle the case where the + // MozPromise resolve runnable is already in the event queue when the + // worker wants to shut down. + const SafeRefPtr<CacheWorkerRef> holder = GetWorkerRefPtr().clonePtr(); + + SendOpenStream(aId)->Then( + GetCurrentSerialEventTarget(), __func__, + [aResolver, + holder = holder.clonePtr()](const Maybe<IPCStream>& aOptionalStream) { + nsCOMPtr<nsIInputStream> stream = DeserializeIPCStream(aOptionalStream); + aResolver(std::move(stream)); + }, + [aResolver, holder = holder.clonePtr()](ResponseRejectReason&& aReason) { + aResolver(nullptr); + }); +} + +void CacheStreamControlChild::NoteClosedAfterForget(const nsID& aId) { + NS_ASSERT_OWNINGTHREAD(CacheStreamControlChild); + + QM_WARNONLY_TRY(OkIf(SendNoteClosed(aId))); + + // A stream has closed. If we delayed StartDestry() due to this stream + // being read, then we should check to see if any of the remaining streams + // are active. If none of our other streams have been read, then we can + // proceed with the shutdown now. + if (mDestroyDelayed && !HasEverBeenRead()) { + mDestroyDelayed = false; + RecvCloseAll(); + } +} + +#ifdef DEBUG +void CacheStreamControlChild::AssertOwningThread() { + NS_ASSERT_OWNINGTHREAD(CacheStreamControlChild); +} +#endif + +void CacheStreamControlChild::ActorDestroy(ActorDestroyReason aReason) { + NS_ASSERT_OWNINGTHREAD(CacheStreamControlChild); + CloseAllReadStreamsWithoutReporting(); + RemoveWorkerRef(); +} + +mozilla::ipc::IPCResult CacheStreamControlChild::RecvCloseAll() { + NS_ASSERT_OWNINGTHREAD(CacheStreamControlChild); + CloseAllReadStreams(); + return IPC_OK(); +} + +} // namespace mozilla::dom::cache diff --git a/dom/cache/CacheStreamControlChild.h b/dom/cache/CacheStreamControlChild.h new file mode 100644 index 0000000000..fa78b8caf2 --- /dev/null +++ b/dom/cache/CacheStreamControlChild.h @@ -0,0 +1,60 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_CacheStreamControlChild_h +#define mozilla_dom_cache_CacheStreamControlChild_h + +#include "mozilla/dom/cache/ActorChild.h" +#include "mozilla/dom/cache/PCacheStreamControlChild.h" +#include "mozilla/dom/cache/StreamControl.h" +#include "nsTObserverArray.h" + +namespace mozilla::dom::cache { + +class ReadStream; + +class CacheStreamControlChild final : public PCacheStreamControlChild, + public StreamControl, + public ActorChild { + friend class PCacheStreamControlChild; + + public: + CacheStreamControlChild(); + + // ActorChild methods + virtual void StartDestroy() override; + + // StreamControl methods + virtual void SerializeControl(CacheReadStream* aReadStreamOut) override; + + virtual void SerializeStream(CacheReadStream* aReadStreamOut, + nsIInputStream* aStream) override; + + virtual void OpenStream(const nsID& aId, + InputStreamResolver&& aResolver) override; + + NS_DECL_OWNINGTHREAD + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(CacheStreamControlChild, override) + private: + ~CacheStreamControlChild(); + virtual void NoteClosedAfterForget(const nsID& aId) override; + +#ifdef DEBUG + virtual void AssertOwningThread() override; +#endif + + // PCacheStreamControlChild methods + virtual void ActorDestroy(ActorDestroyReason aReason) override; + mozilla::ipc::IPCResult RecvClose(const nsID& aId); + mozilla::ipc::IPCResult RecvCloseAll(); + + bool mDestroyStarted; + bool mDestroyDelayed; +}; + +} // namespace mozilla::dom::cache + +#endif // mozilla_dom_cache_CacheStreamControlChild_h diff --git a/dom/cache/CacheStreamControlParent.cpp b/dom/cache/CacheStreamControlParent.cpp new file mode 100644 index 0000000000..77e70c566b --- /dev/null +++ b/dom/cache/CacheStreamControlParent.cpp @@ -0,0 +1,165 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/CacheStreamControlParent.h" + +#include "mozilla/UniquePtr.h" +#include "mozilla/Unused.h" +#include "mozilla/dom/cache/CacheTypes.h" +#include "mozilla/dom/cache/ReadStream.h" +#include "mozilla/dom/cache/StreamList.h" +#include "mozilla/ipc/IPCStreamUtils.h" +#include "mozilla/ipc/PBackgroundParent.h" +#include "nsISupportsImpl.h" +#include "nsTArray.h" + +namespace mozilla::dom::cache { + +using mozilla::ipc::FileDescriptor; + +// declared in ActorUtils.h +void DeallocPCacheStreamControlParent(PCacheStreamControlParent* aActor) { + delete aActor; +} + +CacheStreamControlParent::CacheStreamControlParent() { + MOZ_COUNT_CTOR(cache::CacheStreamControlParent); +} + +CacheStreamControlParent::~CacheStreamControlParent() { + NS_ASSERT_OWNINGTHREAD(CacheStreamControlParent); + MOZ_DIAGNOSTIC_ASSERT(!mStreamList); + MOZ_COUNT_DTOR(cache::CacheStreamControlParent); +} + +void CacheStreamControlParent::SerializeControl( + CacheReadStream* aReadStreamOut) { + NS_ASSERT_OWNINGTHREAD(CacheStreamControlParent); + MOZ_DIAGNOSTIC_ASSERT(aReadStreamOut); + aReadStreamOut->control() = this; +} + +void CacheStreamControlParent::SerializeStream(CacheReadStream* aReadStreamOut, + nsIInputStream* aStream) { + NS_ASSERT_OWNINGTHREAD(CacheStreamControlParent); + MOZ_DIAGNOSTIC_ASSERT(aReadStreamOut); + + DebugOnly<bool> ok = mozilla::ipc::SerializeIPCStream( + do_AddRef(aStream), aReadStreamOut->stream(), /* aAllowLazy */ false); + MOZ_ASSERT(ok); +} + +void CacheStreamControlParent::OpenStream(const nsID& aId, + InputStreamResolver&& aResolver) { + NS_ASSERT_OWNINGTHREAD(CacheStreamControlParent); + MOZ_DIAGNOSTIC_ASSERT(aResolver); + + if (!mStreamList || !mStreamList->ShouldOpenStreamFor(aId)) { + aResolver(nullptr); + return; + } + + // Make sure to add ourself as a Listener even thought we are using + // a separate resolver function to signal the completion of the + // operation. The Manager uses the existence of the Listener to ensure + // that its safe to complete the operation. + mStreamList->GetManager().ExecuteOpenStream(this, std::move(aResolver), aId); +} + +void CacheStreamControlParent::NoteClosedAfterForget(const nsID& aId) { + NS_ASSERT_OWNINGTHREAD(CacheStreamControlParent); + RecvNoteClosed(aId); +} + +#ifdef DEBUG +void CacheStreamControlParent::AssertOwningThread() { + NS_ASSERT_OWNINGTHREAD(CacheStreamControlParent); +} +#endif + +void CacheStreamControlParent::ActorDestroy(ActorDestroyReason aReason) { + NS_ASSERT_OWNINGTHREAD(CacheStreamControlParent); + CloseAllReadStreamsWithoutReporting(); + // If the initial SendPStreamControlConstructor() fails we will + // be called before mStreamList is set. + if (!mStreamList) { + return; + } + mStreamList->GetManager().RemoveListener(this); +#ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED + mStreamList->GetManager().RecordHaveDeletedCSCP(Id()); +#endif + mStreamList->RemoveStreamControl(this); + mStreamList->NoteClosedAll(); + mStreamList = nullptr; +} + +void CacheStreamControlParent::AssertWillDelete() { +#ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED + // If we cannot send, Send__delete__ will not do any cleanup. + // But if we are still unfreed, that might be wrong. + if (mStreamList && !CanSend()) { + MOZ_ASSERT(false, "Attempt to delete blocking CSCP that cannot send."); + mStreamList->GetManager().RecordMayNotDeleteCSCP(Id()); + } +#endif +} + +mozilla::ipc::IPCResult CacheStreamControlParent::RecvOpenStream( + const nsID& aStreamId, OpenStreamResolver&& aResolver) { + NS_ASSERT_OWNINGTHREAD(CacheStreamControlParent); + + OpenStream(aStreamId, [aResolver, self = RefPtr{this}]( + nsCOMPtr<nsIInputStream>&& aStream) { + Maybe<IPCStream> stream; + if (self->CanSend() && + mozilla::ipc::SerializeIPCStream(aStream.forget(), stream, + /* aAllowLazy */ false)) { + aResolver(stream); + } else { + aResolver(Nothing()); + } + }); + + return IPC_OK(); +} + +mozilla::ipc::IPCResult CacheStreamControlParent::RecvNoteClosed( + const nsID& aId) { + NS_ASSERT_OWNINGTHREAD(CacheStreamControlParent); + MOZ_DIAGNOSTIC_ASSERT(mStreamList); + mStreamList->NoteClosed(aId); + return IPC_OK(); +} + +void CacheStreamControlParent::SetStreamList( + SafeRefPtr<StreamList> aStreamList) { + NS_ASSERT_OWNINGTHREAD(CacheStreamControlParent); + MOZ_DIAGNOSTIC_ASSERT(!mStreamList); + mStreamList = std::move(aStreamList); +} + +void CacheStreamControlParent::CloseAll() { + NS_ASSERT_OWNINGTHREAD(CacheStreamControlParent); + NotifyCloseAll(); + + QM_WARNONLY_TRY(OkIf(SendCloseAll())); +} + +void CacheStreamControlParent::Shutdown() { + NS_ASSERT_OWNINGTHREAD(CacheStreamControlParent); + + AssertWillDelete(); + // If child process is gone, warn and allow actor to clean up normally + QM_WARNONLY_TRY(OkIf(Send__delete__(this))); +} + +void CacheStreamControlParent::NotifyCloseAll() { + NS_ASSERT_OWNINGTHREAD(CacheStreamControlParent); + CloseAllReadStreams(); +} + +} // namespace mozilla::dom::cache diff --git a/dom/cache/CacheStreamControlParent.h b/dom/cache/CacheStreamControlParent.h new file mode 100644 index 0000000000..aa883b32f2 --- /dev/null +++ b/dom/cache/CacheStreamControlParent.h @@ -0,0 +1,71 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_CacheStreamControlParent_h +#define mozilla_dom_cache_CacheStreamControlParent_h + +#include "mozilla/dom/cache/Manager.h" +#include "mozilla/dom/cache/PCacheStreamControlParent.h" +#include "mozilla/dom/cache/StreamControl.h" +#include "nsTObserverArray.h" + +namespace mozilla::dom::cache { + +class StreamList; + +class CacheStreamControlParent final : public PCacheStreamControlParent, + public StreamControl, + Manager::Listener { + friend class PCacheStreamControlParent; + + public: + CacheStreamControlParent(); + + void SetStreamList(SafeRefPtr<StreamList> aStreamList); + void CloseAll(); + void Shutdown(); + + // StreamControl methods + virtual void SerializeControl(CacheReadStream* aReadStreamOut) override; + + virtual void SerializeStream(CacheReadStream* aReadStreamOut, + nsIInputStream* aStream) override; + + virtual void OpenStream(const nsID& aId, + InputStreamResolver&& aResolver) override; + + void AssertWillDelete(); + + private: + ~CacheStreamControlParent(); + virtual void NoteClosedAfterForget(const nsID& aId) override; + +#ifdef DEBUG + virtual void AssertOwningThread() override; +#endif + + // PCacheStreamControlParent methods + virtual void ActorDestroy(ActorDestroyReason aReason) override; + + mozilla::ipc::IPCResult RecvOpenStream(const nsID& aStreamId, + OpenStreamResolver&& aResolve); + + mozilla::ipc::IPCResult RecvNoteClosed(const nsID& aId); + + void NotifyCloseAll(); + + // Cycle with StreamList via a weak-ref to us. Cleanup occurs when the actor + // is deleted by the PBackground manager. ActorDestroy() then calls + // StreamList::RemoveStreamControl() to clear the weak ref. + SafeRefPtr<StreamList> mStreamList; + + NS_DECL_OWNINGTHREAD + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(CacheStreamControlParent, override) +}; + +} // namespace mozilla::dom::cache + +#endif // mozilla_dom_cache_CacheStreamControlParent_h diff --git a/dom/cache/CacheTypes.ipdlh b/dom/cache/CacheTypes.ipdlh new file mode 100644 index 0000000000..dee420e211 --- /dev/null +++ b/dom/cache/CacheTypes.ipdlh @@ -0,0 +1,236 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +include protocol PCache; +include protocol PCacheStreamControl; +include IPCStream; +include PBackgroundSharedTypes; + +include "mozilla/dom/cache/IPCUtils.h"; +include "mozilla/dom/FetchIPCTypes.h"; +include "mozilla/ipc/TransportSecurityInfoUtils.h"; + +using mozilla::dom::HeadersGuardEnum from "mozilla/dom/HeadersBinding.h"; +using mozilla::dom::cache::Namespace from "mozilla/dom/cache/Types.h"; +using mozilla::dom::cache::OpenMode from "mozilla/dom/cache/Types.h"; +using mozilla::dom::ReferrerPolicy from "mozilla/dom/ReferrerPolicyBinding.h"; +using mozilla::dom::RequestCredentials from "mozilla/dom/RequestBinding.h"; +using mozilla::dom::RequestMode from "mozilla/dom/RequestBinding.h"; +using mozilla::dom::RequestCache from "mozilla/dom/RequestBinding.h"; +using mozilla::dom::RequestRedirect from "mozilla/dom/RequestBinding.h"; +using mozilla::dom::ResponseType from "mozilla/dom/ResponseBinding.h"; +using mozilla::void_t from "mozilla/ipc/IPCCore.h"; +using nsContentPolicyType from "nsIContentPolicy.h"; +using struct nsID from "nsID.h"; +using nsILoadInfo::CrossOriginEmbedderPolicy from "nsILoadInfo.h"; +[RefCounted] using class nsITransportSecurityInfo from "nsITransportSecurityInfo.h"; + +namespace mozilla { +namespace dom { +namespace cache { + +struct CacheQueryParams +{ + bool ignoreSearch; + bool ignoreMethod; + bool ignoreVary; + bool cacheNameSet; + nsString cacheName; +}; + +struct CacheReadStream +{ + nsID id; + nullable PCacheStreamControl control; + IPCStream? stream; +}; + +struct HeadersEntry +{ + nsCString name; + nsCString value; +}; +struct CacheRequest +{ + nsCString method; + nsCString urlWithoutQuery; + nsCString urlQuery; + nsCString urlFragment; + HeadersEntry[] headers; + HeadersGuardEnum headersGuard; + nsString referrer; + ReferrerPolicy referrerPolicy; + RequestMode mode; + RequestCredentials credentials; + CacheReadStream? body; + nsContentPolicyType contentPolicyType; + RequestCache requestCache; + RequestRedirect requestRedirect; + nsString integrity; + CrossOriginEmbedderPolicy loadingEmbedderPolicy; + PrincipalInfo? principalInfo; +}; + +struct CacheResponse +{ + ResponseType type; + nsCString[] urlList; + uint32_t status; + nsCString statusText; + HeadersEntry[] headers; + HeadersGuardEnum headersGuard; + CacheReadStream? body; + nullable nsITransportSecurityInfo securityInfo; + PrincipalInfo? principalInfo; + uint32_t paddingInfo; + int64_t paddingSize; + RequestCredentials credentials; +}; + +struct CacheRequestResponse +{ + CacheRequest request; + CacheResponse response; +}; + +struct CacheMatchArgs +{ + CacheRequest request; + CacheQueryParams params; + OpenMode openMode; +}; + +struct CacheMatchAllArgs +{ + CacheRequest? maybeRequest; + CacheQueryParams params; + OpenMode openMode; +}; + +struct CachePutAllArgs +{ + CacheRequestResponse[] requestResponseList; +}; + +struct CacheDeleteArgs +{ + CacheRequest request; + CacheQueryParams params; +}; + +struct CacheKeysArgs +{ + CacheRequest? maybeRequest; + CacheQueryParams params; + OpenMode openMode; +}; + +struct StorageMatchArgs +{ + CacheRequest request; + CacheQueryParams params; + OpenMode openMode; +}; + +struct StorageHasArgs +{ + nsString key; +}; + +struct StorageOpenArgs +{ + nsString key; +}; + +struct StorageDeleteArgs +{ + nsString key; +}; + +struct StorageKeysArgs +{ +}; + +union CacheOpArgs +{ + CacheMatchArgs; + CacheMatchAllArgs; + CachePutAllArgs; + CacheDeleteArgs; + CacheKeysArgs; + StorageMatchArgs; + StorageHasArgs; + StorageOpenArgs; + StorageDeleteArgs; + StorageKeysArgs; +}; + +struct CacheMatchResult +{ + CacheResponse? maybeResponse; +}; + +struct CacheMatchAllResult +{ + CacheResponse[] responseList; +}; + +struct CachePutAllResult +{ +}; + +struct CacheDeleteResult +{ + bool success; +}; + +struct CacheKeysResult +{ + CacheRequest[] requestList; +}; + +struct StorageMatchResult +{ + CacheResponse? maybeResponse; +}; + +struct StorageHasResult +{ + bool success; +}; + +struct StorageOpenResult +{ + nullable PCache actor; + Namespace ns; +}; + +struct StorageDeleteResult +{ + bool success; +}; + +struct StorageKeysResult +{ + nsString[] keyList; +}; + +union CacheOpResult +{ + void_t; + CacheMatchResult; + CacheMatchAllResult; + CachePutAllResult; + CacheDeleteResult; + CacheKeysResult; + StorageMatchResult; + StorageHasResult; + StorageOpenResult; + StorageDeleteResult; + StorageKeysResult; +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla diff --git a/dom/cache/CacheWorkerRef.cpp b/dom/cache/CacheWorkerRef.cpp new file mode 100644 index 0000000000..992de976e1 --- /dev/null +++ b/dom/cache/CacheWorkerRef.cpp @@ -0,0 +1,121 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/CacheWorkerRef.h" + +#include "mozilla/dom/cache/ActorChild.h" +#include "mozilla/dom/WorkerPrivate.h" +#include "mozilla/dom/WorkerRef.h" + +namespace mozilla::dom::cache { + +// static +SafeRefPtr<CacheWorkerRef> CacheWorkerRef::Create(WorkerPrivate* aWorkerPrivate, + Behavior aBehavior) { + MOZ_DIAGNOSTIC_ASSERT(aWorkerPrivate); + + // XXX This looks as if this could be simplified now by moving into the ctor + // of CacheWorkerRef, since we can now use SafeRefPtrFromThis in the ctor + auto workerRef = + MakeSafeRefPtr<CacheWorkerRef>(aBehavior, ConstructorGuard{}); + auto notify = [workerRef = workerRef.clonePtr()] { workerRef->Notify(); }; + if (aBehavior == eStrongWorkerRef) { + workerRef->mStrongWorkerRef = StrongWorkerRef::Create( + aWorkerPrivate, "CacheWorkerRef-Strong", std::move(notify)); + } else { + MOZ_ASSERT(aBehavior == eIPCWorkerRef); + workerRef->mIPCWorkerRef = IPCWorkerRef::Create( + aWorkerPrivate, "CacheWorkerRef-IPC", std::move(notify)); + } + + if (NS_WARN_IF(!workerRef->mIPCWorkerRef && !workerRef->mStrongWorkerRef)) { + return nullptr; + } + + return workerRef; +} + +// static +SafeRefPtr<CacheWorkerRef> CacheWorkerRef::PreferBehavior( + SafeRefPtr<CacheWorkerRef> aCurrentRef, Behavior aBehavior) { + if (!aCurrentRef) { + return nullptr; + } + + SafeRefPtr<CacheWorkerRef> orig = std::move(aCurrentRef); + if (orig->mBehavior == aBehavior) { + return orig; + } + + WorkerPrivate* workerPrivate = nullptr; + if (orig->mBehavior == eStrongWorkerRef) { + workerPrivate = orig->mStrongWorkerRef->Private(); + } else { + MOZ_ASSERT(orig->mBehavior == eIPCWorkerRef); + workerPrivate = orig->mIPCWorkerRef->Private(); + } + + MOZ_ASSERT(workerPrivate); + + SafeRefPtr<CacheWorkerRef> replace = Create(workerPrivate, aBehavior); + return static_cast<bool>(replace) ? std::move(replace) : std::move(orig); +} + +void CacheWorkerRef::AddActor(ActorChild& aActor) { + NS_ASSERT_OWNINGTHREAD(CacheWorkerRef); + MOZ_ASSERT(!mActorList.Contains(&aActor)); + + mActorList.AppendElement(WrapNotNullUnchecked(&aActor)); + + // Allow an actor to be added after we've entered the Notifying case. We + // can't stop the actor creation from racing with out destruction of the + // other actors and we need to wait for this extra one to close as well. + // Signal it should destroy itself right away. + if (mNotified) { + aActor.StartDestroy(); + } +} + +void CacheWorkerRef::RemoveActor(ActorChild& aActor) { + NS_ASSERT_OWNINGTHREAD(CacheWorkerRef); + +#if defined(RELEASE_OR_BETA) + mActorList.RemoveElement(&aActor); +#else + MOZ_DIAGNOSTIC_ASSERT(mActorList.RemoveElement(&aActor)); +#endif + + MOZ_ASSERT(!mActorList.Contains(&aActor)); + + if (mActorList.IsEmpty()) { + mStrongWorkerRef = nullptr; + mIPCWorkerRef = nullptr; + } +} + +bool CacheWorkerRef::Notified() const { return mNotified; } + +void CacheWorkerRef::Notify() { + NS_ASSERT_OWNINGTHREAD(CacheWorkerRef); + + mNotified = true; + + // Start the asynchronous destruction of our actors. These will call back + // into RemoveActor() once the actor is destroyed. + for (const auto& actor : mActorList) { + actor->StartDestroy(); + } +} + +CacheWorkerRef::CacheWorkerRef(Behavior aBehavior, ConstructorGuard) + : mBehavior(aBehavior), mNotified(false) {} + +CacheWorkerRef::~CacheWorkerRef() { + NS_ASSERT_OWNINGTHREAD(CacheWorkerRef); + MOZ_DIAGNOSTIC_ASSERT(mActorList.IsEmpty()); +} + +} // namespace mozilla::dom::cache diff --git a/dom/cache/CacheWorkerRef.h b/dom/cache/CacheWorkerRef.h new file mode 100644 index 0000000000..1e41772057 --- /dev/null +++ b/dom/cache/CacheWorkerRef.h @@ -0,0 +1,67 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_CacheWorkerRef_h +#define mozilla_dom_cache_CacheWorkerRef_h + +#include "mozilla/dom/SafeRefPtr.h" +#include "nsISupportsImpl.h" +#include "nsTArray.h" + +namespace mozilla::dom { + +class IPCWorkerRef; +class StrongWorkerRef; +class WorkerPrivate; + +namespace cache { + +class ActorChild; + +class CacheWorkerRef final : public SafeRefCounted<CacheWorkerRef> { + public: + enum Behavior { + eStrongWorkerRef, + eIPCWorkerRef, + }; + + static SafeRefPtr<CacheWorkerRef> Create(WorkerPrivate* aWorkerPrivate, + Behavior aBehavior); + + static SafeRefPtr<CacheWorkerRef> PreferBehavior( + SafeRefPtr<CacheWorkerRef> aCurrentRef, Behavior aBehavior); + + void AddActor(ActorChild& aActor); + void RemoveActor(ActorChild& aActor); + + bool Notified() const; + + private: + struct ConstructorGuard {}; + + void Notify(); + + nsTArray<NotNull<ActorChild*>> mActorList; + + Behavior mBehavior; + bool mNotified; + + RefPtr<StrongWorkerRef> mStrongWorkerRef; + RefPtr<IPCWorkerRef> mIPCWorkerRef; + + public: + CacheWorkerRef(Behavior aBehavior, ConstructorGuard); + + ~CacheWorkerRef(); + + NS_DECL_OWNINGTHREAD + MOZ_DECLARE_REFCOUNTED_TYPENAME(mozilla::dom::cache::CacheWorkerRef) +}; + +} // namespace cache +} // namespace mozilla::dom + +#endif // mozilla_dom_cache_CacheWorkerRef_h diff --git a/dom/cache/Connection.cpp b/dom/cache/Connection.cpp new file mode 100644 index 0000000000..a2ecba9978 --- /dev/null +++ b/dom/cache/Connection.cpp @@ -0,0 +1,279 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/Connection.h" + +#include "mozilla/dom/cache/DBSchema.h" +#include "mozStorageHelper.h" + +namespace mozilla::dom::cache { + +using mozilla::dom::quota::QuotaObject; + +NS_IMPL_ISUPPORTS(cache::Connection, mozIStorageAsyncConnection, + mozIStorageConnection); + +Connection::Connection(mozIStorageConnection* aBase) + : mBase(aBase), mClosed(false) { + MOZ_DIAGNOSTIC_ASSERT(mBase); +} + +Connection::~Connection() { + NS_ASSERT_OWNINGTHREAD(Connection); + MOZ_ALWAYS_SUCCEEDS(Close()); +} + +NS_IMETHODIMP +Connection::Close() { + NS_ASSERT_OWNINGTHREAD(Connection); + + if (mClosed) { + return NS_OK; + } + mClosed = true; + + // If we are closing here, then Cache must not have a transaction + // open anywhere else. This may fail if storage is corrupted. + Unused << NS_WARN_IF(NS_FAILED(db::IncrementalVacuum(*this))); + + return mBase->Close(); +} + +// The following methods are all boilerplate that either forward to the +// base connection or block the method. All the async execution methods +// are blocked because Cache does not use them and they would require more +// work to wrap properly. + +// mozIStorageAsyncConnection methods + +NS_IMETHODIMP +Connection::AsyncVacuum(mozIStorageCompletionCallback*, bool, int32_t) { + // async methods are not supported + return NS_ERROR_NOT_IMPLEMENTED; +} + +NS_IMETHODIMP +Connection::AsyncClose(mozIStorageCompletionCallback*) { + // async methods are not supported + return NS_ERROR_NOT_IMPLEMENTED; +} + +NS_IMETHODIMP +Connection::SpinningSynchronousClose() { + // not supported + return NS_ERROR_NOT_IMPLEMENTED; +} + +NS_IMETHODIMP +Connection::AsyncClone(bool, mozIStorageCompletionCallback*) { + // async methods are not supported + return NS_ERROR_NOT_IMPLEMENTED; +} + +NS_IMETHODIMP +Connection::GetDatabaseFile(nsIFile** aFileOut) { + return mBase->GetDatabaseFile(aFileOut); +} + +NS_IMETHODIMP +Connection::CreateAsyncStatement(const nsACString&, + mozIStorageAsyncStatement**) { + // async methods are not supported + return NS_ERROR_NOT_IMPLEMENTED; +} + +NS_IMETHODIMP +Connection::ExecuteAsync(const nsTArray<RefPtr<mozIStorageBaseStatement>>&, + mozIStorageStatementCallback*, + mozIStoragePendingStatement**) { + // async methods are not supported + return NS_ERROR_NOT_IMPLEMENTED; +} + +NS_IMETHODIMP +Connection::ExecuteSimpleSQLAsync(const nsACString&, + mozIStorageStatementCallback*, + mozIStoragePendingStatement**) { + // async methods are not supported + return NS_ERROR_NOT_IMPLEMENTED; +} + +NS_IMETHODIMP +Connection::CreateFunction(const nsACString& aFunctionName, + int32_t aNumArguments, + mozIStorageFunction* aFunction) { + // async methods are not supported + return NS_ERROR_NOT_IMPLEMENTED; +} + +NS_IMETHODIMP +Connection::RemoveFunction(const nsACString& aFunctionName) { + return mBase->RemoveFunction(aFunctionName); +} + +NS_IMETHODIMP +Connection::SetProgressHandler(int32_t aGranularity, + mozIStorageProgressHandler* aHandler, + mozIStorageProgressHandler** aHandlerOut) { + return mBase->SetProgressHandler(aGranularity, aHandler, aHandlerOut); +} + +NS_IMETHODIMP +Connection::RemoveProgressHandler(mozIStorageProgressHandler** aHandlerOut) { + return mBase->RemoveProgressHandler(aHandlerOut); +} + +// mozIStorageConnection methods + +NS_IMETHODIMP +Connection::Clone(bool aReadOnly, mozIStorageConnection** aConnectionOut) { + nsCOMPtr<mozIStorageConnection> conn; + nsresult rv = mBase->Clone(aReadOnly, getter_AddRefs(conn)); + if (NS_WARN_IF(NS_FAILED(rv))) { + return rv; + } + + nsCOMPtr<mozIStorageConnection> wrapped = new Connection(conn); + wrapped.forget(aConnectionOut); + + return rv; +} + +NS_IMETHODIMP +Connection::Interrupt() { return mBase->Interrupt(); } + +NS_IMETHODIMP +Connection::GetDefaultPageSize(int32_t* aSizeOut) { + return mBase->GetDefaultPageSize(aSizeOut); +} + +NS_IMETHODIMP +Connection::GetConnectionReady(bool* aReadyOut) { + return mBase->GetConnectionReady(aReadyOut); +} + +NS_IMETHODIMP +Connection::GetLastInsertRowID(int64_t* aRowIdOut) { + return mBase->GetLastInsertRowID(aRowIdOut); +} + +NS_IMETHODIMP +Connection::GetAffectedRows(int32_t* aCountOut) { + return mBase->GetAffectedRows(aCountOut); +} + +NS_IMETHODIMP +Connection::GetLastError(int32_t* aErrorOut) { + return mBase->GetLastError(aErrorOut); +} + +NS_IMETHODIMP +Connection::GetLastErrorString(nsACString& aErrorOut) { + return mBase->GetLastErrorString(aErrorOut); +} + +NS_IMETHODIMP +Connection::GetSchemaVersion(int32_t* aVersionOut) { + return mBase->GetSchemaVersion(aVersionOut); +} + +NS_IMETHODIMP +Connection::SetSchemaVersion(int32_t aVersion) { + return mBase->SetSchemaVersion(aVersion); +} + +NS_IMETHODIMP +Connection::CreateStatement(const nsACString& aQuery, + mozIStorageStatement** aStatementOut) { + return mBase->CreateStatement(aQuery, aStatementOut); +} + +NS_IMETHODIMP +Connection::ExecuteSimpleSQL(const nsACString& aQuery) { + return mBase->ExecuteSimpleSQL(aQuery); +} + +NS_IMETHODIMP +Connection::TableExists(const nsACString& aTableName, bool* aExistsOut) { + return mBase->TableExists(aTableName, aExistsOut); +} + +NS_IMETHODIMP +Connection::IndexExists(const nsACString& aIndexName, bool* aExistsOut) { + return mBase->IndexExists(aIndexName, aExistsOut); +} + +NS_IMETHODIMP +Connection::GetTransactionInProgress(bool* aResultOut) { + return mBase->GetTransactionInProgress(aResultOut); +} + +NS_IMETHODIMP +Connection::GetDefaultTransactionType(int32_t* aResultOut) { + return mBase->GetDefaultTransactionType(aResultOut); +} + +NS_IMETHODIMP +Connection::SetDefaultTransactionType(int32_t aType) { + return mBase->SetDefaultTransactionType(aType); +} + +NS_IMETHODIMP +Connection::GetVariableLimit(int32_t* aResultOut) { + return mBase->GetVariableLimit(aResultOut); +} + +NS_IMETHODIMP +Connection::BeginTransaction() { return mBase->BeginTransaction(); } + +NS_IMETHODIMP +Connection::CommitTransaction() { return mBase->CommitTransaction(); } + +NS_IMETHODIMP +Connection::RollbackTransaction() { return mBase->RollbackTransaction(); } + +NS_IMETHODIMP +Connection::CreateTable(const char* aTable, const char* aSchema) { + return mBase->CreateTable(aTable, aSchema); +} + +NS_IMETHODIMP +Connection::SetGrowthIncrement(int32_t aIncrement, + const nsACString& aDatabase) { + return mBase->SetGrowthIncrement(aIncrement, aDatabase); +} + +NS_IMETHODIMP +Connection::EnableModule(const nsACString& aModule) { + return mBase->EnableModule(aModule); +} + +NS_IMETHODIMP +Connection::GetQuotaObjects(QuotaObject** aDatabaseQuotaObject, + QuotaObject** aJournalQuotaObject) { + return mBase->GetQuotaObjects(aDatabaseQuotaObject, aJournalQuotaObject); +} + +mozilla::storage::SQLiteMutex& Connection::GetSharedDBMutex() { + return mBase->GetSharedDBMutex(); +} + +uint32_t Connection::GetTransactionNestingLevel( + const mozilla::storage::SQLiteMutexAutoLock& aProofOfLock) { + return mBase->GetTransactionNestingLevel(aProofOfLock); +} + +uint32_t Connection::IncreaseTransactionNestingLevel( + const mozilla::storage::SQLiteMutexAutoLock& aProofOfLock) { + return mBase->IncreaseTransactionNestingLevel(aProofOfLock); +} + +uint32_t Connection::DecreaseTransactionNestingLevel( + const mozilla::storage::SQLiteMutexAutoLock& aProofOfLock) { + return mBase->DecreaseTransactionNestingLevel(aProofOfLock); +} + +} // namespace mozilla::dom::cache diff --git a/dom/cache/Connection.h b/dom/cache/Connection.h new file mode 100644 index 0000000000..13ef416c3e --- /dev/null +++ b/dom/cache/Connection.h @@ -0,0 +1,32 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_Connection_h +#define mozilla_dom_cache_Connection_h + +#include "mozIStorageConnection.h" +#include "nsCOMPtr.h" + +namespace mozilla::dom::cache { + +class Connection final : public mozIStorageConnection { + public: + explicit Connection(mozIStorageConnection* aBase); + + private: + ~Connection(); + + nsCOMPtr<mozIStorageConnection> mBase; + bool mClosed; + + NS_DECL_ISUPPORTS + NS_DECL_MOZISTORAGEASYNCCONNECTION + NS_DECL_MOZISTORAGECONNECTION +}; + +} // namespace mozilla::dom::cache + +#endif // mozilla_dom_cache_Connection_h diff --git a/dom/cache/Context.cpp b/dom/cache/Context.cpp new file mode 100644 index 0000000000..15fd9cb79a --- /dev/null +++ b/dom/cache/Context.cpp @@ -0,0 +1,1076 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/Context.h" + +#include "CacheCommon.h" + +#include "mozilla/AutoRestore.h" +#include "mozilla/dom/SafeRefPtr.h" +#include "mozilla/dom/cache/Action.h" +#include "mozilla/dom/cache/FileUtils.h" +#include "mozilla/dom/cache/Manager.h" +#include "mozilla/dom/cache/ManagerId.h" +#include "mozilla/dom/quota/Assertions.h" +#include "mozilla/dom/quota/DirectoryLock.h" +#include "mozilla/dom/quota/QuotaManager.h" +#include "mozilla/dom/quota/ResultExtensions.h" +#include "mozilla/ipc/PBackgroundSharedTypes.h" +#include "mozIStorageConnection.h" +#include "nsIPrincipal.h" +#include "nsIRunnable.h" +#include "nsIThread.h" +#include "nsThreadUtils.h" + +namespace { + +using mozilla::dom::cache::Action; +using mozilla::dom::cache::CacheDirectoryMetadata; + +class NullAction final : public Action { + public: + NullAction() = default; + + virtual void RunOnTarget(mozilla::SafeRefPtr<Resolver> aResolver, + const mozilla::Maybe<CacheDirectoryMetadata>&, + Data*) override { + // Resolve success immediately. This Action does no actual work. + MOZ_DIAGNOSTIC_ASSERT(aResolver); + aResolver->Resolve(NS_OK); + } +}; + +} // namespace + +namespace mozilla::dom::cache { + +using mozilla::dom::quota::AssertIsOnIOThread; +using mozilla::dom::quota::DirectoryLock; +using mozilla::dom::quota::OpenDirectoryListener; +using mozilla::dom::quota::PERSISTENCE_TYPE_DEFAULT; +using mozilla::dom::quota::PersistenceType; +using mozilla::dom::quota::QuotaManager; + +class Context::Data final : public Action::Data { + public: + explicit Data(nsISerialEventTarget* aTarget) : mTarget(aTarget) { + MOZ_DIAGNOSTIC_ASSERT(mTarget); + } + + virtual mozIStorageConnection* GetConnection() const override { + MOZ_ASSERT(mTarget->IsOnCurrentThread()); + return mConnection; + } + + virtual void SetConnection(mozIStorageConnection* aConn) override { + MOZ_ASSERT(mTarget->IsOnCurrentThread()); + MOZ_DIAGNOSTIC_ASSERT(!mConnection); + mConnection = aConn; + MOZ_DIAGNOSTIC_ASSERT(mConnection); + } + + private: + ~Data() { + // We could proxy release our data here, but instead just assert. The + // Context code should guarantee that we are destroyed on the target + // thread once the connection is initialized. If we're not, then + // QuotaManager might race and try to clear the origin out from under us. + MOZ_ASSERT_IF(mConnection, mTarget->IsOnCurrentThread()); + } + + nsCOMPtr<nsISerialEventTarget> mTarget; + nsCOMPtr<mozIStorageConnection> mConnection; + + // Threadsafe counting because we're created on the PBackground thread + // and destroyed on the target IO thread. + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(Context::Data) +}; + +// Executed to perform the complicated dance of steps necessary to initialize +// the QuotaManager. This must be performed for each origin before any disk +// IO occurrs. +class Context::QuotaInitRunnable final : public nsIRunnable, + public OpenDirectoryListener { + public: + QuotaInitRunnable(SafeRefPtr<Context> aContext, SafeRefPtr<Manager> aManager, + Data* aData, nsISerialEventTarget* aTarget, + SafeRefPtr<Action> aInitAction) + : mContext(std::move(aContext)), + mThreadsafeHandle(mContext->CreateThreadsafeHandle()), + mManager(std::move(aManager)), + mData(aData), + mTarget(aTarget), + mInitAction(std::move(aInitAction)), + mInitiatingEventTarget(GetCurrentSerialEventTarget()), + mResult(NS_OK), + mState(STATE_INIT), + mCanceled(false) { + MOZ_DIAGNOSTIC_ASSERT(mContext); + MOZ_DIAGNOSTIC_ASSERT(mManager); + MOZ_DIAGNOSTIC_ASSERT(mData); + MOZ_DIAGNOSTIC_ASSERT(mTarget); + MOZ_DIAGNOSTIC_ASSERT(mInitiatingEventTarget); + MOZ_DIAGNOSTIC_ASSERT(mInitAction); + } + + Maybe<DirectoryLock&> MaybeDirectoryLockRef() const { + NS_ASSERT_OWNINGTHREAD(QuotaInitRunnable); + + return ToMaybeRef(mDirectoryLock.get()); + } + + nsresult Dispatch() { + NS_ASSERT_OWNINGTHREAD(QuotaInitRunnable); + MOZ_DIAGNOSTIC_ASSERT(mState == STATE_INIT); + + mState = STATE_GET_INFO; + nsresult rv = NS_DispatchToMainThread(this, nsIThread::DISPATCH_NORMAL); + if (NS_WARN_IF(NS_FAILED(rv))) { + mState = STATE_COMPLETE; + Clear(); + } + return rv; + } + + void Cancel() { + NS_ASSERT_OWNINGTHREAD(QuotaInitRunnable); + MOZ_DIAGNOSTIC_ASSERT(!mCanceled); + mCanceled = true; + mInitAction->CancelOnInitiatingThread(); + } + + // OpenDirectoryListener methods + virtual void DirectoryLockAcquired(DirectoryLock* aLock) override; + + virtual void DirectoryLockFailed() override; + + private: + class SyncResolver final : public Action::Resolver { + public: + SyncResolver() : mResolved(false), mResult(NS_OK) {} + + virtual void Resolve(nsresult aRv) override { + MOZ_DIAGNOSTIC_ASSERT(!mResolved); + mResolved = true; + mResult = aRv; + }; + + bool Resolved() const { return mResolved; } + nsresult Result() const { return mResult; } + + private: + ~SyncResolver() = default; + + bool mResolved; + nsresult mResult; + + NS_INLINE_DECL_REFCOUNTING(Context::QuotaInitRunnable::SyncResolver, + override) + }; + + ~QuotaInitRunnable() { + MOZ_DIAGNOSTIC_ASSERT(mState == STATE_COMPLETE); + MOZ_DIAGNOSTIC_ASSERT(!mContext); + MOZ_DIAGNOSTIC_ASSERT(!mInitAction); + } + + enum State { + STATE_INIT, + STATE_GET_INFO, + STATE_CREATE_QUOTA_MANAGER, + STATE_WAIT_FOR_DIRECTORY_LOCK, + STATE_ENSURE_ORIGIN_INITIALIZED, + STATE_RUN_ON_TARGET, + STATE_RUNNING, + STATE_COMPLETING, + STATE_COMPLETE + }; + + void Complete(nsresult aResult) { + MOZ_DIAGNOSTIC_ASSERT(mState == STATE_RUNNING || NS_FAILED(aResult)); + + MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(mResult)); + mResult = aResult; + + mState = STATE_COMPLETING; + MOZ_ALWAYS_SUCCEEDS( + mInitiatingEventTarget->Dispatch(this, nsIThread::DISPATCH_NORMAL)); + } + + void Clear() { + NS_ASSERT_OWNINGTHREAD(QuotaInitRunnable); + MOZ_DIAGNOSTIC_ASSERT(mContext); + mContext = nullptr; + mManager = nullptr; + mInitAction = nullptr; + } + + SafeRefPtr<Context> mContext; + SafeRefPtr<ThreadsafeHandle> mThreadsafeHandle; + SafeRefPtr<Manager> mManager; + RefPtr<Data> mData; + nsCOMPtr<nsISerialEventTarget> mTarget; + SafeRefPtr<Action> mInitAction; + nsCOMPtr<nsIEventTarget> mInitiatingEventTarget; + nsresult mResult; + Maybe<mozilla::ipc::PrincipalInfo> mPrincipalInfo; + Maybe<CacheDirectoryMetadata> mDirectoryMetadata; + RefPtr<DirectoryLock> mDirectoryLock; + State mState; + Atomic<bool> mCanceled; + + public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSIRUNNABLE +}; + +void Context::QuotaInitRunnable::DirectoryLockAcquired(DirectoryLock* aLock) { + NS_ASSERT_OWNINGTHREAD(QuotaInitRunnable); + MOZ_DIAGNOSTIC_ASSERT(aLock); + MOZ_DIAGNOSTIC_ASSERT(mState == STATE_WAIT_FOR_DIRECTORY_LOCK); + MOZ_DIAGNOSTIC_ASSERT(!mDirectoryLock); + + mDirectoryLock = aLock; + + MOZ_DIAGNOSTIC_ASSERT(mDirectoryLock->Id() >= 0); + mDirectoryMetadata->mDirectoryLockId = mDirectoryLock->Id(); + + if (mCanceled) { + Complete(NS_ERROR_ABORT); + return; + } + + QuotaManager* qm = QuotaManager::Get(); + MOZ_DIAGNOSTIC_ASSERT(qm); + + mState = STATE_ENSURE_ORIGIN_INITIALIZED; + nsresult rv = qm->IOThread()->Dispatch(this, nsIThread::DISPATCH_NORMAL); + if (NS_WARN_IF(NS_FAILED(rv))) { + Complete(rv); + return; + } +} + +void Context::QuotaInitRunnable::DirectoryLockFailed() { + NS_ASSERT_OWNINGTHREAD(QuotaInitRunnable); + MOZ_DIAGNOSTIC_ASSERT(mState == STATE_WAIT_FOR_DIRECTORY_LOCK); + MOZ_DIAGNOSTIC_ASSERT(!mDirectoryLock); + + NS_WARNING("Failed to acquire a directory lock!"); + + Complete(NS_ERROR_FAILURE); +} + +NS_IMPL_ISUPPORTS(mozilla::dom::cache::Context::QuotaInitRunnable, nsIRunnable); + +// The QuotaManager init state machine is represented in the following diagram: +// +// +---------------+ +// | Start | Resolve(error) +// | (Orig Thread) +---------------------+ +// +-------+-------+ | +// | | +// +----------v-----------+ | +// | GetInfo | Resolve(error) | +// | (Main Thread) +-----------------+ +// +----------+-----------+ | +// | | +// +----------v-----------+ | +// | CreateQuotaManager | Resolve(error) | +// | (Orig Thread) +-----------------+ +// +----------+-----------+ | +// | | +// +----------v-----------+ | +// | WaitForDirectoryLock | Resolve(error) | +// | (Orig Thread) +-----------------+ +// +----------+-----------+ | +// | | +// +----------v------------+ | +// |EnsureOriginInitialized| Resolve(error) | +// | (Quota IO Thread) +----------------+ +// +----------+------------+ | +// | | +// +----------v------------+ | +// | RunOnTarget | Resolve(error) | +// | (Target Thread) +----------------+ +// +----------+------------+ | +// | | +// +---------v---------+ +------v------+ +// | Running | | Completing | +// | (Target Thread) +------------>(Orig Thread)| +// +-------------------+ +------+------+ +// | +// +-----v----+ +// | Complete | +// +----------+ +// +// The initialization process proceeds through the main states. If an error +// occurs, then we transition to Completing state back on the original thread. +NS_IMETHODIMP +Context::QuotaInitRunnable::Run() { + // May run on different threads depending on the state. See individual + // state cases for thread assertions. + + SafeRefPtr<SyncResolver> resolver = MakeSafeRefPtr<SyncResolver>(); + + switch (mState) { + // ----------------------------------- + case STATE_GET_INFO: { + MOZ_ASSERT(NS_IsMainThread()); + + auto res = [this]() -> Result<Ok, nsresult> { + if (mCanceled) { + return Err(NS_ERROR_ABORT); + } + + nsCOMPtr<nsIPrincipal> principal = mManager->GetManagerId().Principal(); + + mozilla::ipc::PrincipalInfo principalInfo; + QM_TRY( + MOZ_TO_RESULT(PrincipalToPrincipalInfo(principal, &principalInfo))); + + mPrincipalInfo.emplace(std::move(principalInfo)); + + mState = STATE_CREATE_QUOTA_MANAGER; + + MOZ_ALWAYS_SUCCEEDS( + mInitiatingEventTarget->Dispatch(this, nsIThread::DISPATCH_NORMAL)); + + return Ok{}; + }(); + + if (res.isErr()) { + resolver->Resolve(res.inspectErr()); + } + + break; + } + // ---------------------------------- + case STATE_CREATE_QUOTA_MANAGER: { + NS_ASSERT_OWNINGTHREAD(QuotaInitRunnable); + + if (mCanceled || QuotaManager::IsShuttingDown()) { + resolver->Resolve(NS_ERROR_ABORT); + break; + } + + QM_TRY(QuotaManager::EnsureCreated(), QM_PROPAGATE, + [&resolver](const auto rv) { resolver->Resolve(rv); }); + + auto* const quotaManager = QuotaManager::Get(); + MOZ_DIAGNOSTIC_ASSERT(quotaManager); + + QM_TRY_UNWRAP( + auto principalMetadata, + quotaManager->GetInfoFromValidatedPrincipalInfo(*mPrincipalInfo)); + + mDirectoryMetadata.emplace(std::move(principalMetadata)); + + // Open directory + RefPtr<DirectoryLock> directoryLock = quotaManager->CreateDirectoryLock( + PERSISTENCE_TYPE_DEFAULT, *mDirectoryMetadata, + quota::Client::DOMCACHE, + /* aExclusive */ false); + + // DirectoryLock::Acquire() will hold a reference to us as a listener. We + // will then get DirectoryLockAcquired() on the owning thread when it is + // safe to access our storage directory. + mState = STATE_WAIT_FOR_DIRECTORY_LOCK; + directoryLock->Acquire(this); + + break; + } + // ---------------------------------- + case STATE_ENSURE_ORIGIN_INITIALIZED: { + AssertIsOnIOThread(); + + auto res = [this]() -> Result<Ok, nsresult> { + if (mCanceled) { + return Err(NS_ERROR_ABORT); + } + + QuotaManager* quotaManager = QuotaManager::Get(); + MOZ_DIAGNOSTIC_ASSERT(quotaManager); + + QM_TRY(MOZ_TO_RESULT(quotaManager->EnsureStorageIsInitialized())); + + QM_TRY( + MOZ_TO_RESULT(quotaManager->EnsureTemporaryStorageIsInitialized())); + + QM_TRY_UNWRAP(mDirectoryMetadata->mDir, + quotaManager + ->EnsureTemporaryOriginIsInitialized( + PERSISTENCE_TYPE_DEFAULT, *mDirectoryMetadata) + .map([](const auto& res) { return res.first; })); + + mState = STATE_RUN_ON_TARGET; + + MOZ_ALWAYS_SUCCEEDS( + mTarget->Dispatch(this, nsIThread::DISPATCH_NORMAL)); + + return Ok{}; + }(); + + if (res.isErr()) { + resolver->Resolve(res.inspectErr()); + } + + break; + } + // ------------------- + case STATE_RUN_ON_TARGET: { + MOZ_ASSERT(mTarget->IsOnCurrentThread()); + + mState = STATE_RUNNING; + + // Execute the provided initialization Action. The Action must Resolve() + // before returning. + mInitAction->RunOnTarget(resolver.clonePtr(), mDirectoryMetadata, mData); + MOZ_DIAGNOSTIC_ASSERT(resolver->Resolved()); + + mData = nullptr; + + // If the database was opened, then we should always succeed when creating + // the marker file. If it wasn't opened successfully, then no need to + // create a marker file anyway. + if (NS_SUCCEEDED(resolver->Result())) { + MOZ_ALWAYS_SUCCEEDS(CreateMarkerFile(*mDirectoryMetadata)); + } + + break; + } + // ------------------- + case STATE_COMPLETING: { + NS_ASSERT_OWNINGTHREAD(QuotaInitRunnable); + mInitAction->CompleteOnInitiatingThread(mResult); + mContext->OnQuotaInit(mResult, mDirectoryMetadata, + mDirectoryLock.forget()); + mState = STATE_COMPLETE; + + // Explicitly cleanup here as the destructor could fire on any of + // the threads we have bounced through. + Clear(); + break; + } + // ----- + case STATE_WAIT_FOR_DIRECTORY_LOCK: + default: { + MOZ_CRASH("unexpected state in QuotaInitRunnable"); + } + } + + if (resolver->Resolved()) { + Complete(resolver->Result()); + } + + return NS_OK; +} + +// Runnable wrapper around Action objects dispatched on the Context. This +// runnable executes the Action on the appropriate threads while the Context +// is initialized. +class Context::ActionRunnable final : public nsIRunnable, + public Action::Resolver, + public Context::Activity { + public: + ActionRunnable(SafeRefPtr<Context> aContext, Data* aData, + nsISerialEventTarget* aTarget, SafeRefPtr<Action> aAction, + const Maybe<CacheDirectoryMetadata>& aDirectoryMetadata) + : mContext(std::move(aContext)), + mData(aData), + mTarget(aTarget), + mAction(std::move(aAction)), + mDirectoryMetadata(aDirectoryMetadata), + mInitiatingThread(GetCurrentSerialEventTarget()), + mState(STATE_INIT), + mResult(NS_OK), + mExecutingRunOnTarget(false) { + MOZ_DIAGNOSTIC_ASSERT(mContext); + // mData may be nullptr + MOZ_DIAGNOSTIC_ASSERT(mTarget); + MOZ_DIAGNOSTIC_ASSERT(mAction); + // mDirectoryMetadata.mDir may be nullptr if QuotaInitRunnable failed + MOZ_DIAGNOSTIC_ASSERT(mInitiatingThread); + } + + nsresult Dispatch() { + NS_ASSERT_OWNINGTHREAD(ActionRunnable); + MOZ_DIAGNOSTIC_ASSERT(mState == STATE_INIT); + + mState = STATE_RUN_ON_TARGET; + nsresult rv = mTarget->Dispatch(this, nsIEventTarget::DISPATCH_NORMAL); + if (NS_WARN_IF(NS_FAILED(rv))) { + mState = STATE_COMPLETE; + Clear(); + } + return rv; + } + + virtual bool MatchesCacheId(CacheId aCacheId) const override { + NS_ASSERT_OWNINGTHREAD(ActionRunnable); + return mAction->MatchesCacheId(aCacheId); + } + + virtual void Cancel() override { + NS_ASSERT_OWNINGTHREAD(ActionRunnable); + mAction->CancelOnInitiatingThread(); + } + + virtual void Resolve(nsresult aRv) override { + MOZ_ASSERT(mTarget->IsOnCurrentThread()); + MOZ_DIAGNOSTIC_ASSERT(mState == STATE_RUNNING); + + mResult = aRv; + + // We ultimately must complete on the initiating thread, but bounce through + // the current thread again to ensure that we don't destroy objects and + // state out from under the currently running action's stack. + mState = STATE_RESOLVING; + + // If we were resolved synchronously within Action::RunOnTarget() then we + // can avoid a thread bounce and just resolve once RunOnTarget() returns. + // The Run() method will handle this by looking at mState after + // RunOnTarget() returns. + if (mExecutingRunOnTarget) { + return; + } + + // Otherwise we are in an asynchronous resolve. And must perform a thread + // bounce to run on the target thread again. + MOZ_ALWAYS_SUCCEEDS(mTarget->Dispatch(this, nsIThread::DISPATCH_NORMAL)); + } + + private: + ~ActionRunnable() { + MOZ_DIAGNOSTIC_ASSERT(mState == STATE_COMPLETE); + MOZ_DIAGNOSTIC_ASSERT(!mContext); + MOZ_DIAGNOSTIC_ASSERT(!mAction); + } + + void Clear() { + NS_ASSERT_OWNINGTHREAD(ActionRunnable); + MOZ_DIAGNOSTIC_ASSERT(mContext); + MOZ_DIAGNOSTIC_ASSERT(mAction); + mContext->RemoveActivity(*this); + mContext = nullptr; + mAction = nullptr; + } + + enum State { + STATE_INIT, + STATE_RUN_ON_TARGET, + STATE_RUNNING, + STATE_RESOLVING, + STATE_COMPLETING, + STATE_COMPLETE + }; + + SafeRefPtr<Context> mContext; + RefPtr<Data> mData; + nsCOMPtr<nsISerialEventTarget> mTarget; + SafeRefPtr<Action> mAction; + const Maybe<CacheDirectoryMetadata> mDirectoryMetadata; + nsCOMPtr<nsIEventTarget> mInitiatingThread; + State mState; + nsresult mResult; + + // Only accessible on target thread; + bool mExecutingRunOnTarget; + + public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSIRUNNABLE +}; + +NS_IMPL_ISUPPORTS(mozilla::dom::cache::Context::ActionRunnable, nsIRunnable); + +// The ActionRunnable has a simpler state machine. It basically needs to run +// the action on the target thread and then complete on the original thread. +// +// +-------------+ +// | Start | +// |(Orig Thread)| +// +-----+-------+ +// | +// +-------v---------+ +// | RunOnTarget | +// |Target IO Thread)+---+ Resolve() +// +-------+---------+ | +// | | +// +-------v----------+ | +// | Running | | +// |(Target IO Thread)| | +// +------------------+ | +// | Resolve() | +// +-------v----------+ | +// | Resolving <--+ +-------------+ +// | | | Completing | +// |(Target IO Thread)+---------------------->(Orig Thread)| +// +------------------+ +-------+-----+ +// | +// | +// +----v---+ +// |Complete| +// +--------+ +// +// Its important to note that synchronous actions will effectively Resolve() +// out of the Running state immediately. Asynchronous Actions may remain +// in the Running state for some time, but normally the ActionRunnable itself +// does not see any execution there. Its all handled internal to the Action. +NS_IMETHODIMP +Context::ActionRunnable::Run() { + switch (mState) { + // ---------------------- + case STATE_RUN_ON_TARGET: { + MOZ_ASSERT(mTarget->IsOnCurrentThread()); + MOZ_DIAGNOSTIC_ASSERT(!mExecutingRunOnTarget); + + // Note that we are calling RunOnTarget(). This lets us detect + // if Resolve() is called synchronously. + AutoRestore<bool> executingRunOnTarget(mExecutingRunOnTarget); + mExecutingRunOnTarget = true; + + mState = STATE_RUNNING; + mAction->RunOnTarget(SafeRefPtrFromThis(), mDirectoryMetadata, mData); + + mData = nullptr; + + // Resolve was called synchronously from RunOnTarget(). We can + // immediately move to completing now since we are sure RunOnTarget() + // completed. + if (mState == STATE_RESOLVING) { + // Use recursion instead of switch case fall-through... Seems slightly + // easier to understand. + Run(); + } + + break; + } + // ----------------- + case STATE_RESOLVING: { + MOZ_ASSERT(mTarget->IsOnCurrentThread()); + // The call to Action::RunOnTarget() must have returned now if we + // are running on the target thread again. We may now proceed + // with completion. + mState = STATE_COMPLETING; + // Shutdown must be delayed until all Contexts are destroyed. Crash + // for this invariant violation. + MOZ_ALWAYS_SUCCEEDS( + mInitiatingThread->Dispatch(this, nsIThread::DISPATCH_NORMAL)); + break; + } + // ------------------- + case STATE_COMPLETING: { + NS_ASSERT_OWNINGTHREAD(ActionRunnable); + mAction->CompleteOnInitiatingThread(mResult); + mState = STATE_COMPLETE; + // Explicitly cleanup here as the destructor could fire on any of + // the threads we have bounced through. + Clear(); + break; + } + // ----------------- + default: { + MOZ_CRASH("unexpected state in ActionRunnable"); + break; + } + } + return NS_OK; +} + +void Context::ThreadsafeHandle::AllowToClose() { + if (mOwningEventTarget->IsOnCurrentThread()) { + AllowToCloseOnOwningThread(); + return; + } + + // Dispatch is guaranteed to succeed here because we block shutdown until + // all Contexts have been destroyed. + nsCOMPtr<nsIRunnable> runnable = NewRunnableMethod( + "dom::cache::Context::ThreadsafeHandle::AllowToCloseOnOwningThread", this, + &ThreadsafeHandle::AllowToCloseOnOwningThread); + MOZ_ALWAYS_SUCCEEDS(mOwningEventTarget->Dispatch(runnable.forget(), + nsIThread::DISPATCH_NORMAL)); +} + +void Context::ThreadsafeHandle::InvalidateAndAllowToClose() { + if (mOwningEventTarget->IsOnCurrentThread()) { + InvalidateAndAllowToCloseOnOwningThread(); + return; + } + + // Dispatch is guaranteed to succeed here because we block shutdown until + // all Contexts have been destroyed. + nsCOMPtr<nsIRunnable> runnable = NewRunnableMethod( + "dom::cache::Context::ThreadsafeHandle::" + "InvalidateAndAllowToCloseOnOwningThread", + this, &ThreadsafeHandle::InvalidateAndAllowToCloseOnOwningThread); + MOZ_ALWAYS_SUCCEEDS(mOwningEventTarget->Dispatch(runnable.forget(), + nsIThread::DISPATCH_NORMAL)); +} + +Context::ThreadsafeHandle::ThreadsafeHandle(SafeRefPtr<Context> aContext) + : mStrongRef(std::move(aContext)), + mWeakRef(mStrongRef.unsafeGetRawPtr()), + mOwningEventTarget(GetCurrentSerialEventTarget()) {} + +Context::ThreadsafeHandle::~ThreadsafeHandle() { + // Normally we only touch mStrongRef on the owning thread. This is safe, + // however, because when we do use mStrongRef on the owning thread we are + // always holding a strong ref to the ThreadsafeHandle via the owning + // runnable. So we cannot run the ThreadsafeHandle destructor simultaneously. + if (!mStrongRef || mOwningEventTarget->IsOnCurrentThread()) { + return; + } + + // Dispatch in NS_ProxyRelease is guaranteed to succeed here because we block + // shutdown until all Contexts have been destroyed. Therefore it is ok to have + // MOZ_ALWAYS_SUCCEED here. + MOZ_ALWAYS_SUCCEEDS(NS_ProxyRelease("Context::ThreadsafeHandle::mStrongRef", + mOwningEventTarget, mStrongRef.forget())); +} + +void Context::ThreadsafeHandle::AllowToCloseOnOwningThread() { + MOZ_ASSERT(mOwningEventTarget->IsOnCurrentThread()); + + // A Context "closes" when its ref count drops to zero. Dropping this + // strong ref is necessary, but not sufficient for the close to occur. + // Any outstanding IO will continue and keep the Context alive. Once + // the Context is idle, it will be destroyed. + + // First, tell the context to flush any target thread shared data. This + // data must be released on the target thread prior to running the Context + // destructor. This will schedule an Action which ensures that the + // ~Context() is not immediately executed when we drop the strong ref. + if (mStrongRef) { + mStrongRef->DoomTargetData(); + } + + // Now drop our strong ref and let Context finish running any outstanding + // Actions. + mStrongRef = nullptr; +} + +void Context::ThreadsafeHandle::InvalidateAndAllowToCloseOnOwningThread() { + MOZ_ASSERT(mOwningEventTarget->IsOnCurrentThread()); + // Cancel the Context through the weak reference. This means we can + // allow the Context to close by dropping the strong ref, but then + // still cancel ongoing IO if necessary. + if (mWeakRef) { + mWeakRef->Invalidate(); + } + // We should synchronously have AllowToCloseOnOwningThread called when + // the Context is canceled. + MOZ_DIAGNOSTIC_ASSERT(!mStrongRef); +} + +void Context::ThreadsafeHandle::ContextDestroyed(Context& aContext) { + MOZ_ASSERT(mOwningEventTarget->IsOnCurrentThread()); + MOZ_DIAGNOSTIC_ASSERT(!mStrongRef); + MOZ_DIAGNOSTIC_ASSERT(mWeakRef); + MOZ_DIAGNOSTIC_ASSERT(mWeakRef == &aContext); + mWeakRef = nullptr; +} + +// static +SafeRefPtr<Context> Context::Create(SafeRefPtr<Manager> aManager, + nsISerialEventTarget* aTarget, + SafeRefPtr<Action> aInitAction, + Maybe<Context&> aOldContext) { + auto context = MakeSafeRefPtr<Context>(std::move(aManager), aTarget, + std::move(aInitAction)); + context->Init(aOldContext); + return context; +} + +Context::Context(SafeRefPtr<Manager> aManager, nsISerialEventTarget* aTarget, + SafeRefPtr<Action> aInitAction) + : mManager(std::move(aManager)), + mTarget(aTarget), + mData(new Data(aTarget)), + mState(STATE_CONTEXT_PREINIT), + mOrphanedData(false), + mInitAction(std::move(aInitAction)) { + MOZ_DIAGNOSTIC_ASSERT(mManager); + MOZ_DIAGNOSTIC_ASSERT(mTarget); +} + +void Context::Dispatch(SafeRefPtr<Action> aAction) { + NS_ASSERT_OWNINGTHREAD(Context); + MOZ_DIAGNOSTIC_ASSERT(aAction); + MOZ_DIAGNOSTIC_ASSERT(mState != STATE_CONTEXT_CANCELED); + + if (mState == STATE_CONTEXT_CANCELED) { + return; + } + + if (mState == STATE_CONTEXT_INIT || mState == STATE_CONTEXT_PREINIT) { + PendingAction* pending = mPendingActions.AppendElement(); + pending->mAction = std::move(aAction); + return; + } + + MOZ_DIAGNOSTIC_ASSERT(mState == STATE_CONTEXT_READY); + DispatchAction(std::move(aAction)); +} + +Maybe<DirectoryLock&> Context::MaybeDirectoryLockRef() const { + NS_ASSERT_OWNINGTHREAD(Context); + + if (mState == STATE_CONTEXT_PREINIT) { + MOZ_DIAGNOSTIC_ASSERT(!mInitRunnable); + MOZ_DIAGNOSTIC_ASSERT(!mDirectoryLock); + + return Nothing(); + } + + if (mState == STATE_CONTEXT_INIT) { + MOZ_DIAGNOSTIC_ASSERT(!mDirectoryLock); + + return mInitRunnable->MaybeDirectoryLockRef(); + } + + return ToMaybeRef(mDirectoryLock.get()); +} + +void Context::CancelAll() { + NS_ASSERT_OWNINGTHREAD(Context); + + // In PREINIT state we have not dispatch the init action yet. Just + // forget it. + if (mState == STATE_CONTEXT_PREINIT) { + MOZ_DIAGNOSTIC_ASSERT(!mInitRunnable); + mInitAction = nullptr; + + // In INIT state we have dispatched the runnable, but not received the + // async completion yet. Cancel the runnable, but don't forget about it + // until we get OnQuotaInit() callback. + } else if (mState == STATE_CONTEXT_INIT) { + mInitRunnable->Cancel(); + } + + mState = STATE_CONTEXT_CANCELED; + mPendingActions.Clear(); + for (const auto& activity : mActivityList.ForwardRange()) { + activity->Cancel(); + } + AllowToClose(); +} + +bool Context::IsCanceled() const { + NS_ASSERT_OWNINGTHREAD(Context); + return mState == STATE_CONTEXT_CANCELED; +} + +void Context::Invalidate() { + NS_ASSERT_OWNINGTHREAD(Context); + mManager->NoteClosing(); + CancelAll(); +} + +void Context::AllowToClose() { + NS_ASSERT_OWNINGTHREAD(Context); + if (mThreadsafeHandle) { + mThreadsafeHandle->AllowToClose(); + } +} + +void Context::CancelForCacheId(CacheId aCacheId) { + NS_ASSERT_OWNINGTHREAD(Context); + + // Remove matching pending actions + mPendingActions.RemoveElementsBy([aCacheId](const auto& pendingAction) { + return pendingAction.mAction->MatchesCacheId(aCacheId); + }); + + // Cancel activities and let them remove themselves + for (const auto& activity : mActivityList.ForwardRange()) { + if (activity->MatchesCacheId(aCacheId)) { + activity->Cancel(); + } + } +} + +Context::~Context() { + NS_ASSERT_OWNINGTHREAD(Context); + MOZ_DIAGNOSTIC_ASSERT(mManager); + MOZ_DIAGNOSTIC_ASSERT(!mData); + + if (mThreadsafeHandle) { + mThreadsafeHandle->ContextDestroyed(*this); + } + + // Note, this may set the mOrphanedData flag. + mManager->RemoveContext(*this); + + if (mDirectoryMetadata && mDirectoryMetadata->mDir && !mOrphanedData) { + MOZ_ALWAYS_SUCCEEDS(DeleteMarkerFile(*mDirectoryMetadata)); + } + + if (mNextContext) { + mNextContext->Start(); + } +} + +void Context::Init(Maybe<Context&> aOldContext) { + NS_ASSERT_OWNINGTHREAD(Context); + + if (aOldContext) { + aOldContext->SetNextContext(SafeRefPtrFromThis()); + return; + } + + Start(); +} + +void Context::Start() { + NS_ASSERT_OWNINGTHREAD(Context); + + // Previous context closing delayed our start, but then we were canceled. + // In this case, just do nothing here. + if (mState == STATE_CONTEXT_CANCELED) { + MOZ_DIAGNOSTIC_ASSERT(!mInitRunnable); + MOZ_DIAGNOSTIC_ASSERT(!mInitAction); + // If we can't initialize the quota subsystem we will never be able to + // clear our shared data object via the target IO thread. Instead just + // clear it here to maintain the invariant that the shared data is + // cleared before Context destruction. + mData = nullptr; + return; + } + + MOZ_DIAGNOSTIC_ASSERT(mState == STATE_CONTEXT_PREINIT); + MOZ_DIAGNOSTIC_ASSERT(!mInitRunnable); + + mInitRunnable = + new QuotaInitRunnable(SafeRefPtrFromThis(), mManager.clonePtr(), mData, + mTarget, std::move(mInitAction)); + mState = STATE_CONTEXT_INIT; + + nsresult rv = mInitRunnable->Dispatch(); + if (NS_FAILED(rv)) { + // Shutdown must be delayed until all Contexts are destroyed. Shutdown + // must also prevent any new Contexts from being constructed. Crash + // for this invariant violation. + MOZ_CRASH("Failed to dispatch QuotaInitRunnable."); + } +} + +void Context::DispatchAction(SafeRefPtr<Action> aAction, bool aDoomData) { + NS_ASSERT_OWNINGTHREAD(Context); + + auto runnable = + MakeSafeRefPtr<ActionRunnable>(SafeRefPtrFromThis(), mData, mTarget, + std::move(aAction), mDirectoryMetadata); + + if (aDoomData) { + mData = nullptr; + } + + nsresult rv = runnable->Dispatch(); + if (NS_FAILED(rv)) { + // Shutdown must be delayed until all Contexts are destroyed. Crash + // for this invariant violation. + MOZ_CRASH("Failed to dispatch ActionRunnable to target thread."); + } + AddActivity(*runnable); +} + +void Context::OnQuotaInit( + nsresult aRv, const Maybe<CacheDirectoryMetadata>& aDirectoryMetadata, + already_AddRefed<DirectoryLock> aDirectoryLock) { + NS_ASSERT_OWNINGTHREAD(Context); + + MOZ_DIAGNOSTIC_ASSERT(mInitRunnable); + mInitRunnable = nullptr; + + if (aDirectoryMetadata) { + mDirectoryMetadata.emplace(*aDirectoryMetadata); + } + + // Always save the directory lock to ensure QuotaManager does not shutdown + // before the Context has gone away. + MOZ_DIAGNOSTIC_ASSERT(!mDirectoryLock); + mDirectoryLock = aDirectoryLock; + + // If we opening the context failed, but we were not explicitly canceled, + // still treat the entire context as canceled. We don't want to allow + // new actions to be dispatched. We also cannot leave the context in + // the INIT state after failing to open. + if (NS_FAILED(aRv)) { + mState = STATE_CONTEXT_CANCELED; + } + + if (mState == STATE_CONTEXT_CANCELED) { + for (uint32_t i = 0; i < mPendingActions.Length(); ++i) { + mPendingActions[i].mAction->CompleteOnInitiatingThread(aRv); + } + mPendingActions.Clear(); + mThreadsafeHandle->AllowToClose(); + // Context will destruct after return here and last ref is released. + return; + } + + MOZ_DIAGNOSTIC_ASSERT(mState == STATE_CONTEXT_INIT); + mState = STATE_CONTEXT_READY; + + for (uint32_t i = 0; i < mPendingActions.Length(); ++i) { + DispatchAction(std::move(mPendingActions[i].mAction)); + } + mPendingActions.Clear(); +} + +void Context::AddActivity(Activity& aActivity) { + NS_ASSERT_OWNINGTHREAD(Context); + MOZ_ASSERT(!mActivityList.Contains(&aActivity)); + mActivityList.AppendElement(WrapNotNullUnchecked(&aActivity)); +} + +void Context::RemoveActivity(Activity& aActivity) { + NS_ASSERT_OWNINGTHREAD(Context); + MOZ_ALWAYS_TRUE(mActivityList.RemoveElement(&aActivity)); + MOZ_ASSERT(!mActivityList.Contains(&aActivity)); +} + +void Context::NoteOrphanedData() { + NS_ASSERT_OWNINGTHREAD(Context); + // This may be called more than once + mOrphanedData = true; +} + +SafeRefPtr<Context::ThreadsafeHandle> Context::CreateThreadsafeHandle() { + NS_ASSERT_OWNINGTHREAD(Context); + if (!mThreadsafeHandle) { + mThreadsafeHandle = MakeSafeRefPtr<ThreadsafeHandle>(SafeRefPtrFromThis()); + } + return mThreadsafeHandle.clonePtr(); +} + +void Context::SetNextContext(SafeRefPtr<Context> aNextContext) { + NS_ASSERT_OWNINGTHREAD(Context); + MOZ_DIAGNOSTIC_ASSERT(aNextContext); + MOZ_DIAGNOSTIC_ASSERT(!mNextContext); + mNextContext = std::move(aNextContext); +} + +void Context::DoomTargetData() { + NS_ASSERT_OWNINGTHREAD(Context); + MOZ_DIAGNOSTIC_ASSERT(mData); + + // We are about to drop our reference to the Data. We need to ensure that + // the ~Context() destructor does not run until contents of Data have been + // released on the Target thread. + + // Dispatch a no-op Action. This will hold the Context alive through a + // roundtrip to the target thread and back to the owning thread. The + // ref to the Data object is cleared on the owning thread after creating + // the ActionRunnable, but before dispatching it. + DispatchAction(MakeSafeRefPtr<NullAction>(), true /* doomed data */); + + MOZ_DIAGNOSTIC_ASSERT(!mData); +} + +} // namespace mozilla::dom::cache diff --git a/dom/cache/Context.h b/dom/cache/Context.h new file mode 100644 index 0000000000..036f315b82 --- /dev/null +++ b/dom/cache/Context.h @@ -0,0 +1,224 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_Context_h +#define mozilla_dom_cache_Context_h + +#include "mozilla/dom/SafeRefPtr.h" +#include "mozilla/dom/cache/Types.h" +#include "nsCOMPtr.h" +#include "nsISupportsImpl.h" +#include "nsProxyRelease.h" +#include "nsString.h" +#include "nsTArray.h" +#include "nsTObserverArray.h" + +class nsIEventTarget; +class nsIThread; + +namespace mozilla::dom { + +namespace quota { + +class DirectoryLock; + +} // namespace quota + +namespace cache { + +class Action; +class Manager; + +// The Context class is RAII-style class for managing IO operations within the +// Cache. +// +// When a Context is created it performs the complicated steps necessary to +// initialize the QuotaManager. Action objects dispatched on the Context are +// delayed until this initialization is complete. They are then allow to +// execute on any specified thread. Once all references to the Context are +// gone, then the steps necessary to release the QuotaManager are performed. +// After initialization the Context holds a self reference, so it will stay +// alive until one of three conditions occur: +// +// 1) The Manager will call Context::AllowToClose() when all of the actors +// have removed themselves as listener. This means an idle context with +// no active DOM objects will close gracefully. +// 2) The QuotaManager aborts all operations so it can delete the files. +// In this case the QuotaManager calls Client::AbortOperationsForLocks() +// which in turn cancels all existing Action objects and then marks the +// Manager as invalid. +// 3) Browser shutdown occurs and the Manager calls Context::CancelAll(). +// +// In either case, though, the Action objects must be destroyed first to +// allow the Context to be destroyed. +// +// While the Context performs operations asynchronously on threads, all of +// methods in its public interface must be called on the same thread +// originally used to create the Context. +// +// As an invariant, all Context objects must be destroyed before permitting +// the "profile-before-change" shutdown event to complete. This is ensured +// via the code in ShutdownObserver.cpp. +class Context final : public SafeRefCounted<Context> { + using DirectoryLock = mozilla::dom::quota::DirectoryLock; + + public: + // Define a class allowing other threads to hold the Context alive. This also + // allows these other threads to safely close or cancel the Context. + class ThreadsafeHandle final : public AtomicSafeRefCounted<ThreadsafeHandle> { + friend class Context; + + public: + explicit ThreadsafeHandle(SafeRefPtr<Context> aContext); + ~ThreadsafeHandle(); + + MOZ_DECLARE_REFCOUNTED_TYPENAME(cache::Context::ThreadsafeHandle) + + void AllowToClose(); + void InvalidateAndAllowToClose(); + + private: + void AllowToCloseOnOwningThread(); + void InvalidateAndAllowToCloseOnOwningThread(); + + void ContextDestroyed(Context& aContext); + + // Cleared to allow the Context to close. Only safe to access on + // owning thread. + SafeRefPtr<Context> mStrongRef; + + // Used to support cancelation even while the Context is already allowed + // to close. Cleared by ~Context() calling ContextDestroyed(). Only + // safe to access on owning thread. + Context* mWeakRef; + + nsCOMPtr<nsISerialEventTarget> mOwningEventTarget; + }; + + // Different objects hold references to the Context while some work is being + // performed asynchronously. These objects must implement the Activity + // interface and register themselves with the AddActivity(). When they are + // destroyed they must call RemoveActivity(). This allows the Context to + // cancel any outstanding Activity work when the Context is cancelled. + class Activity { + public: + virtual void Cancel() = 0; + virtual bool MatchesCacheId(CacheId aCacheId) const = 0; + }; + + // Create a Context attached to the given Manager. The given Action + // will run on the QuotaManager IO thread. Note, this Action must + // be execute synchronously. + static SafeRefPtr<Context> Create(SafeRefPtr<Manager> aManager, + nsISerialEventTarget* aTarget, + SafeRefPtr<Action> aInitAction, + Maybe<Context&> aOldContext); + + // Execute given action on the target once the quota manager has been + // initialized. + // + // Only callable from the thread that created the Context. + void Dispatch(SafeRefPtr<Action> aAction); + + Maybe<DirectoryLock&> MaybeDirectoryLockRef() const; + + // Cancel any Actions running or waiting to run. This should allow the + // Context to be released and Listener::RemoveContext() will be called + // when complete. + // + // Only callable from the thread that created the Context. + void CancelAll(); + + // True if CancelAll() has been called. + bool IsCanceled() const; + + // Like CancelAll(), but also marks the Manager as "invalid". + void Invalidate(); + + // Remove any self references and allow the Context to be released when + // there are no more Actions to process. + void AllowToClose(); + + // Cancel any Actions running or waiting to run that operate on the given + // cache ID. + // + // Only callable from the thread that created the Context. + void CancelForCacheId(CacheId aCacheId); + + void AddActivity(Activity& aActivity); + void RemoveActivity(Activity& aActivity); + + // Tell the Context that some state information has been orphaned in the + // data store and won't be cleaned up. The Context will leave the marker + // in place to trigger cleanup the next times its opened. + void NoteOrphanedData(); + + private: + class Data; + class QuotaInitRunnable; + class ActionRunnable; + + enum State { + STATE_CONTEXT_PREINIT, + STATE_CONTEXT_INIT, + STATE_CONTEXT_READY, + STATE_CONTEXT_CANCELED + }; + + struct PendingAction { + nsCOMPtr<nsIEventTarget> mTarget; + SafeRefPtr<Action> mAction; + }; + + void Init(Maybe<Context&> aOldContext); + void Start(); + void DispatchAction(SafeRefPtr<Action> aAction, bool aDoomData = false); + void OnQuotaInit(nsresult aRv, + const Maybe<CacheDirectoryMetadata>& aDirectoryMetadata, + already_AddRefed<DirectoryLock> aDirectoryLock); + + SafeRefPtr<ThreadsafeHandle> CreateThreadsafeHandle(); + + void SetNextContext(SafeRefPtr<Context> aNextContext); + + void DoomTargetData(); + + SafeRefPtr<Manager> mManager; + nsCOMPtr<nsISerialEventTarget> mTarget; + RefPtr<Data> mData; + State mState; + bool mOrphanedData; + Maybe<CacheDirectoryMetadata> mDirectoryMetadata; + RefPtr<QuotaInitRunnable> mInitRunnable; + SafeRefPtr<Action> mInitAction; + nsTArray<PendingAction> mPendingActions; + + // Weak refs since activites must remove themselves from this list before + // being destroyed by calling RemoveActivity(). + nsTObserverArray<NotNull<Activity*>> mActivityList; + + // The ThreadsafeHandle may have a strong ref back to us. This creates + // a ref-cycle that keeps the Context alive. The ref-cycle is broken + // when ThreadsafeHandle::AllowToClose() is called. + SafeRefPtr<ThreadsafeHandle> mThreadsafeHandle; + + RefPtr<DirectoryLock> mDirectoryLock; + SafeRefPtr<Context> mNextContext; + + public: + // XXX Consider adding a private guard parameter. + Context(SafeRefPtr<Manager> aManager, nsISerialEventTarget* aTarget, + SafeRefPtr<Action> aInitAction); + ~Context(); + + NS_DECL_OWNINGTHREAD + MOZ_DECLARE_REFCOUNTED_TYPENAME(cache::Context) +}; + +} // namespace cache +} // namespace mozilla::dom + +#endif // mozilla_dom_cache_Context_h diff --git a/dom/cache/DBAction.cpp b/dom/cache/DBAction.cpp new file mode 100644 index 0000000000..0a51260fe5 --- /dev/null +++ b/dom/cache/DBAction.cpp @@ -0,0 +1,226 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/DBAction.h" + +#include "mozilla/dom/cache/Connection.h" +#include "mozilla/dom/cache/DBSchema.h" +#include "mozilla/dom/cache/FileUtils.h" +#include "mozilla/dom/cache/QuotaClient.h" +#include "mozilla/dom/quota/Assertions.h" +#include "mozilla/dom/quota/PersistenceType.h" +#include "mozilla/dom/quota/ResultExtensions.h" +#include "mozilla/net/nsFileProtocolHandler.h" +#include "mozIStorageConnection.h" +#include "mozIStorageService.h" +#include "mozStorageCID.h" +#include "nsIFile.h" +#include "nsIURI.h" +#include "nsIURIMutator.h" +#include "nsIFileURL.h" +#include "nsThreadUtils.h" + +namespace mozilla::dom::cache { + +using mozilla::dom::quota::AssertIsOnIOThread; +using mozilla::dom::quota::Client; +using mozilla::dom::quota::CloneFileAndAppend; +using mozilla::dom::quota::IsDatabaseCorruptionError; +using mozilla::dom::quota::PERSISTENCE_TYPE_DEFAULT; +using mozilla::dom::quota::PersistenceType; + +namespace { + +nsresult WipeDatabase(const CacheDirectoryMetadata& aDirectoryMetadata, + nsIFile& aDBFile) { + QM_TRY_INSPECT(const auto& dbDir, MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsCOMPtr<nsIFile>, aDBFile, GetParent)); + + QM_TRY(MOZ_TO_RESULT(RemoveNsIFile(aDirectoryMetadata, aDBFile))); + + // Note, the -wal journal file will be automatically deleted by sqlite when + // the new database is created. No need to explicitly delete it here. + + // Delete the morgue as well. + QM_TRY(MOZ_TO_RESULT(BodyDeleteDir(aDirectoryMetadata, *dbDir))); + + QM_TRY(MOZ_TO_RESULT(WipePaddingFile(aDirectoryMetadata, dbDir))); + + return NS_OK; +} + +} // namespace + +DBAction::DBAction(Mode aMode) : mMode(aMode) {} + +DBAction::~DBAction() = default; + +void DBAction::RunOnTarget( + SafeRefPtr<Resolver> aResolver, + const Maybe<CacheDirectoryMetadata>& aDirectoryMetadata, + Data* aOptionalData) { + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aResolver); + MOZ_DIAGNOSTIC_ASSERT(aDirectoryMetadata); + MOZ_DIAGNOSTIC_ASSERT(aDirectoryMetadata->mDir); + + if (IsCanceled()) { + aResolver->Resolve(NS_ERROR_ABORT); + return; + } + + const auto resolveErr = [&aResolver](const nsresult rv) { + aResolver->Resolve(rv); + }; + + QM_TRY_INSPECT(const auto& dbDir, + CloneFileAndAppend(*(aDirectoryMetadata->mDir), u"cache"_ns), + QM_VOID, resolveErr); + + nsCOMPtr<mozIStorageConnection> conn; + + // Attempt to reuse the connection opened by a previous Action. + if (aOptionalData) { + conn = aOptionalData->GetConnection(); + } + + // If there is no previous Action, then we must open one. + if (!conn) { + QM_TRY_UNWRAP(conn, OpenConnection(*aDirectoryMetadata, *dbDir), QM_VOID, + resolveErr); + MOZ_DIAGNOSTIC_ASSERT(conn); + + // Save this connection in the shared Data object so later Actions can + // use it. This avoids opening a new connection for every Action. + if (aOptionalData) { + // Since we know this connection will be around for as long as the + // Cache is open, use our special wrapped connection class. This + // will let us perform certain operations once the Cache origin + // is closed. + nsCOMPtr<mozIStorageConnection> wrapped = new Connection(conn); + aOptionalData->SetConnection(wrapped); + } + } + + RunWithDBOnTarget(std::move(aResolver), *aDirectoryMetadata, dbDir, conn); +} + +Result<nsCOMPtr<mozIStorageConnection>, nsresult> DBAction::OpenConnection( + const CacheDirectoryMetadata& aDirectoryMetadata, nsIFile& aDBDir) { + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aDirectoryMetadata.mDirectoryLockId >= 0); + + QM_TRY_INSPECT(const bool& exists, + MOZ_TO_RESULT_INVOKE_MEMBER(aDBDir, Exists)); + + if (!exists) { + QM_TRY(OkIf(mMode == Create), Err(NS_ERROR_FILE_NOT_FOUND)); + QM_TRY(MOZ_TO_RESULT(aDBDir.Create(nsIFile::DIRECTORY_TYPE, 0755))); + } + + QM_TRY_INSPECT(const auto& dbFile, + CloneFileAndAppend(aDBDir, kCachesSQLiteFilename)); + + QM_TRY_RETURN(OpenDBConnection(aDirectoryMetadata, *dbFile)); +} + +SyncDBAction::SyncDBAction(Mode aMode) : DBAction(aMode) {} + +SyncDBAction::~SyncDBAction() = default; + +void SyncDBAction::RunWithDBOnTarget( + SafeRefPtr<Resolver> aResolver, + const CacheDirectoryMetadata& aDirectoryMetadata, nsIFile* aDBDir, + mozIStorageConnection* aConn) { + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aResolver); + MOZ_DIAGNOSTIC_ASSERT(aDBDir); + MOZ_DIAGNOSTIC_ASSERT(aConn); + + nsresult rv = RunSyncWithDBOnTarget(aDirectoryMetadata, aDBDir, aConn); + aResolver->Resolve(rv); +} + +Result<nsCOMPtr<mozIStorageConnection>, nsresult> OpenDBConnection( + const CacheDirectoryMetadata& aDirectoryMetadata, nsIFile& aDBFile) { + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aDirectoryMetadata.mDirectoryLockId >= -1); + + // Use our default file:// protocol handler directly to construct the database + // URL. This avoids any problems if a plugin registers a custom file:// + // handler. If such a custom handler used javascript, then we would have a + // bad time running off the main thread here. + auto handler = MakeRefPtr<nsFileProtocolHandler>(); + QM_TRY(MOZ_TO_RESULT(handler->Init())); + + QM_TRY_INSPECT(const auto& mutator, MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsCOMPtr<nsIURIMutator>, handler, + NewFileURIMutator, &aDBFile)); + + const nsCString directoryLockIdClause = + aDirectoryMetadata.mDirectoryLockId >= 0 + ? "&directoryLockId="_ns + + IntToCString(aDirectoryMetadata.mDirectoryLockId) + : EmptyCString(); + + nsCOMPtr<nsIFileURL> dbFileUrl; + QM_TRY(MOZ_TO_RESULT(NS_MutateURI(mutator) + .SetQuery("cache=private"_ns + directoryLockIdClause) + .Finalize(dbFileUrl))); + + QM_TRY_INSPECT(const auto& storageService, + MOZ_TO_RESULT_GET_TYPED(nsCOMPtr<mozIStorageService>, + MOZ_SELECT_OVERLOAD(do_GetService), + MOZ_STORAGE_SERVICE_CONTRACTID), + Err(NS_ERROR_UNEXPECTED)); + + QM_TRY_UNWRAP( + auto conn, + QM_OR_ELSE_WARN_IF( + // Expression. + MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsCOMPtr<mozIStorageConnection>, storageService, + OpenDatabaseWithFileURL, dbFileUrl, ""_ns, + mozIStorageService::CONNECTION_DEFAULT), + // Predicate. + IsDatabaseCorruptionError, + // Fallback. + ([&aDirectoryMetadata, &aDBFile, &storageService, + &dbFileUrl](const nsresult rv) + -> Result<nsCOMPtr<mozIStorageConnection>, nsresult> { + NS_WARNING("Cache database corrupted. Recreating empty database."); + + // There is nothing else we can do to recover. Also, this data + // can be deleted by QuotaManager at any time anyways. + QM_TRY(MOZ_TO_RESULT(WipeDatabase(aDirectoryMetadata, aDBFile))); + + QM_TRY_RETURN(MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsCOMPtr<mozIStorageConnection>, storageService, + OpenDatabaseWithFileURL, dbFileUrl, ""_ns, + mozIStorageService::CONNECTION_DEFAULT)); + }))); + + // Check the schema to make sure it is not too old. + QM_TRY_INSPECT(const int32_t& schemaVersion, + MOZ_TO_RESULT_INVOKE_MEMBER(conn, GetSchemaVersion)); + if (schemaVersion > 0 && schemaVersion < db::kFirstShippedSchemaVersion) { + // Close existing connection before wiping database. + conn = nullptr; + + QM_TRY(MOZ_TO_RESULT(WipeDatabase(aDirectoryMetadata, aDBFile))); + + QM_TRY_UNWRAP(conn, MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsCOMPtr<mozIStorageConnection>, storageService, + OpenDatabaseWithFileURL, dbFileUrl, ""_ns, + mozIStorageService::CONNECTION_DEFAULT)); + } + + QM_TRY(MOZ_TO_RESULT(db::InitializeConnection(*conn))); + + return conn; +} + +} // namespace mozilla::dom::cache diff --git a/dom/cache/DBAction.h b/dom/cache/DBAction.h new file mode 100644 index 0000000000..27b1c489cc --- /dev/null +++ b/dom/cache/DBAction.h @@ -0,0 +1,72 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_DBAction_h +#define mozilla_dom_cache_DBAction_h + +#include "mozilla/dom/cache/Action.h" +#include "mozilla/RefPtr.h" +#include "nsString.h" + +class mozIStorageConnection; +class nsIFile; + +namespace mozilla::dom::cache { + +Result<nsCOMPtr<mozIStorageConnection>, nsresult> OpenDBConnection( + const CacheDirectoryMetadata& aDirectoryMetadata, nsIFile& aDBFile); + +class DBAction : public Action { + protected: + // The mode specifies whether the database should already exist or if its + // ok to create a new database. + enum Mode { Existing, Create }; + + explicit DBAction(Mode aMode); + + // Action objects are deleted through their base pointer + virtual ~DBAction(); + + // Just as the resolver must be ref'd until resolve, you may also + // ref the DB connection. The connection can only be referenced from the + // target thread and must be released upon resolve. + virtual void RunWithDBOnTarget( + SafeRefPtr<Resolver> aResolver, + const CacheDirectoryMetadata& aDirectoryMetadata, nsIFile* aDBDir, + mozIStorageConnection* aConn) = 0; + + private: + void RunOnTarget(SafeRefPtr<Resolver> aResolver, + const Maybe<CacheDirectoryMetadata>& aDirectoryMetadata, + Data* aOptionalData) override; + + Result<nsCOMPtr<mozIStorageConnection>, nsresult> OpenConnection( + const CacheDirectoryMetadata& aDirectoryMetadata, nsIFile& aDBDir); + + const Mode mMode; +}; + +class SyncDBAction : public DBAction { + protected: + explicit SyncDBAction(Mode aMode); + + // Action objects are deleted through their base pointer + virtual ~SyncDBAction(); + + virtual nsresult RunSyncWithDBOnTarget( + const CacheDirectoryMetadata& aDirectoryMetadata, nsIFile* aDBDir, + mozIStorageConnection* aConn) = 0; + + private: + virtual void RunWithDBOnTarget( + SafeRefPtr<Resolver> aResolver, + const CacheDirectoryMetadata& aDirectoryMetadata, nsIFile* aDBDir, + mozIStorageConnection* aConn) override; +}; + +} // namespace mozilla::dom::cache + +#endif // mozilla_dom_cache_DBAction_h diff --git a/dom/cache/DBSchema.cpp b/dom/cache/DBSchema.cpp new file mode 100644 index 0000000000..5ee1e6908c --- /dev/null +++ b/dom/cache/DBSchema.cpp @@ -0,0 +1,2988 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/DBSchema.h" + +#include "ipc/IPCMessageUtils.h" +#include "mozIStorageConnection.h" +#include "mozIStorageStatement.h" +#include "mozStorageHelper.h" +#include "mozilla/BasePrincipal.h" +#include "mozilla/ResultExtensions.h" +#include "mozilla/StaticPrefs_extensions.h" +#include "mozilla/Telemetry.h" +#include "mozilla/dom/HeadersBinding.h" +#include "mozilla/dom/InternalHeaders.h" +#include "mozilla/dom/InternalResponse.h" +#include "mozilla/dom/RequestBinding.h" +#include "mozilla/dom/ResponseBinding.h" +#include "mozilla/dom/cache/CacheCommon.h" +#include "mozilla/dom/cache/CacheTypes.h" +#include "mozilla/dom/cache/SavedTypes.h" +#include "mozilla/dom/cache/TypeUtils.h" +#include "mozilla/dom/cache/Types.h" +#include "mozilla/dom/quota/ResultExtensions.h" +#include "mozilla/net/MozURL.h" +#include "mozilla/psm/TransportSecurityInfo.h" +#include "nsCOMPtr.h" +#include "nsCharSeparatedTokenizer.h" +#include "nsComponentManagerUtils.h" +#include "nsHttp.h" +#include "nsIContentPolicy.h" +#include "nsICryptoHash.h" +#include "nsNetCID.h" +#include "nsPrintfCString.h" +#include "nsTArray.h" + +namespace mozilla::dom::cache::db { +const int32_t kFirstShippedSchemaVersion = 15; +namespace { +// ## Firefox 57 Cache API v25/v26/v27 Schema Hack Info +// ### Overview +// In Firefox 57 we introduced Cache API schema version 26 and Quota Manager +// schema v3 to support tracking padding for opaque responses. Unfortunately, +// Firefox 57 is a big release that may potentially result in users downgrading +// to Firefox 56 due to 57 retiring add-ons. These schema changes have the +// unfortunate side-effect of causing QuotaManager and all its clients to break +// if the user downgrades to 56. In order to avoid making a bad situation +// worse, we're now retrofitting 57 so that Firefox 56 won't freak out. +// +// ### Implementation +// We're introducing a new schema version 27 that uses an on-disk schema version +// of v25. We differentiate v25 from v27 by the presence of the column added +// by v26. This translates to: +// - v25: on-disk schema=25, no "response_padding_size" column in table +// "entries". +// - v26: on-disk schema=26, yes "response_padding_size" column in table +// "entries". +// - v27: on-disk schema=25, yes "response_padding_size" column in table +// "entries". +// +// ### Fallout +// Firefox 57 is happy because it sees schema 27 and everything is as it +// expects. +// +// Firefox 56 non-DEBUG build is fine/happy, but DEBUG builds will not be. +// - Our QuotaClient will invoke `NS_WARNING("Unknown Cache file found!");` +// at QuotaManager init time. This is harmless but annoying and potentially +// misleading. +// - The DEBUG-only Validate() call will error out whenever an attempt is made +// to open a DOM Cache database because it will notice the schema is broken +// and there is no attempt at recovery. +// +const int32_t kHackyDowngradeSchemaVersion = 25; +const int32_t kHackyPaddingSizePresentVersion = 27; +// +// Update this whenever the DB schema is changed. +const int32_t kLatestSchemaVersion = 28; +// --------- +// The following constants define the SQL schema. These are defined in the +// same order the SQL should be executed in CreateOrMigrateSchema(). They are +// broken out as constants for convenient use in validation and migration. +// --------- +// The caches table is the single source of truth about what Cache +// objects exist for the origin. The contents of the Cache are stored +// in the entries table that references back to caches. +// +// The caches table is also referenced from storage. Rows in storage +// represent named Cache objects. There are cases, however, where +// a Cache can still exist, but not be in a named Storage. For example, +// when content is still using the Cache after CacheStorage::Delete() +// has been run. +// +// For now, the caches table mainly exists for data integrity with +// foreign keys, but could be expanded to contain additional cache object +// information. +// +// AUTOINCREMENT is necessary to prevent CacheId values from being reused. +const char kTableCaches[] = + "CREATE TABLE caches (" + "id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT " + ")"; + +// Security blobs are quite large and duplicated for every Response from +// the same https origin. This table is used to de-duplicate this data. +const char kTableSecurityInfo[] = + "CREATE TABLE security_info (" + "id INTEGER NOT NULL PRIMARY KEY, " + "hash BLOB NOT NULL, " // first 8-bytes of the sha1 hash of data column + "data BLOB NOT NULL, " // full security info data, usually a few KB + "refcount INTEGER NOT NULL" + ")"; + +// Index the smaller hash value instead of the large security data blob. +const char kIndexSecurityInfoHash[] = + "CREATE INDEX security_info_hash_index ON security_info (hash)"; + +const char kTableEntries[] = + "CREATE TABLE entries (" + "id INTEGER NOT NULL PRIMARY KEY, " + "request_method TEXT NOT NULL, " + "request_url_no_query TEXT NOT NULL, " + "request_url_no_query_hash BLOB NOT NULL, " // first 8-bytes of sha1 hash + "request_url_query TEXT NOT NULL, " + "request_url_query_hash BLOB NOT NULL, " // first 8-bytes of sha1 hash + "request_referrer TEXT NOT NULL, " + "request_headers_guard INTEGER NOT NULL, " + "request_mode INTEGER NOT NULL, " + "request_credentials INTEGER NOT NULL, " + "request_contentpolicytype INTEGER NOT NULL, " + "request_cache INTEGER NOT NULL, " + "request_body_id TEXT NULL, " + "response_type INTEGER NOT NULL, " + "response_status INTEGER NOT NULL, " + "response_status_text TEXT NOT NULL, " + "response_headers_guard INTEGER NOT NULL, " + "response_body_id TEXT NULL, " + "response_security_info_id INTEGER NULL REFERENCES security_info(id), " + "response_principal_info TEXT NOT NULL, " + "cache_id INTEGER NOT NULL REFERENCES caches(id) ON DELETE CASCADE, " + "request_redirect INTEGER NOT NULL, " + "request_referrer_policy INTEGER NOT NULL, " + "request_integrity TEXT NOT NULL, " + "request_url_fragment TEXT NOT NULL, " + "response_padding_size INTEGER NULL " + // New columns must be added at the end of table to migrate and + // validate properly. + ")"; +// Create an index to support the QueryCache() matching algorithm. This +// needs to quickly find entries in a given Cache that match the request +// URL. The url query is separated in order to support the ignoreSearch +// option. Finally, we index hashes of the URL values instead of the +// actual strings to avoid excessive disk bloat. The index will duplicate +// the contents of the columsn in the index. The hash index will prune +// the vast majority of values from the query result so that normal +// scanning only has to be done on a few values to find an exact URL match. +const char kIndexEntriesRequest[] = + "CREATE INDEX entries_request_match_index " + "ON entries (cache_id, request_url_no_query_hash, " + "request_url_query_hash)"; + +const char kTableRequestHeaders[] = + "CREATE TABLE request_headers (" + "name TEXT NOT NULL, " + "value TEXT NOT NULL, " + "entry_id INTEGER NOT NULL REFERENCES entries(id) ON DELETE CASCADE" + ")"; + +const char kTableResponseHeaders[] = + "CREATE TABLE response_headers (" + "name TEXT NOT NULL, " + "value TEXT NOT NULL, " + "entry_id INTEGER NOT NULL REFERENCES entries(id) ON DELETE CASCADE" + ")"; + +// We need an index on response_headers, but not on request_headers, +// because we quickly need to determine if a VARY header is present. +const char kIndexResponseHeadersName[] = + "CREATE INDEX response_headers_name_index " + "ON response_headers (name)"; + +const char kTableResponseUrlList[] = + "CREATE TABLE response_url_list (" + "url TEXT NOT NULL, " + "entry_id INTEGER NOT NULL REFERENCES entries(id) ON DELETE CASCADE" + ")"; + +// NOTE: key allows NULL below since that is how "" is represented +// in a BLOB column. We use BLOB to avoid encoding issues +// with storing DOMStrings. +const char kTableStorage[] = + "CREATE TABLE storage (" + "namespace INTEGER NOT NULL, " + "key BLOB NULL, " + "cache_id INTEGER NOT NULL REFERENCES caches(id), " + "PRIMARY KEY(namespace, key) " + ")"; + +// --------- +// End schema definition +// --------- + +const uint32_t kMaxEntriesPerStatement = 255; + +const uint32_t kPageSize = 4 * 1024; + +// Grow the database in chunks to reduce fragmentation +const uint32_t kGrowthSize = 32 * 1024; +const uint32_t kGrowthPages = kGrowthSize / kPageSize; +static_assert(kGrowthSize % kPageSize == 0, + "Growth size must be multiple of page size"); + +// Only release free pages when we have more than this limit +const int32_t kMaxFreePages = kGrowthPages; + +// Limit WAL journal to a reasonable size +const uint32_t kWalAutoCheckpointSize = 512 * 1024; +const uint32_t kWalAutoCheckpointPages = kWalAutoCheckpointSize / kPageSize; +static_assert(kWalAutoCheckpointSize % kPageSize == 0, + "WAL checkpoint size must be multiple of page size"); + +} // namespace + +// If any of the static_asserts below fail, it means that you have changed +// the corresponding WebIDL enum in a way that may be incompatible with the +// existing data stored in the DOM Cache. You would need to update the Cache +// database schema accordingly and adjust the failing static_assert. +static_assert(int(HeadersGuardEnum::None) == 0 && + int(HeadersGuardEnum::Request) == 1 && + int(HeadersGuardEnum::Request_no_cors) == 2 && + int(HeadersGuardEnum::Response) == 3 && + int(HeadersGuardEnum::Immutable) == 4 && + HeadersGuardEnumValues::Count == 5, + "HeadersGuardEnum values are as expected"); +static_assert(int(ReferrerPolicy::_empty) == 0 && + int(ReferrerPolicy::No_referrer) == 1 && + int(ReferrerPolicy::No_referrer_when_downgrade) == 2 && + int(ReferrerPolicy::Origin) == 3 && + int(ReferrerPolicy::Origin_when_cross_origin) == 4 && + int(ReferrerPolicy::Unsafe_url) == 5 && + int(ReferrerPolicy::Same_origin) == 6 && + int(ReferrerPolicy::Strict_origin) == 7 && + int(ReferrerPolicy::Strict_origin_when_cross_origin) == 8 && + ReferrerPolicyValues::Count == 9, + "ReferrerPolicy values are as expected"); +static_assert(int(RequestMode::Same_origin) == 0 && + int(RequestMode::No_cors) == 1 && + int(RequestMode::Cors) == 2 && + int(RequestMode::Navigate) == 3 && + RequestModeValues::Count == 4, + "RequestMode values are as expected"); +static_assert(int(RequestCredentials::Omit) == 0 && + int(RequestCredentials::Same_origin) == 1 && + int(RequestCredentials::Include) == 2 && + RequestCredentialsValues::Count == 3, + "RequestCredentials values are as expected"); +static_assert(int(RequestCache::Default) == 0 && + int(RequestCache::No_store) == 1 && + int(RequestCache::Reload) == 2 && + int(RequestCache::No_cache) == 3 && + int(RequestCache::Force_cache) == 4 && + int(RequestCache::Only_if_cached) == 5 && + RequestCacheValues::Count == 6, + "RequestCache values are as expected"); +static_assert(int(RequestRedirect::Follow) == 0 && + int(RequestRedirect::Error) == 1 && + int(RequestRedirect::Manual) == 2 && + RequestRedirectValues::Count == 3, + "RequestRedirect values are as expected"); +static_assert(int(ResponseType::Basic) == 0 && int(ResponseType::Cors) == 1 && + int(ResponseType::Default) == 2 && + int(ResponseType::Error) == 3 && + int(ResponseType::Opaque) == 4 && + int(ResponseType::Opaqueredirect) == 5 && + ResponseTypeValues::Count == 6, + "ResponseType values are as expected"); + +// If the static_asserts below fails, it means that you have changed the +// Namespace enum in a way that may be incompatible with the existing data +// stored in the DOM Cache. You would need to update the Cache database schema +// accordingly and adjust the failing static_assert. +static_assert(DEFAULT_NAMESPACE == 0 && CHROME_ONLY_NAMESPACE == 1 && + NUMBER_OF_NAMESPACES == 2, + "Namespace values are as expected"); + +// If the static_asserts below fails, it means that you have changed the +// nsContentPolicy enum in a way that may be incompatible with the existing data +// stored in the DOM Cache. You would need to update the Cache database schema +// accordingly and adjust the failing static_assert. +static_assert( + nsIContentPolicy::TYPE_INVALID == 0 && nsIContentPolicy::TYPE_OTHER == 1 && + nsIContentPolicy::TYPE_SCRIPT == 2 && + nsIContentPolicy::TYPE_IMAGE == 3 && + nsIContentPolicy::TYPE_STYLESHEET == 4 && + nsIContentPolicy::TYPE_OBJECT == 5 && + nsIContentPolicy::TYPE_DOCUMENT == 6 && + nsIContentPolicy::TYPE_SUBDOCUMENT == 7 && + nsIContentPolicy::TYPE_PING == 10 && + nsIContentPolicy::TYPE_XMLHTTPREQUEST == 11 && + nsIContentPolicy::TYPE_OBJECT_SUBREQUEST == 12 && + nsIContentPolicy::TYPE_DTD == 13 && nsIContentPolicy::TYPE_FONT == 14 && + nsIContentPolicy::TYPE_MEDIA == 15 && + nsIContentPolicy::TYPE_WEBSOCKET == 16 && + nsIContentPolicy::TYPE_CSP_REPORT == 17 && + nsIContentPolicy::TYPE_XSLT == 18 && + nsIContentPolicy::TYPE_BEACON == 19 && + nsIContentPolicy::TYPE_FETCH == 20 && + nsIContentPolicy::TYPE_IMAGESET == 21 && + nsIContentPolicy::TYPE_WEB_MANIFEST == 22 && + nsIContentPolicy::TYPE_INTERNAL_SCRIPT == 23 && + nsIContentPolicy::TYPE_INTERNAL_WORKER == 24 && + nsIContentPolicy::TYPE_INTERNAL_SHARED_WORKER == 25 && + nsIContentPolicy::TYPE_INTERNAL_EMBED == 26 && + nsIContentPolicy::TYPE_INTERNAL_OBJECT == 27 && + nsIContentPolicy::TYPE_INTERNAL_FRAME == 28 && + nsIContentPolicy::TYPE_INTERNAL_IFRAME == 29 && + nsIContentPolicy::TYPE_INTERNAL_AUDIO == 30 && + nsIContentPolicy::TYPE_INTERNAL_VIDEO == 31 && + nsIContentPolicy::TYPE_INTERNAL_TRACK == 32 && + nsIContentPolicy::TYPE_INTERNAL_XMLHTTPREQUEST == 33 && + nsIContentPolicy::TYPE_INTERNAL_EVENTSOURCE == 34 && + nsIContentPolicy::TYPE_INTERNAL_SERVICE_WORKER == 35 && + nsIContentPolicy::TYPE_INTERNAL_SCRIPT_PRELOAD == 36 && + nsIContentPolicy::TYPE_INTERNAL_IMAGE == 37 && + nsIContentPolicy::TYPE_INTERNAL_IMAGE_PRELOAD == 38 && + nsIContentPolicy::TYPE_INTERNAL_STYLESHEET == 39 && + nsIContentPolicy::TYPE_INTERNAL_STYLESHEET_PRELOAD == 40 && + nsIContentPolicy::TYPE_INTERNAL_IMAGE_FAVICON == 41 && + nsIContentPolicy::TYPE_INTERNAL_WORKER_IMPORT_SCRIPTS == 42 && + nsIContentPolicy::TYPE_SAVEAS_DOWNLOAD == 43 && + nsIContentPolicy::TYPE_SPECULATIVE == 44 && + nsIContentPolicy::TYPE_INTERNAL_MODULE == 45 && + nsIContentPolicy::TYPE_INTERNAL_MODULE_PRELOAD == 46 && + nsIContentPolicy::TYPE_INTERNAL_DTD == 47 && + nsIContentPolicy::TYPE_INTERNAL_FORCE_ALLOWED_DTD == 48 && + nsIContentPolicy::TYPE_INTERNAL_AUDIOWORKLET == 49 && + nsIContentPolicy::TYPE_INTERNAL_PAINTWORKLET == 50 && + nsIContentPolicy::TYPE_INTERNAL_FONT_PRELOAD == 51 && + nsIContentPolicy::TYPE_INTERNAL_CHROMEUTILS_COMPILED_SCRIPT == 52 && + nsIContentPolicy::TYPE_INTERNAL_FRAME_MESSAGEMANAGER_SCRIPT == 53 && + nsIContentPolicy::TYPE_INTERNAL_FETCH_PRELOAD == 54 && + nsIContentPolicy::TYPE_UA_FONT == 55 && + nsIContentPolicy::TYPE_WEB_IDENTITY == 57 && + nsIContentPolicy::TYPE_INTERNAL_WORKER_STATIC_MODULE == 58 && + nsIContentPolicy::TYPE_WEB_TRANSPORT == 59 && + nsIContentPolicy::TYPE_END == 60, + "nsContentPolicyType values are as expected"); + +namespace { + +using EntryId = int32_t; + +struct IdCount { + explicit IdCount(int32_t aId) : mId(aId), mCount(1) {} + int32_t mId; + int32_t mCount; +}; + +using EntryIds = AutoTArray<EntryId, 256>; + +static Result<EntryIds, nsresult> QueryAll(mozIStorageConnection& aConn, + CacheId aCacheId); +static Result<EntryIds, nsresult> QueryCache(mozIStorageConnection& aConn, + CacheId aCacheId, + const CacheRequest& aRequest, + const CacheQueryParams& aParams, + uint32_t aMaxResults = UINT32_MAX); +static Result<bool, nsresult> MatchByVaryHeader(mozIStorageConnection& aConn, + const CacheRequest& aRequest, + EntryId entryId); +// Returns a success tuple containing the deleted body ids, deleted security ids +// and deleted padding size. +static Result<std::tuple<nsTArray<nsID>, AutoTArray<IdCount, 16>, int64_t>, + nsresult> +DeleteEntries(mozIStorageConnection& aConn, + const nsTArray<EntryId>& aEntryIdList); + +static Result<std::tuple<nsTArray<nsID>, AutoTArray<IdCount, 16>, int64_t>, + nsresult> +DeleteAllCacheEntries(mozIStorageConnection& aConn, CacheId& aCacheId); + +static Result<int32_t, nsresult> InsertSecurityInfo( + mozIStorageConnection& aConn, nsICryptoHash& aCrypto, + nsITransportSecurityInfo* aSecurityInfo); +static nsresult DeleteSecurityInfo(mozIStorageConnection& aConn, int32_t aId, + int32_t aCount); +static nsresult DeleteSecurityInfoList( + mozIStorageConnection& aConn, + const nsTArray<IdCount>& aDeletedStorageIdList); +static nsresult InsertEntry(mozIStorageConnection& aConn, CacheId aCacheId, + const CacheRequest& aRequest, + const nsID* aRequestBodyId, + const CacheResponse& aResponse, + const nsID* aResponseBodyId); +static Result<SavedResponse, nsresult> ReadResponse( + mozIStorageConnection& aConn, EntryId aEntryId); +static Result<SavedRequest, nsresult> ReadRequest(mozIStorageConnection& aConn, + EntryId aEntryId); + +static void AppendListParamsToQuery(nsACString& aQuery, size_t aLen); +static nsresult BindListParamsToQuery(mozIStorageStatement& aState, + const Span<const EntryId>& aEntryIdList); +static nsresult BindId(mozIStorageStatement& aState, const nsACString& aName, + const nsID* aId); +static Result<nsID, nsresult> ExtractId(mozIStorageStatement& aState, + uint32_t aPos); +static Result<NotNull<nsCOMPtr<mozIStorageStatement>>, nsresult> +CreateAndBindKeyStatement(mozIStorageConnection& aConn, + const char* aQueryFormat, const nsAString& aKey); +static Result<nsAutoCString, nsresult> HashCString(nsICryptoHash& aCrypto, + const nsACString& aIn); +Result<int32_t, nsresult> GetEffectiveSchemaVersion( + mozIStorageConnection& aConn); +nsresult Validate(mozIStorageConnection& aConn); +nsresult Migrate(mozIStorageConnection& aConn); +} // namespace + +class MOZ_RAII AutoDisableForeignKeyChecking { + public: + explicit AutoDisableForeignKeyChecking(mozIStorageConnection* aConn) + : mConn(aConn), mForeignKeyCheckingDisabled(false) { + QM_TRY_INSPECT(const auto& state, + quota::CreateAndExecuteSingleStepStatement( + *mConn, "PRAGMA foreign_keys;"_ns), + QM_VOID); + + QM_TRY_INSPECT(const int32_t& mode, + MOZ_TO_RESULT_INVOKE_MEMBER(*state, GetInt32, 0), QM_VOID); + + if (mode) { + QM_WARNONLY_TRY(MOZ_TO_RESULT(mConn->ExecuteSimpleSQL( + "PRAGMA foreign_keys = OFF;"_ns)) + .andThen([this](const auto) -> Result<Ok, nsresult> { + mForeignKeyCheckingDisabled = true; + return Ok{}; + })); + } + } + + ~AutoDisableForeignKeyChecking() { + if (mForeignKeyCheckingDisabled) { + QM_WARNONLY_TRY(QM_TO_RESULT( + mConn->ExecuteSimpleSQL("PRAGMA foreign_keys = ON;"_ns))); + } + } + + private: + nsCOMPtr<mozIStorageConnection> mConn; + bool mForeignKeyCheckingDisabled; +}; + +nsresult IntegrityCheck(mozIStorageConnection& aConn) { + // CACHE_INTEGRITY_CHECK_COUNT is designed to report at most once. + static bool reported = false; + if (reported) { + return NS_OK; + } + + QM_TRY_INSPECT(const auto& stmt, + quota::CreateAndExecuteSingleStepStatement( + aConn, + "SELECT COUNT(*) FROM pragma_integrity_check() " + "WHERE integrity_check != 'ok';"_ns)); + + QM_TRY_INSPECT(const auto& result, MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsString, *stmt, GetString, 0)); + + nsresult rv; + const uint32_t count = result.ToInteger(&rv); + QM_TRY(OkIf(NS_SUCCEEDED(rv)), rv); + + Telemetry::ScalarSet(Telemetry::ScalarID::CACHE_INTEGRITY_CHECK_COUNT, count); + + reported = true; + + return NS_OK; +} + +nsresult CreateOrMigrateSchema(mozIStorageConnection& aConn) { + MOZ_ASSERT(!NS_IsMainThread()); + + QM_TRY_UNWRAP(int32_t schemaVersion, GetEffectiveSchemaVersion(aConn)); + + if (schemaVersion == kLatestSchemaVersion) { + // We already have the correct schema version. Validate it matches + // our expected schema and then proceed. + QM_TRY(MOZ_TO_RESULT(Validate(aConn))); + + return NS_OK; + } + + // Turn off checking foreign keys before starting a transaction, and restore + // it once we're done. + AutoDisableForeignKeyChecking restoreForeignKeyChecking(&aConn); + mozStorageTransaction trans(&aConn, false, + mozIStorageConnection::TRANSACTION_IMMEDIATE); + + QM_TRY(MOZ_TO_RESULT(trans.Start())); + + const bool migrating = schemaVersion != 0; + + if (migrating) { + // A schema exists, but its not the current version. Attempt to + // migrate it to our new schema. + QM_TRY(MOZ_TO_RESULT(Migrate(aConn))); + } else { + // There is no schema installed. Create the database from scratch. + QM_TRY( + MOZ_TO_RESULT(aConn.ExecuteSimpleSQL(nsLiteralCString(kTableCaches)))); + QM_TRY(MOZ_TO_RESULT( + aConn.ExecuteSimpleSQL(nsLiteralCString(kTableSecurityInfo)))); + QM_TRY(MOZ_TO_RESULT( + aConn.ExecuteSimpleSQL(nsLiteralCString(kIndexSecurityInfoHash)))); + QM_TRY( + MOZ_TO_RESULT(aConn.ExecuteSimpleSQL(nsLiteralCString(kTableEntries)))); + QM_TRY(MOZ_TO_RESULT( + aConn.ExecuteSimpleSQL(nsLiteralCString(kIndexEntriesRequest)))); + QM_TRY(MOZ_TO_RESULT( + aConn.ExecuteSimpleSQL(nsLiteralCString(kTableRequestHeaders)))); + QM_TRY(MOZ_TO_RESULT( + aConn.ExecuteSimpleSQL(nsLiteralCString(kTableResponseHeaders)))); + QM_TRY(MOZ_TO_RESULT( + aConn.ExecuteSimpleSQL(nsLiteralCString(kIndexResponseHeadersName)))); + QM_TRY(MOZ_TO_RESULT( + aConn.ExecuteSimpleSQL(nsLiteralCString(kTableResponseUrlList)))); + QM_TRY( + MOZ_TO_RESULT(aConn.ExecuteSimpleSQL(nsLiteralCString(kTableStorage)))); + QM_TRY(MOZ_TO_RESULT(aConn.SetSchemaVersion(kLatestSchemaVersion))); + QM_TRY_UNWRAP(schemaVersion, GetEffectiveSchemaVersion(aConn)); + } + + QM_TRY(MOZ_TO_RESULT(Validate(aConn))); + QM_TRY(MOZ_TO_RESULT(trans.Commit())); + + if (migrating) { + // Migrations happen infrequently and reflect a chance in DB structure. + // This is a good time to rebuild the database. It also helps catch + // if a new migration is incorrect by fast failing on the corruption. + // Unfortunately, this must be performed outside of the transaction. + + QM_TRY(MOZ_TO_RESULT(aConn.ExecuteSimpleSQL("VACUUM"_ns)), QM_PROPAGATE, + ([&aConn](const nsresult rv) { + if (rv == NS_ERROR_STORAGE_CONSTRAINT) { + QM_WARNONLY_TRY(QM_TO_RESULT(IntegrityCheck(aConn))); + } + })); + } + + return NS_OK; +} + +nsresult InitializeConnection(mozIStorageConnection& aConn) { + MOZ_ASSERT(!NS_IsMainThread()); + + // This function needs to perform per-connection initialization tasks that + // need to happen regardless of the schema. + + // Note, the default encoding of UTF-8 is preferred. mozStorage does all + // the work necessary to convert UTF-16 nsString values for us. We don't + // need ordering and the binary equality operations are correct. So, do + // NOT set PRAGMA encoding to UTF-16. + + QM_TRY(MOZ_TO_RESULT(aConn.ExecuteSimpleSQL(nsPrintfCString( + // Use a smaller page size to improve perf/footprint; default is too large + "PRAGMA page_size = %u; " + // Enable auto_vacuum; this must happen after page_size and before WAL + "PRAGMA auto_vacuum = INCREMENTAL; " + "PRAGMA foreign_keys = ON; ", + kPageSize)))); + + // Limit fragmentation by growing the database by many pages at once. + QM_TRY(QM_OR_ELSE_WARN_IF( + // Expression. + MOZ_TO_RESULT(aConn.SetGrowthIncrement(kGrowthSize, ""_ns)), + // Predicate. + IsSpecificError<NS_ERROR_FILE_TOO_BIG>, + // Fallback. + ErrToDefaultOk<>)); + + // Enable WAL journaling. This must be performed in a separate transaction + // after changing the page_size and enabling auto_vacuum. + // Note there is a default journal_size_limit set by mozStorage. + QM_TRY(MOZ_TO_RESULT(aConn.ExecuteSimpleSQL(nsPrintfCString( + // WAL journal can grow to given number of *pages* + "PRAGMA wal_autocheckpoint = %u; " + // WAL must be enabled at the end to allow page size to be changed, etc. + "PRAGMA journal_mode = WAL; ", + kWalAutoCheckpointPages)))); + + // Verify that we successfully set the vacuum mode to incremental. It + // is very easy to put the database in a state where the auto_vacuum + // pragma above fails silently. +#ifdef DEBUG + { + QM_TRY_INSPECT(const auto& state, + quota::CreateAndExecuteSingleStepStatement( + aConn, "PRAGMA auto_vacuum;"_ns)); + + QM_TRY_INSPECT(const int32_t& mode, + MOZ_TO_RESULT_INVOKE_MEMBER(*state, GetInt32, 0)); + + // integer value 2 is incremental mode + QM_TRY(OkIf(mode == 2), NS_ERROR_UNEXPECTED); + } +#endif + + return NS_OK; +} + +Result<CacheId, nsresult> CreateCacheId(mozIStorageConnection& aConn) { + MOZ_ASSERT(!NS_IsMainThread()); + + QM_TRY(MOZ_TO_RESULT( + aConn.ExecuteSimpleSQL("INSERT INTO caches DEFAULT VALUES;"_ns))); + + QM_TRY_INSPECT(const auto& state, + quota::CreateAndExecuteSingleStepStatement< + quota::SingleStepResult::ReturnNullIfNoResult>( + aConn, "SELECT last_insert_rowid()"_ns)); + + QM_TRY(OkIf(state), Err(NS_ERROR_UNEXPECTED)); + + QM_TRY_INSPECT(const CacheId& id, + MOZ_TO_RESULT_INVOKE_MEMBER(state, GetInt64, 0)); + + return id; +} + +Result<DeletionInfo, nsresult> DeleteCacheId(mozIStorageConnection& aConn, + CacheId aCacheId) { + MOZ_ASSERT(!NS_IsMainThread()); + + // XXX only deletedBodyIdList needs to be non-const + QM_TRY_UNWRAP( + (auto [deletedBodyIdList, deletedSecurityIdList, deletedPaddingSize]), + DeleteAllCacheEntries(aConn, aCacheId)); + + QM_TRY(MOZ_TO_RESULT(DeleteSecurityInfoList(aConn, deletedSecurityIdList))); + + // Delete the remainder of the cache using cascade semantics. + QM_TRY_INSPECT(const auto& state, + MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsCOMPtr<mozIStorageStatement>, aConn, CreateStatement, + "DELETE FROM caches WHERE id=:id;"_ns)); + + QM_TRY(MOZ_TO_RESULT(state->BindInt64ByName("id"_ns, aCacheId))); + + QM_TRY(MOZ_TO_RESULT(state->Execute())); + + return DeletionInfo{std::move(deletedBodyIdList), deletedPaddingSize}; +} + +Result<AutoTArray<CacheId, 8>, nsresult> FindOrphanedCacheIds( + mozIStorageConnection& aConn) { + QM_TRY_INSPECT(const auto& state, + MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsCOMPtr<mozIStorageStatement>, aConn, CreateStatement, + "SELECT id FROM caches " + "WHERE id NOT IN (SELECT cache_id from storage);"_ns)); + + QM_TRY_RETURN( + (quota::CollectElementsWhileHasResultTyped<AutoTArray<CacheId, 8>>( + *state, [](auto& stmt) { + QM_TRY_RETURN(MOZ_TO_RESULT_INVOKE_MEMBER(stmt, GetInt64, 0)); + }))); +} + +Result<int64_t, nsresult> FindOverallPaddingSize(mozIStorageConnection& aConn) { + QM_TRY_INSPECT(const auto& state, + MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsCOMPtr<mozIStorageStatement>, aConn, CreateStatement, + "SELECT response_padding_size FROM entries " + "WHERE response_padding_size IS NOT NULL;"_ns)); + + int64_t overallPaddingSize = 0; + + QM_TRY(quota::CollectWhileHasResult( + *state, [&overallPaddingSize](auto& stmt) -> Result<Ok, nsresult> { + QM_TRY_INSPECT(const int64_t& padding_size, + MOZ_TO_RESULT_INVOKE_MEMBER(stmt, GetInt64, 0)); + + MOZ_DIAGNOSTIC_ASSERT(padding_size >= 0); + MOZ_DIAGNOSTIC_ASSERT(INT64_MAX - padding_size >= overallPaddingSize); + overallPaddingSize += padding_size; + + return Ok{}; + })); + + return overallPaddingSize; +} + +Result<nsTArray<nsID>, nsresult> GetKnownBodyIds(mozIStorageConnection& aConn) { + MOZ_ASSERT(!NS_IsMainThread()); + + QM_TRY_INSPECT( + const auto& state, + MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsCOMPtr<mozIStorageStatement>, aConn, CreateStatement, + "SELECT request_body_id, response_body_id FROM entries;"_ns)); + + AutoTArray<nsID, 64> idList; + + QM_TRY(quota::CollectWhileHasResult( + *state, [&idList](auto& stmt) -> Result<Ok, nsresult> { + // extract 0 to 2 nsID structs per row + for (uint32_t i = 0; i < 2; ++i) { + QM_TRY_INSPECT(const bool& isNull, + MOZ_TO_RESULT_INVOKE_MEMBER(stmt, GetIsNull, i)); + + if (!isNull) { + QM_TRY_INSPECT(const auto& id, ExtractId(stmt, i)); + + idList.AppendElement(id); + } + } + + return Ok{}; + })); + + return std::move(idList); +} + +Result<Maybe<SavedResponse>, nsresult> CacheMatch( + mozIStorageConnection& aConn, CacheId aCacheId, + const CacheRequest& aRequest, const CacheQueryParams& aParams) { + MOZ_ASSERT(!NS_IsMainThread()); + + QM_TRY_INSPECT(const auto& matches, + QueryCache(aConn, aCacheId, aRequest, aParams, 1)); + + if (matches.IsEmpty()) { + return Maybe<SavedResponse>(); + } + + QM_TRY_UNWRAP(auto response, ReadResponse(aConn, matches[0])); + + response.mCacheId = aCacheId; + + return Some(std::move(response)); +} + +Result<nsTArray<SavedResponse>, nsresult> CacheMatchAll( + mozIStorageConnection& aConn, CacheId aCacheId, + const Maybe<CacheRequest>& aMaybeRequest, const CacheQueryParams& aParams) { + MOZ_ASSERT(!NS_IsMainThread()); + + QM_TRY_INSPECT( + const auto& matches, ([&aConn, aCacheId, &aMaybeRequest, &aParams] { + if (aMaybeRequest.isNothing()) { + QM_TRY_RETURN(QueryAll(aConn, aCacheId)); + } + + QM_TRY_RETURN( + QueryCache(aConn, aCacheId, aMaybeRequest.ref(), aParams)); + }())); + + // TODO: replace this with a bulk load using SQL IN clause (bug 1110458) + QM_TRY_RETURN(TransformIntoNewArrayAbortOnErr( + matches, + [&aConn, aCacheId](const auto match) -> Result<SavedResponse, nsresult> { + QM_TRY_UNWRAP(auto savedResponse, ReadResponse(aConn, match)); + + savedResponse.mCacheId = aCacheId; + return savedResponse; + }, + fallible)); +} + +Result<DeletionInfo, nsresult> CachePut(mozIStorageConnection& aConn, + CacheId aCacheId, + const CacheRequest& aRequest, + const nsID* aRequestBodyId, + const CacheResponse& aResponse, + const nsID* aResponseBodyId) { + MOZ_ASSERT(!NS_IsMainThread()); + + QM_TRY_INSPECT( + const auto& matches, + QueryCache(aConn, aCacheId, aRequest, + CacheQueryParams(false, false, false, false, u""_ns))); + + // XXX only deletedBodyIdList needs to be non-const + QM_TRY_UNWRAP( + (auto [deletedBodyIdList, deletedSecurityIdList, deletedPaddingSize]), + DeleteEntries(aConn, matches)); + + QM_TRY(MOZ_TO_RESULT(InsertEntry(aConn, aCacheId, aRequest, aRequestBodyId, + aResponse, aResponseBodyId))); + + // Delete the security values after doing the insert to avoid churning + // the security table when its not necessary. + QM_TRY(MOZ_TO_RESULT(DeleteSecurityInfoList(aConn, deletedSecurityIdList))); + + return DeletionInfo{std::move(deletedBodyIdList), deletedPaddingSize}; +} + +Result<Maybe<DeletionInfo>, nsresult> CacheDelete( + mozIStorageConnection& aConn, CacheId aCacheId, + const CacheRequest& aRequest, const CacheQueryParams& aParams) { + MOZ_ASSERT(!NS_IsMainThread()); + + QM_TRY_INSPECT(const auto& matches, + QueryCache(aConn, aCacheId, aRequest, aParams)); + + if (matches.IsEmpty()) { + return Maybe<DeletionInfo>(); + } + + // XXX only deletedBodyIdList needs to be non-const + QM_TRY_UNWRAP( + (auto [deletedBodyIdList, deletedSecurityIdList, deletedPaddingSize]), + DeleteEntries(aConn, matches)); + + QM_TRY(MOZ_TO_RESULT(DeleteSecurityInfoList(aConn, deletedSecurityIdList))); + + return Some(DeletionInfo{std::move(deletedBodyIdList), deletedPaddingSize}); +} + +Result<nsTArray<SavedRequest>, nsresult> CacheKeys( + mozIStorageConnection& aConn, CacheId aCacheId, + const Maybe<CacheRequest>& aMaybeRequest, const CacheQueryParams& aParams) { + MOZ_ASSERT(!NS_IsMainThread()); + + QM_TRY_INSPECT( + const auto& matches, ([&aConn, aCacheId, &aMaybeRequest, &aParams] { + if (aMaybeRequest.isNothing()) { + QM_TRY_RETURN(QueryAll(aConn, aCacheId)); + } + + QM_TRY_RETURN( + QueryCache(aConn, aCacheId, aMaybeRequest.ref(), aParams)); + }())); + + // TODO: replace this with a bulk load using SQL IN clause (bug 1110458) + QM_TRY_RETURN(TransformIntoNewArrayAbortOnErr( + matches, + [&aConn, aCacheId](const auto match) -> Result<SavedRequest, nsresult> { + QM_TRY_UNWRAP(auto savedRequest, ReadRequest(aConn, match)); + + savedRequest.mCacheId = aCacheId; + return savedRequest; + }, + fallible)); +} + +Result<Maybe<SavedResponse>, nsresult> StorageMatch( + mozIStorageConnection& aConn, Namespace aNamespace, + const CacheRequest& aRequest, const CacheQueryParams& aParams) { + MOZ_ASSERT(!NS_IsMainThread()); + + // If we are given a cache to check, then simply find its cache ID + // and perform the match. + if (aParams.cacheNameSet()) { + QM_TRY_INSPECT(const auto& maybeCacheId, + StorageGetCacheId(aConn, aNamespace, aParams.cacheName())); + if (maybeCacheId.isNothing()) { + return Maybe<SavedResponse>(); + } + + return CacheMatch(aConn, maybeCacheId.ref(), aRequest, aParams); + } + + // Otherwise we need to get a list of all the cache IDs in this namespace. + + QM_TRY_INSPECT(const auto& state, + MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsCOMPtr<mozIStorageStatement>, aConn, CreateStatement, + "SELECT cache_id FROM storage WHERE " + "namespace=:namespace ORDER BY rowid;"_ns)); + + QM_TRY(MOZ_TO_RESULT(state->BindInt32ByName("namespace"_ns, aNamespace))); + + QM_TRY_INSPECT( + const auto& cacheIdList, + (quota::CollectElementsWhileHasResultTyped<AutoTArray<CacheId, 32>>( + *state, [](auto& stmt) { + QM_TRY_RETURN(MOZ_TO_RESULT_INVOKE_MEMBER(stmt, GetInt64, 0)); + }))); + + // Now try to find a match in each cache in order + for (const auto cacheId : cacheIdList) { + QM_TRY_UNWRAP(auto matchedResponse, + CacheMatch(aConn, cacheId, aRequest, aParams)); + + if (matchedResponse.isSome()) { + return matchedResponse; + } + } + + return Maybe<SavedResponse>(); +} + +Result<Maybe<CacheId>, nsresult> StorageGetCacheId(mozIStorageConnection& aConn, + Namespace aNamespace, + const nsAString& aKey) { + MOZ_ASSERT(!NS_IsMainThread()); + + // How we constrain the key column depends on the value of our key. Use + // a format string for the query and let CreateAndBindKeyStatement() fill + // it in for us. + const char* const query = + "SELECT cache_id FROM storage " + "WHERE namespace=:namespace AND %s " + "ORDER BY rowid;"; + + QM_TRY_INSPECT(const auto& state, + CreateAndBindKeyStatement(aConn, query, aKey)); + + QM_TRY(MOZ_TO_RESULT(state->BindInt32ByName("namespace"_ns, aNamespace))); + + QM_TRY_INSPECT(const bool& hasMoreData, + MOZ_TO_RESULT_INVOKE_MEMBER(*state, ExecuteStep)); + + if (!hasMoreData) { + return Maybe<CacheId>(); + } + + QM_TRY_RETURN( + MOZ_TO_RESULT_INVOKE_MEMBER(*state, GetInt64, 0).map(Some<CacheId>)); +} + +nsresult StoragePutCache(mozIStorageConnection& aConn, Namespace aNamespace, + const nsAString& aKey, CacheId aCacheId) { + MOZ_ASSERT(!NS_IsMainThread()); + + QM_TRY_INSPECT(const auto& state, + MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsCOMPtr<mozIStorageStatement>, aConn, CreateStatement, + "INSERT INTO storage (namespace, key, cache_id) " + "VALUES (:namespace, :key, :cache_id);"_ns)); + + QM_TRY(MOZ_TO_RESULT(state->BindInt32ByName("namespace"_ns, aNamespace))); + QM_TRY(MOZ_TO_RESULT(state->BindStringAsBlobByName("key"_ns, aKey))); + QM_TRY(MOZ_TO_RESULT(state->BindInt64ByName("cache_id"_ns, aCacheId))); + QM_TRY(MOZ_TO_RESULT(state->Execute())); + + return NS_OK; +} + +nsresult StorageForgetCache(mozIStorageConnection& aConn, Namespace aNamespace, + const nsAString& aKey) { + MOZ_ASSERT(!NS_IsMainThread()); + + // How we constrain the key column depends on the value of our key. Use + // a format string for the query and let CreateAndBindKeyStatement() fill + // it in for us. + const char* const query = + "DELETE FROM storage WHERE namespace=:namespace AND %s;"; + + QM_TRY_INSPECT(const auto& state, + CreateAndBindKeyStatement(aConn, query, aKey)); + + QM_TRY(MOZ_TO_RESULT(state->BindInt32ByName("namespace"_ns, aNamespace))); + + QM_TRY(MOZ_TO_RESULT(state->Execute())); + + return NS_OK; +} + +Result<nsTArray<nsString>, nsresult> StorageGetKeys( + mozIStorageConnection& aConn, Namespace aNamespace) { + MOZ_ASSERT(!NS_IsMainThread()); + + QM_TRY_INSPECT( + const auto& state, + MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsCOMPtr<mozIStorageStatement>, aConn, CreateStatement, + "SELECT key FROM storage WHERE namespace=:namespace ORDER BY rowid;"_ns)); + + QM_TRY(MOZ_TO_RESULT(state->BindInt32ByName("namespace"_ns, aNamespace))); + + QM_TRY_RETURN(quota::CollectElementsWhileHasResult(*state, [](auto& stmt) { + QM_TRY_RETURN( + MOZ_TO_RESULT_INVOKE_MEMBER_TYPED(nsString, stmt, GetBlobAsString, 0)); + })); +} + +namespace { + +Result<EntryIds, nsresult> QueryAll(mozIStorageConnection& aConn, + CacheId aCacheId) { + MOZ_ASSERT(!NS_IsMainThread()); + + QM_TRY_INSPECT( + const auto& state, + MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsCOMPtr<mozIStorageStatement>, aConn, CreateStatement, + "SELECT id FROM entries WHERE cache_id=:cache_id ORDER BY id;"_ns)); + + QM_TRY(MOZ_TO_RESULT(state->BindInt64ByName("cache_id"_ns, aCacheId))); + + QM_TRY_RETURN((quota::CollectElementsWhileHasResultTyped<EntryIds>( + *state, [](auto& stmt) { + QM_TRY_RETURN(MOZ_TO_RESULT_INVOKE_MEMBER(stmt, GetInt32, 0)); + }))); +} + +Result<EntryIds, nsresult> QueryCache(mozIStorageConnection& aConn, + CacheId aCacheId, + const CacheRequest& aRequest, + const CacheQueryParams& aParams, + uint32_t aMaxResults) { + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aMaxResults > 0); + + if (!aParams.ignoreMethod() && + !aRequest.method().LowerCaseEqualsLiteral("get")) { + return Result<EntryIds, nsresult>{std::in_place}; + } + + nsAutoCString query( + "SELECT id, COUNT(response_headers.name) AS vary_count " + "FROM entries " + "LEFT OUTER JOIN response_headers ON " + "entries.id=response_headers.entry_id " + "AND response_headers.name='vary' COLLATE NOCASE " + "WHERE entries.cache_id=:cache_id " + "AND entries.request_url_no_query_hash=:url_no_query_hash "); + + if (!aParams.ignoreSearch()) { + query.AppendLiteral("AND entries.request_url_query_hash=:url_query_hash "); + } + + query.AppendLiteral("AND entries.request_url_no_query=:url_no_query "); + + if (!aParams.ignoreSearch()) { + query.AppendLiteral("AND entries.request_url_query=:url_query "); + } + + query.AppendLiteral("GROUP BY entries.id ORDER BY entries.id;"); + + QM_TRY_INSPECT(const auto& state, MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsCOMPtr<mozIStorageStatement>, aConn, + CreateStatement, query)); + + QM_TRY(MOZ_TO_RESULT(state->BindInt64ByName("cache_id"_ns, aCacheId))); + + QM_TRY_INSPECT(const auto& crypto, + MOZ_TO_RESULT_GET_TYPED(nsCOMPtr<nsICryptoHash>, + MOZ_SELECT_OVERLOAD(do_CreateInstance), + NS_CRYPTO_HASH_CONTRACTID)); + + QM_TRY_INSPECT(const auto& urlWithoutQueryHash, + HashCString(*crypto, aRequest.urlWithoutQuery())); + + QM_TRY(MOZ_TO_RESULT(state->BindUTF8StringAsBlobByName("url_no_query_hash"_ns, + urlWithoutQueryHash))); + + if (!aParams.ignoreSearch()) { + QM_TRY_INSPECT(const auto& urlQueryHash, + HashCString(*crypto, aRequest.urlQuery())); + + QM_TRY(MOZ_TO_RESULT( + state->BindUTF8StringAsBlobByName("url_query_hash"_ns, urlQueryHash))); + } + + QM_TRY(MOZ_TO_RESULT(state->BindUTF8StringByName( + "url_no_query"_ns, aRequest.urlWithoutQuery()))); + + if (!aParams.ignoreSearch()) { + QM_TRY(MOZ_TO_RESULT( + state->BindUTF8StringByName("url_query"_ns, aRequest.urlQuery()))); + } + + EntryIds entryIdList; + + QM_TRY(CollectWhile( + [&state, &entryIdList, aMaxResults]() -> Result<bool, nsresult> { + if (entryIdList.Length() == aMaxResults) { + return false; + } + QM_TRY_RETURN(MOZ_TO_RESULT_INVOKE_MEMBER(state, ExecuteStep)); + }, + [&state, &entryIdList, ignoreVary = aParams.ignoreVary(), &aConn, + &aRequest]() -> Result<Ok, nsresult> { + QM_TRY_INSPECT(const EntryId& entryId, + MOZ_TO_RESULT_INVOKE_MEMBER(state, GetInt32, 0)); + + QM_TRY_INSPECT(const int32_t& varyCount, + MOZ_TO_RESULT_INVOKE_MEMBER(state, GetInt32, 1)); + + if (!ignoreVary && varyCount > 0) { + QM_TRY_INSPECT(const bool& matchedByVary, + MatchByVaryHeader(aConn, aRequest, entryId)); + if (!matchedByVary) { + return Ok{}; + } + } + + entryIdList.AppendElement(entryId); + + return Ok{}; + })); + + return entryIdList; +} + +Result<bool, nsresult> MatchByVaryHeader(mozIStorageConnection& aConn, + const CacheRequest& aRequest, + EntryId entryId) { + MOZ_ASSERT(!NS_IsMainThread()); + + QM_TRY_INSPECT( + const auto& varyValues, + ([&aConn, entryId]() -> Result<AutoTArray<nsCString, 8>, nsresult> { + QM_TRY_INSPECT( + const auto& state, + MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsCOMPtr<mozIStorageStatement>, aConn, CreateStatement, + "SELECT value FROM response_headers " + "WHERE name='vary' COLLATE NOCASE " + "AND entry_id=:entry_id;"_ns)); + + QM_TRY(MOZ_TO_RESULT(state->BindInt32ByName("entry_id"_ns, entryId))); + + QM_TRY_RETURN(( + quota::CollectElementsWhileHasResultTyped<AutoTArray<nsCString, 8>>( + *state, [](auto& stmt) { + QM_TRY_RETURN(MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsCString, stmt, GetUTF8String, 0)); + }))); + }())); + + // Should not have called this function if this was not the case + MOZ_DIAGNOSTIC_ASSERT(!varyValues.IsEmpty()); + + QM_TRY_INSPECT(const auto& state, + MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsCOMPtr<mozIStorageStatement>, aConn, CreateStatement, + "SELECT name, value FROM request_headers " + "WHERE entry_id=:entry_id;"_ns)); + + QM_TRY(MOZ_TO_RESULT(state->BindInt32ByName("entry_id"_ns, entryId))); + + RefPtr<InternalHeaders> cachedHeaders = + new InternalHeaders(HeadersGuardEnum::None); + + QM_TRY(quota::CollectWhileHasResult( + *state, [&cachedHeaders](auto& stmt) -> Result<Ok, nsresult> { + QM_TRY_INSPECT(const auto& name, + MOZ_TO_RESULT_INVOKE_MEMBER_TYPED(nsCString, stmt, + GetUTF8String, 0)); + QM_TRY_INSPECT(const auto& value, + MOZ_TO_RESULT_INVOKE_MEMBER_TYPED(nsCString, stmt, + GetUTF8String, 1)); + + ErrorResult errorResult; + + cachedHeaders->Append(name, value, errorResult); + if (errorResult.Failed()) { + return Err(errorResult.StealNSResult()); + } + + return Ok{}; + })); + + RefPtr<InternalHeaders> queryHeaders = + TypeUtils::ToInternalHeaders(aRequest.headers()); + + // Assume the vary headers match until we find a conflict + bool varyHeadersMatch = true; + + for (const auto& varyValue : varyValues) { + // Extract the header names inside the Vary header value. + bool bailOut = false; + for (const nsACString& header : + nsCCharSeparatedTokenizer(varyValue, NS_HTTP_HEADER_SEP).ToRange()) { + MOZ_DIAGNOSTIC_ASSERT(!header.EqualsLiteral("*"), + "We should have already caught this in " + "TypeUtils::ToPCacheResponseWithoutBody()"); + + ErrorResult errorResult; + nsAutoCString queryValue; + queryHeaders->Get(header, queryValue, errorResult); + if (errorResult.Failed()) { + errorResult.SuppressException(); + MOZ_DIAGNOSTIC_ASSERT(queryValue.IsEmpty()); + } + + nsAutoCString cachedValue; + cachedHeaders->Get(header, cachedValue, errorResult); + if (errorResult.Failed()) { + errorResult.SuppressException(); + MOZ_DIAGNOSTIC_ASSERT(cachedValue.IsEmpty()); + } + + if (queryValue != cachedValue) { + varyHeadersMatch = false; + bailOut = true; + break; + } + } + + if (bailOut) { + break; + } + } + + return varyHeadersMatch; +} + +static nsresult SelectAndDeleteEntriesInternal( + mozIStorageConnection& aConn, const Span<const EntryId>& aEntryIdList, + nsTArray<nsID>& aDeletedBodyIdListOut, + nsTArray<IdCount>& aDeletedSecurityIdListOut, + int64_t& aDeletedPaddingSizeOut) { + nsAutoCString query( + "SELECT " + "request_body_id, " + "response_body_id, " + "response_security_info_id, " + "response_padding_size " + "FROM entries WHERE id IN ("); + + AppendListParamsToQuery(query, aEntryIdList.Length()); + query.AppendLiteral(")"); + + QM_TRY_INSPECT(const auto& state, MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsCOMPtr<mozIStorageStatement>, aConn, + CreateStatement, query)); + + QM_TRY(MOZ_TO_RESULT(BindListParamsToQuery(*state, aEntryIdList))); + + int64_t overallPaddingSize = 0; + + QM_TRY(quota::CollectWhileHasResult( + *state, + [&overallPaddingSize, &aDeletedBodyIdListOut, + &aDeletedSecurityIdListOut](auto& stmt) -> Result<Ok, nsresult> { + // extract 0 to 2 nsID structs per row + for (uint32_t i = 0; i < 2; ++i) { + QM_TRY_INSPECT(const bool& isNull, + MOZ_TO_RESULT_INVOKE_MEMBER(stmt, GetIsNull, i)); + + if (!isNull) { + QM_TRY_INSPECT(const auto& id, ExtractId(stmt, i)); + + aDeletedBodyIdListOut.AppendElement(id); + } + } + + { // and then a possible third entry for the security id + QM_TRY_INSPECT(const bool& isNull, + MOZ_TO_RESULT_INVOKE_MEMBER(stmt, GetIsNull, 2)); + + if (!isNull) { + QM_TRY_INSPECT(const int32_t& securityId, + MOZ_TO_RESULT_INVOKE_MEMBER(stmt, GetInt32, 2)); + + // XXXtt: Consider using map for aDeletedSecuityIdListOut. + auto foundIt = + std::find_if(aDeletedSecurityIdListOut.begin(), + aDeletedSecurityIdListOut.end(), + [securityId](const auto& deletedSecurityId) { + return deletedSecurityId.mId == securityId; + }); + + if (foundIt == aDeletedSecurityIdListOut.end()) { + // Add a new entry for this ID with a count of 1, if it's not in + // the list + aDeletedSecurityIdListOut.AppendElement(IdCount(securityId)); + } else { + // Otherwise, increment the count for this ID + foundIt->mCount += 1; + } + } + } + + { + // It's possible to have null padding size for non-opaque response + QM_TRY_INSPECT(const bool& isNull, + MOZ_TO_RESULT_INVOKE_MEMBER(stmt, GetIsNull, 3)); + + if (!isNull) { + QM_TRY_INSPECT(const int64_t& paddingSize, + MOZ_TO_RESULT_INVOKE_MEMBER(stmt, GetInt64, 3)); + + MOZ_DIAGNOSTIC_ASSERT(paddingSize >= 0); + MOZ_DIAGNOSTIC_ASSERT(INT64_MAX - overallPaddingSize >= + paddingSize); + overallPaddingSize += paddingSize; + } + } + + return Ok{}; + })); + + aDeletedPaddingSizeOut += overallPaddingSize; + + // Dependent records removed via ON DELETE CASCADE + + query = "DELETE FROM entries WHERE id IN ("_ns; + AppendListParamsToQuery(query, aEntryIdList.Length()); + query.AppendLiteral(")"); + + { + QM_TRY_INSPECT(const auto& state, MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsCOMPtr<mozIStorageStatement>, aConn, + CreateStatement, query)); + + QM_TRY(MOZ_TO_RESULT(BindListParamsToQuery(*state, aEntryIdList))); + + QM_TRY(MOZ_TO_RESULT(state->Execute())); + } + + return NS_OK; +} + +static nsresult DeleteEntriesInternal( + mozIStorageConnection& aConn, const nsTArray<EntryId>& aEntryIdList, + nsTArray<nsID>& aDeletedBodyIdListOut, + nsTArray<IdCount>& aDeletedSecurityIdListOut, + int64_t& aDeletedPaddingSizeOut, uint32_t aPos, uint32_t aLen) { + MOZ_ASSERT(!NS_IsMainThread()); + + if (aEntryIdList.IsEmpty()) { + return NS_OK; + } + + MOZ_DIAGNOSTIC_ASSERT(aPos < aEntryIdList.Length()); + + auto remaining = aLen; + uint32_t currPos = 0; + + do { + // Sqlite limits the number of entries allowed for an IN clause, + // so split up larger operations. + auto currLen = std::min(kMaxEntriesPerStatement, remaining); + + SelectAndDeleteEntriesInternal( + aConn, Span<const EntryId>(aEntryIdList.Elements() + currPos, currLen), + aDeletedBodyIdListOut, aDeletedSecurityIdListOut, + aDeletedPaddingSizeOut); + + remaining -= currLen; + currPos += currLen; + + } while (remaining > 0); + + return NS_OK; +} + +Result<std::tuple<nsTArray<nsID>, AutoTArray<IdCount, 16>, int64_t>, nsresult> +DeleteEntries(mozIStorageConnection& aConn, + const nsTArray<EntryId>& aEntryIdList) { + auto result = + std::make_tuple(nsTArray<nsID>{}, AutoTArray<IdCount, 16>{}, int64_t{0}); + + QM_TRY(MOZ_TO_RESULT(DeleteEntriesInternal( + aConn, aEntryIdList, std::get<0>(result), std::get<1>(result), + std::get<2>(result), 0, aEntryIdList.Length()))); + + return result; +} + +Result<std::tuple<nsTArray<nsID>, AutoTArray<IdCount, 16>, int64_t>, nsresult> +DeleteAllCacheEntries(mozIStorageConnection& aConn, CacheId& aCacheId) { + auto result = + std::make_tuple(nsTArray<nsID>{}, AutoTArray<IdCount, 16>{}, int64_t{0}); + auto& deletedBodyIdList = std::get<0>(result); + auto& deletedSecurityIdList = std::get<1>(result); + auto& deletedPaddingSize = std::get<2>(result); + + // XXX: We could create a query string with aggregation that would generate + // a single summary result such that we don't have to go through each row + // just to aggregate the result. This method could become much more + // performant. + // + // The columns could look like: + // + // GROUP_CONCAT(request_body_id || ',' || response_body_id), + // GROUP_CONCAT(response_security_info_id), + // SUM(response_padding_size) + // + // strtok the result row to generate the desired output to fill-in + // deletedBodyIdList, deletedSecurityIdList and deletedPaddingSize + // + // I am not sure about the memory requirements for such operation; + // it will all depend upon the result filtered by the `cache_id`. + nsAutoCString query( + "SELECT " + "request_body_id, " + "response_body_id, " + "response_security_info_id, " + "response_padding_size " + "FROM entries WHERE cache_id=:cache_id ORDER BY id;"_ns); + + QM_TRY_INSPECT(const auto& state, MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsCOMPtr<mozIStorageStatement>, aConn, + CreateStatement, query)); + + QM_TRY(MOZ_TO_RESULT(state->BindInt64ByName("cache_id"_ns, aCacheId))); + + QM_TRY(quota::CollectWhileHasResult( + *state, + [&deletedPaddingSize, &deletedBodyIdList, + &deletedSecurityIdList](auto& stmt) -> Result<Ok, nsresult> { + // extract 0 to 2 nsID structs per row + for (uint32_t i = 0; i < 2; ++i) { + QM_TRY_INSPECT(const bool& isNull, + MOZ_TO_RESULT_INVOKE_MEMBER(stmt, GetIsNull, i)); + + if (!isNull) { + QM_TRY_INSPECT(const auto& id, ExtractId(stmt, i)); + + deletedBodyIdList.AppendElement(id); + } + } + + { // and then a possible third entry for the security id + QM_TRY_INSPECT(const bool& isNull, + MOZ_TO_RESULT_INVOKE_MEMBER(stmt, GetIsNull, 2)); + + if (!isNull) { + QM_TRY_INSPECT(const int32_t& securityId, + MOZ_TO_RESULT_INVOKE_MEMBER(stmt, GetInt32, 2)); + + // XXXtt: Consider using map for aDeletedSecuityIdListOut. + auto foundIt = std::find_if( + deletedSecurityIdList.begin(), deletedSecurityIdList.end(), + [securityId](const auto& deletedSecurityId) { + return deletedSecurityId.mId == securityId; + }); + + if (foundIt == deletedSecurityIdList.end()) { + // Add a new entry for this ID with a count of 1, if it's not in + // the list + deletedSecurityIdList.AppendElement(IdCount(securityId)); + } else { + // Otherwise, increment the count for this ID + foundIt->mCount += 1; + } + } + } + + { + // It's possible to have null padding size for non-opaque response + QM_TRY_INSPECT(const bool& isNull, + MOZ_TO_RESULT_INVOKE_MEMBER(stmt, GetIsNull, 3)); + + if (!isNull) { + QM_TRY_INSPECT(const int64_t& paddingSize, + MOZ_TO_RESULT_INVOKE_MEMBER(stmt, GetInt64, 3)); + + MOZ_DIAGNOSTIC_ASSERT(paddingSize >= 0); + + // Assert overflow + MOZ_DIAGNOSTIC_ASSERT(INT64_MAX - deletedPaddingSize >= + paddingSize); + + deletedPaddingSize += paddingSize; + } + } + + return Ok{}; + })); + + // Dependent records removed via ON DELETE CASCADE + + query = "DELETE FROM entries WHERE cache_id=:cache_id"_ns; + + { + QM_TRY_INSPECT(const auto& state, MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsCOMPtr<mozIStorageStatement>, aConn, + CreateStatement, query)); + + QM_TRY(MOZ_TO_RESULT(state->BindInt64ByName("cache_id"_ns, aCacheId))); + QM_TRY(MOZ_TO_RESULT(state->Execute())); + } + + return result; +} + +Result<int32_t, nsresult> InsertSecurityInfo( + mozIStorageConnection& aConn, nsICryptoHash& aCrypto, + nsITransportSecurityInfo* aSecurityInfo) { + MOZ_DIAGNOSTIC_ASSERT(aSecurityInfo); + if (!aSecurityInfo) { + return Err(NS_ERROR_FAILURE); + } + nsCString data; + nsresult rv = aSecurityInfo->ToString(data); + if (NS_FAILED(rv)) { + return Err(rv); + } + + // We want to use an index to find existing security blobs, but indexing + // the full blob would be quite expensive. Instead, we index a small + // hash value. Calculate this hash as the first 8 bytes of the SHA1 of + // the full data. + QM_TRY_INSPECT(const auto& hash, HashCString(aCrypto, data)); + + // Next, search for an existing entry for this blob by comparing the hash + // value first and then the full data. SQLite is smart enough to use + // the index on the hash to search the table before doing the expensive + // comparison of the large data column. (This was verified with EXPLAIN.) + QM_TRY_INSPECT( + const auto& selectStmt, + quota::CreateAndExecuteSingleStepStatement< + quota::SingleStepResult::ReturnNullIfNoResult>( + aConn, + // Note that hash and data are blobs, but we can use = here since the + // columns are NOT NULL. + "SELECT id, refcount FROM security_info WHERE hash=:hash AND " + "data=:data;"_ns, + [&hash, &data](auto& state) -> Result<Ok, nsresult> { + QM_TRY(MOZ_TO_RESULT( + state.BindUTF8StringAsBlobByName("hash"_ns, hash))); + QM_TRY(MOZ_TO_RESULT( + state.BindUTF8StringAsBlobByName("data"_ns, data))); + + return Ok{}; + })); + + // This security info blob is already in the database + if (selectStmt) { + // get the existing security blob id to return + QM_TRY_INSPECT(const int32_t& id, + MOZ_TO_RESULT_INVOKE_MEMBER(selectStmt, GetInt32, 0)); + QM_TRY_INSPECT(const int32_t& refcount, + MOZ_TO_RESULT_INVOKE_MEMBER(selectStmt, GetInt32, 1)); + + // But first, update the refcount in the database. + QM_TRY_INSPECT( + const auto& state, + MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsCOMPtr<mozIStorageStatement>, aConn, CreateStatement, + "UPDATE security_info SET refcount=:refcount WHERE id=:id;"_ns)); + + QM_TRY(MOZ_TO_RESULT(state->BindInt32ByName("refcount"_ns, refcount + 1))); + QM_TRY(MOZ_TO_RESULT(state->BindInt32ByName("id"_ns, id))); + QM_TRY(MOZ_TO_RESULT(state->Execute())); + + return id; + } + + // This is a new security info blob. Create a new row in the security table + // with an initial refcount of 1. + QM_TRY_INSPECT(const auto& state, + MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsCOMPtr<mozIStorageStatement>, aConn, CreateStatement, + "INSERT INTO security_info (hash, data, refcount) " + "VALUES (:hash, :data, 1);"_ns)); + + QM_TRY(MOZ_TO_RESULT(state->BindUTF8StringAsBlobByName("hash"_ns, hash))); + QM_TRY(MOZ_TO_RESULT(state->BindUTF8StringAsBlobByName("data"_ns, data))); + QM_TRY(MOZ_TO_RESULT(state->Execute())); + + { + QM_TRY_INSPECT(const auto& state, + quota::CreateAndExecuteSingleStepStatement( + aConn, "SELECT last_insert_rowid()"_ns)); + + QM_TRY_RETURN(MOZ_TO_RESULT_INVOKE_MEMBER(*state, GetInt32, 0)); + } +} + +nsresult DeleteSecurityInfo(mozIStorageConnection& aConn, int32_t aId, + int32_t aCount) { + // First, we need to determine the current refcount for this security blob. + QM_TRY_INSPECT( + const int32_t& refcount, ([&aConn, aId]() -> Result<int32_t, nsresult> { + QM_TRY_INSPECT( + const auto& state, + quota::CreateAndExecuteSingleStepStatement( + aConn, "SELECT refcount FROM security_info WHERE id=:id;"_ns, + [aId](auto& state) -> Result<Ok, nsresult> { + QM_TRY(MOZ_TO_RESULT(state.BindInt32ByName("id"_ns, aId))); + return Ok{}; + })); + + QM_TRY_RETURN(MOZ_TO_RESULT_INVOKE_MEMBER(*state, GetInt32, 0)); + }())); + + MOZ_DIAGNOSTIC_ASSERT(refcount >= aCount); + + // Next, calculate the new refcount + int32_t newCount = refcount - aCount; + + // If the last reference to this security blob was removed we can + // just remove the entire row. + if (newCount == 0) { + QM_TRY_INSPECT(const auto& state, + MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsCOMPtr<mozIStorageStatement>, aConn, CreateStatement, + "DELETE FROM security_info WHERE id=:id;"_ns)); + + QM_TRY(MOZ_TO_RESULT(state->BindInt32ByName("id"_ns, aId))); + QM_TRY(MOZ_TO_RESULT(state->Execute())); + + return NS_OK; + } + + // Otherwise update the refcount in the table to reflect the reduced + // number of references to the security blob. + QM_TRY_INSPECT( + const auto& state, + MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsCOMPtr<mozIStorageStatement>, aConn, CreateStatement, + "UPDATE security_info SET refcount=:refcount WHERE id=:id;"_ns)); + + QM_TRY(MOZ_TO_RESULT(state->BindInt32ByName("refcount"_ns, newCount))); + QM_TRY(MOZ_TO_RESULT(state->BindInt32ByName("id"_ns, aId))); + QM_TRY(MOZ_TO_RESULT(state->Execute())); + + return NS_OK; +} + +nsresult DeleteSecurityInfoList( + mozIStorageConnection& aConn, + const nsTArray<IdCount>& aDeletedStorageIdList) { + for (const auto& deletedStorageId : aDeletedStorageIdList) { + QM_TRY(MOZ_TO_RESULT(DeleteSecurityInfo(aConn, deletedStorageId.mId, + deletedStorageId.mCount))); + } + + return NS_OK; +} + +nsresult InsertEntry(mozIStorageConnection& aConn, CacheId aCacheId, + const CacheRequest& aRequest, const nsID* aRequestBodyId, + const CacheResponse& aResponse, + const nsID* aResponseBodyId) { + MOZ_ASSERT(!NS_IsMainThread()); + + QM_TRY_INSPECT(const auto& crypto, + MOZ_TO_RESULT_GET_TYPED(nsCOMPtr<nsICryptoHash>, + MOZ_SELECT_OVERLOAD(do_CreateInstance), + NS_CRYPTO_HASH_CONTRACTID)); + + int32_t securityId = -1; + if (aResponse.securityInfo()) { + QM_TRY_UNWRAP(securityId, + InsertSecurityInfo(aConn, *crypto, aResponse.securityInfo())); + } + + { + QM_TRY_INSPECT(const auto& state, + MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsCOMPtr<mozIStorageStatement>, aConn, CreateStatement, + "INSERT INTO entries (" + "request_method, " + "request_url_no_query, " + "request_url_no_query_hash, " + "request_url_query, " + "request_url_query_hash, " + "request_url_fragment, " + "request_referrer, " + "request_referrer_policy, " + "request_headers_guard, " + "request_mode, " + "request_credentials, " + "request_contentpolicytype, " + "request_cache, " + "request_redirect, " + "request_integrity, " + "request_body_id, " + "response_type, " + "response_status, " + "response_status_text, " + "response_headers_guard, " + "response_body_id, " + "response_security_info_id, " + "response_principal_info, " + "response_padding_size, " + "cache_id " + ") VALUES (" + ":request_method, " + ":request_url_no_query, " + ":request_url_no_query_hash, " + ":request_url_query, " + ":request_url_query_hash, " + ":request_url_fragment, " + ":request_referrer, " + ":request_referrer_policy, " + ":request_headers_guard, " + ":request_mode, " + ":request_credentials, " + ":request_contentpolicytype, " + ":request_cache, " + ":request_redirect, " + ":request_integrity, " + ":request_body_id, " + ":response_type, " + ":response_status, " + ":response_status_text, " + ":response_headers_guard, " + ":response_body_id, " + ":response_security_info_id, " + ":response_principal_info, " + ":response_padding_size, " + ":cache_id " + ");"_ns)); + + QM_TRY(MOZ_TO_RESULT( + state->BindUTF8StringByName("request_method"_ns, aRequest.method()))); + + QM_TRY(MOZ_TO_RESULT(state->BindUTF8StringByName( + "request_url_no_query"_ns, aRequest.urlWithoutQuery()))); + + QM_TRY_INSPECT(const auto& urlWithoutQueryHash, + HashCString(*crypto, aRequest.urlWithoutQuery())); + + QM_TRY(MOZ_TO_RESULT(state->BindUTF8StringAsBlobByName( + "request_url_no_query_hash"_ns, urlWithoutQueryHash))); + + QM_TRY(MOZ_TO_RESULT(state->BindUTF8StringByName("request_url_query"_ns, + aRequest.urlQuery()))); + + QM_TRY_INSPECT(const auto& urlQueryHash, + HashCString(*crypto, aRequest.urlQuery())); + + QM_TRY(MOZ_TO_RESULT(state->BindUTF8StringAsBlobByName( + "request_url_query_hash"_ns, urlQueryHash))); + + QM_TRY(MOZ_TO_RESULT(state->BindUTF8StringByName("request_url_fragment"_ns, + aRequest.urlFragment()))); + + QM_TRY(MOZ_TO_RESULT( + state->BindStringByName("request_referrer"_ns, aRequest.referrer()))); + + QM_TRY(MOZ_TO_RESULT(state->BindInt32ByName( + "request_referrer_policy"_ns, + static_cast<int32_t>(aRequest.referrerPolicy())))); + + QM_TRY(MOZ_TO_RESULT( + state->BindInt32ByName("request_headers_guard"_ns, + static_cast<int32_t>(aRequest.headersGuard())))); + + QM_TRY(MOZ_TO_RESULT(state->BindInt32ByName( + "request_mode"_ns, static_cast<int32_t>(aRequest.mode())))); + + QM_TRY(MOZ_TO_RESULT( + state->BindInt32ByName("request_credentials"_ns, + static_cast<int32_t>(aRequest.credentials())))); + + QM_TRY(MOZ_TO_RESULT(state->BindInt32ByName( + "request_contentpolicytype"_ns, + static_cast<int32_t>(aRequest.contentPolicyType())))); + + QM_TRY(MOZ_TO_RESULT(state->BindInt32ByName( + "request_cache"_ns, static_cast<int32_t>(aRequest.requestCache())))); + + QM_TRY(MOZ_TO_RESULT(state->BindInt32ByName( + "request_redirect"_ns, + static_cast<int32_t>(aRequest.requestRedirect())))); + + QM_TRY(MOZ_TO_RESULT( + state->BindStringByName("request_integrity"_ns, aRequest.integrity()))); + + QM_TRY(MOZ_TO_RESULT(BindId(*state, "request_body_id"_ns, aRequestBodyId))); + + QM_TRY(MOZ_TO_RESULT(state->BindInt32ByName( + "response_type"_ns, static_cast<int32_t>(aResponse.type())))); + + QM_TRY(MOZ_TO_RESULT( + state->BindInt32ByName("response_status"_ns, aResponse.status()))); + + QM_TRY(MOZ_TO_RESULT(state->BindUTF8StringByName("response_status_text"_ns, + aResponse.statusText()))); + + QM_TRY(MOZ_TO_RESULT(state->BindInt32ByName( + "response_headers_guard"_ns, + static_cast<int32_t>(aResponse.headersGuard())))); + + QM_TRY( + MOZ_TO_RESULT(BindId(*state, "response_body_id"_ns, aResponseBodyId))); + + if (!aResponse.securityInfo()) { + QM_TRY( + MOZ_TO_RESULT(state->BindNullByName("response_security_info_id"_ns))); + } else { + QM_TRY(MOZ_TO_RESULT( + state->BindInt32ByName("response_security_info_id"_ns, securityId))); + } + + nsAutoCString serializedInfo; + // We only allow content serviceworkers right now. + if (aResponse.principalInfo().isSome()) { + const mozilla::ipc::PrincipalInfo& principalInfo = + aResponse.principalInfo().ref(); + MOZ_DIAGNOSTIC_ASSERT(principalInfo.type() == + mozilla::ipc::PrincipalInfo::TContentPrincipalInfo); + const mozilla::ipc::ContentPrincipalInfo& cInfo = + principalInfo.get_ContentPrincipalInfo(); + + serializedInfo.Append(cInfo.spec()); + + nsAutoCString suffix; + cInfo.attrs().CreateSuffix(suffix); + serializedInfo.Append(suffix); + } + + QM_TRY(MOZ_TO_RESULT(state->BindUTF8StringByName( + "response_principal_info"_ns, serializedInfo))); + + if (aResponse.paddingSize() == InternalResponse::UNKNOWN_PADDING_SIZE) { + MOZ_DIAGNOSTIC_ASSERT(aResponse.type() != ResponseType::Opaque); + QM_TRY(MOZ_TO_RESULT(state->BindNullByName("response_padding_size"_ns))); + } else { + MOZ_DIAGNOSTIC_ASSERT(aResponse.paddingSize() >= 0); + MOZ_DIAGNOSTIC_ASSERT(aResponse.type() == ResponseType::Opaque); + + QM_TRY(MOZ_TO_RESULT(state->BindInt64ByName("response_padding_size"_ns, + aResponse.paddingSize()))); + } + + QM_TRY(MOZ_TO_RESULT(state->BindInt64ByName("cache_id"_ns, aCacheId))); + + QM_TRY(MOZ_TO_RESULT(state->Execute())); + } + + QM_TRY_INSPECT( + const int32_t& entryId, ([&aConn]() -> Result<int32_t, nsresult> { + QM_TRY_INSPECT(const auto& state, + quota::CreateAndExecuteSingleStepStatement( + aConn, "SELECT last_insert_rowid()"_ns)); + + QM_TRY_RETURN(MOZ_TO_RESULT_INVOKE_MEMBER(*state, GetInt32, 0)); + }())); + + { + QM_TRY_INSPECT(const auto& state, + MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsCOMPtr<mozIStorageStatement>, aConn, CreateStatement, + "INSERT INTO request_headers (" + "name, " + "value, " + "entry_id " + ") VALUES (:name, :value, :entry_id)"_ns)); + + for (const auto& requestHeader : aRequest.headers()) { + QM_TRY(MOZ_TO_RESULT( + state->BindUTF8StringByName("name"_ns, requestHeader.name()))); + + QM_TRY(MOZ_TO_RESULT( + state->BindUTF8StringByName("value"_ns, requestHeader.value()))); + + QM_TRY(MOZ_TO_RESULT(state->BindInt32ByName("entry_id"_ns, entryId))); + + QM_TRY(MOZ_TO_RESULT(state->Execute())); + } + } + + { + QM_TRY_INSPECT(const auto& state, + MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsCOMPtr<mozIStorageStatement>, aConn, CreateStatement, + "INSERT INTO response_headers (" + "name, " + "value, " + "entry_id " + ") VALUES (:name, :value, :entry_id)"_ns)); + + for (const auto& responseHeader : aResponse.headers()) { + QM_TRY(MOZ_TO_RESULT( + state->BindUTF8StringByName("name"_ns, responseHeader.name()))); + QM_TRY(MOZ_TO_RESULT( + state->BindUTF8StringByName("value"_ns, responseHeader.value()))); + QM_TRY(MOZ_TO_RESULT(state->BindInt32ByName("entry_id"_ns, entryId))); + QM_TRY(MOZ_TO_RESULT(state->Execute())); + } + } + + { + QM_TRY_INSPECT(const auto& state, + MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsCOMPtr<mozIStorageStatement>, aConn, CreateStatement, + "INSERT INTO response_url_list (" + "url, " + "entry_id " + ") VALUES (:url, :entry_id)"_ns)); + + for (const auto& responseUrl : aResponse.urlList()) { + QM_TRY(MOZ_TO_RESULT(state->BindUTF8StringByName("url"_ns, responseUrl))); + QM_TRY(MOZ_TO_RESULT(state->BindInt32ByName("entry_id"_ns, entryId))); + QM_TRY(MOZ_TO_RESULT(state->Execute())); + } + } + + return NS_OK; +} + +/** + * Gets a HeadersEntry from a storage statement by retrieving the first column + * as the name and the second column as the value. + */ +Result<HeadersEntry, nsresult> GetHeadersEntryFromStatement( + mozIStorageStatement& aStmt) { + HeadersEntry header; + + QM_TRY_UNWRAP(header.name(), MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsCString, aStmt, GetUTF8String, 0)); + QM_TRY_UNWRAP(header.value(), MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsCString, aStmt, GetUTF8String, 1)); + + return header; +} + +Result<SavedResponse, nsresult> ReadResponse(mozIStorageConnection& aConn, + EntryId aEntryId) { + MOZ_ASSERT(!NS_IsMainThread()); + + SavedResponse savedResponse; + + QM_TRY_INSPECT( + const auto& state, + quota::CreateAndExecuteSingleStepStatement( + aConn, + "SELECT " + "entries.response_type, " + "entries.response_status, " + "entries.response_status_text, " + "entries.response_headers_guard, " + "entries.response_body_id, " + "entries.response_principal_info, " + "entries.response_padding_size, " + "security_info.data, " + "entries.request_credentials " + "FROM entries " + "LEFT OUTER JOIN security_info " + "ON entries.response_security_info_id=security_info.id " + "WHERE entries.id=:id;"_ns, + [aEntryId](auto& state) -> Result<Ok, nsresult> { + QM_TRY(MOZ_TO_RESULT(state.BindInt32ByName("id"_ns, aEntryId))); + + return Ok{}; + })); + + QM_TRY_INSPECT(const int32_t& type, + MOZ_TO_RESULT_INVOKE_MEMBER(*state, GetInt32, 0)); + savedResponse.mValue.type() = static_cast<ResponseType>(type); + + QM_TRY_INSPECT(const int32_t& status, + MOZ_TO_RESULT_INVOKE_MEMBER(*state, GetInt32, 1)); + savedResponse.mValue.status() = static_cast<uint32_t>(status); + + QM_TRY(MOZ_TO_RESULT( + state->GetUTF8String(2, savedResponse.mValue.statusText()))); + + QM_TRY_INSPECT(const int32_t& guard, + MOZ_TO_RESULT_INVOKE_MEMBER(*state, GetInt32, 3)); + savedResponse.mValue.headersGuard() = static_cast<HeadersGuardEnum>(guard); + + QM_TRY_INSPECT(const bool& nullBody, + MOZ_TO_RESULT_INVOKE_MEMBER(*state, GetIsNull, 4)); + savedResponse.mHasBodyId = !nullBody; + + if (savedResponse.mHasBodyId) { + QM_TRY_UNWRAP(savedResponse.mBodyId, ExtractId(*state, 4)); + } + + QM_TRY_INSPECT(const auto& serializedInfo, + MOZ_TO_RESULT_INVOKE_MEMBER_TYPED(nsAutoCString, *state, + GetUTF8String, 5)); + + savedResponse.mValue.principalInfo() = Nothing(); + if (!serializedInfo.IsEmpty()) { + nsAutoCString specNoSuffix; + OriginAttributes attrs; + if (!attrs.PopulateFromOrigin(serializedInfo, specNoSuffix)) { + NS_WARNING("Something went wrong parsing a serialized principal!"); + return Err(NS_ERROR_FAILURE); + } + + RefPtr<net::MozURL> url; + QM_TRY(MOZ_TO_RESULT(net::MozURL::Init(getter_AddRefs(url), specNoSuffix))); + +#ifdef DEBUG + nsDependentCSubstring scheme = url->Scheme(); + + MOZ_ASSERT( + scheme == "http" || scheme == "https" || scheme == "file" || + // A cached response entry may have a moz-extension principal if: + // + // - This is an extension background service worker. The response for + // the main script is expected tobe a moz-extension content principal + // (the pref "extensions.backgroundServiceWorker.enabled" must be + // enabled, if the pref is toggled to false at runtime then any + // service worker registered for a moz-extension principal will be + // unregistered on the next startup). + // + // - An extension is redirecting a script being imported info a worker + // created from a regular webpage to a web-accessible extension + // script. The reponse for these redirects will have a moz-extension + // principal. Although extensions can attempt to redirect the main + // script of service workers, this will always cause the install + // process to fail. + scheme == "moz-extension"); +#endif + + nsCString origin; + url->Origin(origin); + + nsCString baseDomain; + QM_TRY(MOZ_TO_RESULT(url->BaseDomain(baseDomain))); + + savedResponse.mValue.principalInfo() = + Some(mozilla::ipc::ContentPrincipalInfo(attrs, origin, specNoSuffix, + Nothing(), baseDomain)); + } + + QM_TRY_INSPECT(const bool& nullPadding, + MOZ_TO_RESULT_INVOKE_MEMBER(*state, GetIsNull, 6)); + + if (nullPadding) { + MOZ_DIAGNOSTIC_ASSERT(savedResponse.mValue.type() != ResponseType::Opaque); + savedResponse.mValue.paddingSize() = InternalResponse::UNKNOWN_PADDING_SIZE; + } else { + MOZ_DIAGNOSTIC_ASSERT(savedResponse.mValue.type() == ResponseType::Opaque); + QM_TRY_INSPECT(const int64_t& paddingSize, + MOZ_TO_RESULT_INVOKE_MEMBER(*state, GetInt64, 6)); + + MOZ_DIAGNOSTIC_ASSERT(paddingSize >= 0); + savedResponse.mValue.paddingSize() = paddingSize; + } + + nsCString data; + QM_TRY(MOZ_TO_RESULT(state->GetBlobAsUTF8String(7, data))); + if (!data.IsEmpty()) { + nsCOMPtr<nsITransportSecurityInfo> securityInfo; + nsresult rv = mozilla::psm::TransportSecurityInfo::Read( + data, getter_AddRefs(securityInfo)); + if (NS_FAILED(rv)) { + return Err(rv); + } + if (!securityInfo) { + return Err(NS_ERROR_FAILURE); + } + savedResponse.mValue.securityInfo() = securityInfo.forget(); + } + + QM_TRY_INSPECT(const int32_t& credentials, + MOZ_TO_RESULT_INVOKE_MEMBER(*state, GetInt32, 8)); + savedResponse.mValue.credentials() = + static_cast<RequestCredentials>(credentials); + + { + QM_TRY_INSPECT(const auto& state, + MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsCOMPtr<mozIStorageStatement>, aConn, CreateStatement, + "SELECT " + "name, " + "value " + "FROM response_headers " + "WHERE entry_id=:entry_id;"_ns)); + + QM_TRY(MOZ_TO_RESULT(state->BindInt32ByName("entry_id"_ns, aEntryId))); + + QM_TRY_UNWRAP(savedResponse.mValue.headers(), + quota::CollectElementsWhileHasResult( + *state, GetHeadersEntryFromStatement)); + } + + { + QM_TRY_INSPECT(const auto& state, + MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsCOMPtr<mozIStorageStatement>, aConn, CreateStatement, + "SELECT " + "url " + "FROM response_url_list " + "WHERE entry_id=:entry_id;"_ns)); + + QM_TRY(MOZ_TO_RESULT(state->BindInt32ByName("entry_id"_ns, aEntryId))); + + QM_TRY_UNWRAP(savedResponse.mValue.urlList(), + quota::CollectElementsWhileHasResult( + *state, [](auto& stmt) -> Result<nsCString, nsresult> { + QM_TRY_RETURN(MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsCString, stmt, GetUTF8String, 0)); + })); + } + + return savedResponse; +} + +Result<SavedRequest, nsresult> ReadRequest(mozIStorageConnection& aConn, + EntryId aEntryId) { + MOZ_ASSERT(!NS_IsMainThread()); + + SavedRequest savedRequest; + + QM_TRY_INSPECT( + const auto& state, + quota::CreateAndExecuteSingleStepStatement< + quota::SingleStepResult::ReturnNullIfNoResult>( + aConn, + "SELECT " + "request_method, " + "request_url_no_query, " + "request_url_query, " + "request_url_fragment, " + "request_referrer, " + "request_referrer_policy, " + "request_headers_guard, " + "request_mode, " + "request_credentials, " + "request_contentpolicytype, " + "request_cache, " + "request_redirect, " + "request_integrity, " + "request_body_id " + "FROM entries " + "WHERE id=:id;"_ns, + [aEntryId](auto& state) -> Result<Ok, nsresult> { + QM_TRY(MOZ_TO_RESULT(state.BindInt32ByName("id"_ns, aEntryId))); + + return Ok{}; + })); + + QM_TRY(OkIf(state), Err(NS_ERROR_UNEXPECTED)); + + QM_TRY(MOZ_TO_RESULT(state->GetUTF8String(0, savedRequest.mValue.method()))); + QM_TRY(MOZ_TO_RESULT( + state->GetUTF8String(1, savedRequest.mValue.urlWithoutQuery()))); + QM_TRY( + MOZ_TO_RESULT(state->GetUTF8String(2, savedRequest.mValue.urlQuery()))); + QM_TRY(MOZ_TO_RESULT( + state->GetUTF8String(3, savedRequest.mValue.urlFragment()))); + QM_TRY(MOZ_TO_RESULT(state->GetString(4, savedRequest.mValue.referrer()))); + + QM_TRY_INSPECT(const int32_t& referrerPolicy, + MOZ_TO_RESULT_INVOKE_MEMBER(state, GetInt32, 5)); + savedRequest.mValue.referrerPolicy() = + static_cast<ReferrerPolicy>(referrerPolicy); + + QM_TRY_INSPECT(const int32_t& guard, + MOZ_TO_RESULT_INVOKE_MEMBER(state, GetInt32, 6)); + savedRequest.mValue.headersGuard() = static_cast<HeadersGuardEnum>(guard); + + QM_TRY_INSPECT(const int32_t& mode, + MOZ_TO_RESULT_INVOKE_MEMBER(state, GetInt32, 7)); + savedRequest.mValue.mode() = static_cast<RequestMode>(mode); + + QM_TRY_INSPECT(const int32_t& credentials, + MOZ_TO_RESULT_INVOKE_MEMBER(state, GetInt32, 8)); + savedRequest.mValue.credentials() = + static_cast<RequestCredentials>(credentials); + + QM_TRY_INSPECT(const int32_t& requestContentPolicyType, + MOZ_TO_RESULT_INVOKE_MEMBER(state, GetInt32, 9)); + savedRequest.mValue.contentPolicyType() = + static_cast<nsContentPolicyType>(requestContentPolicyType); + + QM_TRY_INSPECT(const int32_t& requestCache, + MOZ_TO_RESULT_INVOKE_MEMBER(state, GetInt32, 10)); + savedRequest.mValue.requestCache() = static_cast<RequestCache>(requestCache); + + QM_TRY_INSPECT(const int32_t& requestRedirect, + MOZ_TO_RESULT_INVOKE_MEMBER(state, GetInt32, 11)); + savedRequest.mValue.requestRedirect() = + static_cast<RequestRedirect>(requestRedirect); + + QM_TRY(MOZ_TO_RESULT(state->GetString(12, savedRequest.mValue.integrity()))); + + QM_TRY_INSPECT(const bool& nullBody, + MOZ_TO_RESULT_INVOKE_MEMBER(state, GetIsNull, 13)); + savedRequest.mHasBodyId = !nullBody; + if (savedRequest.mHasBodyId) { + QM_TRY_UNWRAP(savedRequest.mBodyId, ExtractId(*state, 13)); + } + + { + QM_TRY_INSPECT(const auto& state, + MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsCOMPtr<mozIStorageStatement>, aConn, CreateStatement, + "SELECT " + "name, " + "value " + "FROM request_headers " + "WHERE entry_id=:entry_id;"_ns)); + + QM_TRY(MOZ_TO_RESULT(state->BindInt32ByName("entry_id"_ns, aEntryId))); + + QM_TRY_UNWRAP(savedRequest.mValue.headers(), + quota::CollectElementsWhileHasResult( + *state, GetHeadersEntryFromStatement)); + } + + return savedRequest; +} + +void AppendListParamsToQuery(nsACString& aQuery, size_t aLen) { + MOZ_ASSERT(!NS_IsMainThread()); + + aQuery.AppendLiteral("?"); + for (size_t i = 1; i < aLen; ++i) { + aQuery.AppendLiteral(",?"); + } +} + +nsresult BindListParamsToQuery(mozIStorageStatement& aState, + const Span<const EntryId>& aEntryIdList) { + MOZ_ASSERT(!NS_IsMainThread()); + for (size_t i = 0, n = aEntryIdList.Length(); i < n; ++i) { + QM_TRY(MOZ_TO_RESULT(aState.BindInt32ByIndex(i, aEntryIdList[i]))); + } + return NS_OK; +} + +nsresult BindId(mozIStorageStatement& aState, const nsACString& aName, + const nsID* aId) { + MOZ_ASSERT(!NS_IsMainThread()); + + if (!aId) { + QM_TRY(MOZ_TO_RESULT(aState.BindNullByName(aName))); + return NS_OK; + } + + char idBuf[NSID_LENGTH]; + aId->ToProvidedString(idBuf); + QM_TRY(MOZ_TO_RESULT( + aState.BindUTF8StringByName(aName, nsDependentCString(idBuf)))); + + return NS_OK; +} + +Result<nsID, nsresult> ExtractId(mozIStorageStatement& aState, uint32_t aPos) { + MOZ_ASSERT(!NS_IsMainThread()); + + QM_TRY_INSPECT(const auto& idString, + MOZ_TO_RESULT_INVOKE_MEMBER_TYPED(nsAutoCString, aState, + GetUTF8String, aPos)); + + nsID id; + QM_TRY(OkIf(id.Parse(idString.get())), Err(NS_ERROR_UNEXPECTED)); + + return id; +} + +Result<NotNull<nsCOMPtr<mozIStorageStatement>>, nsresult> +CreateAndBindKeyStatement(mozIStorageConnection& aConn, + const char* const aQueryFormat, + const nsAString& aKey) { + MOZ_DIAGNOSTIC_ASSERT(aQueryFormat); + + // The key is stored as a blob to avoid encoding issues. An empty string + // is mapped to NULL for blobs. Normally we would just write the query + // as "key IS :key" to do the proper NULL checking, but that prevents + // sqlite from using the key index. Therefore use "IS NULL" explicitly + // if the key is empty, otherwise use "=:key" so that sqlite uses the + // index. + + QM_TRY_UNWRAP( + auto state, + MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsCOMPtr<mozIStorageStatement>, aConn, CreateStatement, + nsPrintfCString(aQueryFormat, + aKey.IsEmpty() ? "key IS NULL" : "key=:key"))); + + if (!aKey.IsEmpty()) { + QM_TRY(MOZ_TO_RESULT(state->BindStringAsBlobByName("key"_ns, aKey))); + } + + return WrapNotNull(std::move(state)); +} + +Result<nsAutoCString, nsresult> HashCString(nsICryptoHash& aCrypto, + const nsACString& aIn) { + QM_TRY(MOZ_TO_RESULT(aCrypto.Init(nsICryptoHash::SHA1))); + + QM_TRY(MOZ_TO_RESULT(aCrypto.Update( + reinterpret_cast<const uint8_t*>(aIn.BeginReading()), aIn.Length()))); + + nsAutoCString fullHash; + QM_TRY(MOZ_TO_RESULT(aCrypto.Finish(false /* based64 result */, fullHash))); + + return Result<nsAutoCString, nsresult>{std::in_place, + Substring(fullHash, 0, 8)}; +} + +} // namespace + +nsresult IncrementalVacuum(mozIStorageConnection& aConn) { + // Determine how much free space is in the database. + QM_TRY_INSPECT(const auto& state, quota::CreateAndExecuteSingleStepStatement( + aConn, "PRAGMA freelist_count;"_ns)); + + QM_TRY_INSPECT(const int32_t& freePages, + MOZ_TO_RESULT_INVOKE_MEMBER(*state, GetInt32, 0)); + + // We have a relatively small page size, so we want to be careful to avoid + // fragmentation. We already use a growth incremental which will cause + // sqlite to allocate and release multiple pages at the same time. We can + // further reduce fragmentation by making our allocated chunks a bit + // "sticky". This is done by creating some hysteresis where we allocate + // pages/chunks as soon as we need them, but we only release pages/chunks + // when we have a large amount of free space. This helps with the case + // where a page is adding and remove resources causing it to dip back and + // forth across a chunk boundary. + // + // So only proceed with releasing pages if we have more than our constant + // threshold. + if (freePages <= kMaxFreePages) { + return NS_OK; + } + + // Release the excess pages back to the sqlite VFS. This may also release + // chunks of multiple pages back to the OS. + const int32_t pagesToRelease = freePages - kMaxFreePages; + + QM_TRY(MOZ_TO_RESULT(aConn.ExecuteSimpleSQL( + nsPrintfCString("PRAGMA incremental_vacuum(%d);", pagesToRelease)))); + + // Verify that our incremental vacuum actually did something +#ifdef DEBUG + { + QM_TRY_INSPECT(const auto& state, + quota::CreateAndExecuteSingleStepStatement( + aConn, "PRAGMA freelist_count;"_ns)); + + QM_TRY_INSPECT(const int32_t& freePages, + MOZ_TO_RESULT_INVOKE_MEMBER(*state, GetInt32, 0)); + + MOZ_ASSERT(freePages <= kMaxFreePages); + } +#endif + + return NS_OK; +} + +namespace { + +// Wrapper around mozIStorageConnection::GetSchemaVersion() that compensates +// for hacky downgrade schema version tricks. See the block comments for +// kHackyDowngradeSchemaVersion and kHackyPaddingSizePresentVersion. +Result<int32_t, nsresult> GetEffectiveSchemaVersion( + mozIStorageConnection& aConn) { + QM_TRY_INSPECT(const int32_t& schemaVersion, + MOZ_TO_RESULT_INVOKE_MEMBER(aConn, GetSchemaVersion)); + + if (schemaVersion == kHackyDowngradeSchemaVersion) { + // This is the special case. Check for the existence of the + // "response_padding_size" colum in table "entries". + // + // (pragma_table_info is a table-valued function format variant of + // "PRAGMA table_info" supported since SQLite 3.16.0. Firefox 53 shipped + // was the first release with this functionality, shipping 3.16.2.) + // + // If there are any result rows, then the column is present. + QM_TRY_INSPECT(const bool& hasColumn, + quota::CreateAndExecuteSingleStepStatement< + quota::SingleStepResult::ReturnNullIfNoResult>( + aConn, + "SELECT name FROM pragma_table_info('entries') WHERE " + "name = 'response_padding_size'"_ns)); + + if (hasColumn) { + return kHackyPaddingSizePresentVersion; + } + } + + return schemaVersion; +} + +#ifdef DEBUG +struct Expect { + // Expect exact SQL + Expect(const char* aName, const char* aType, const char* aSql) + : mName(aName), mType(aType), mSql(aSql), mIgnoreSql(false) {} + + // Ignore SQL + Expect(const char* aName, const char* aType) + : mName(aName), mType(aType), mIgnoreSql(true) {} + + const nsCString mName; + const nsCString mType; + const nsCString mSql; + const bool mIgnoreSql; +}; +#endif + +nsresult Validate(mozIStorageConnection& aConn) { + QM_TRY_INSPECT(const int32_t& schemaVersion, + GetEffectiveSchemaVersion(aConn)); + QM_TRY(OkIf(schemaVersion == kLatestSchemaVersion), NS_ERROR_FAILURE); + +#ifdef DEBUG + // This is the schema we expect the database at the latest version to + // contain. Update this list if you add a new table or index. + const Expect expects[] = { + Expect("caches", "table", kTableCaches), + Expect("sqlite_sequence", "table"), // auto-gen by sqlite + Expect("security_info", "table", kTableSecurityInfo), + Expect("security_info_hash_index", "index", kIndexSecurityInfoHash), + Expect("entries", "table", kTableEntries), + Expect("entries_request_match_index", "index", kIndexEntriesRequest), + Expect("request_headers", "table", kTableRequestHeaders), + Expect("response_headers", "table", kTableResponseHeaders), + Expect("response_headers_name_index", "index", kIndexResponseHeadersName), + Expect("response_url_list", "table", kTableResponseUrlList), + Expect("storage", "table", kTableStorage), + Expect("sqlite_autoindex_storage_1", "index"), // auto-gen by sqlite + }; + + // Read the schema from the sqlite_master table and compare. + QM_TRY_INSPECT(const auto& state, + MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsCOMPtr<mozIStorageStatement>, aConn, CreateStatement, + "SELECT name, type, sql FROM sqlite_master;"_ns)); + + QM_TRY(quota::CollectWhileHasResult( + *state, [&expects](auto& stmt) -> Result<Ok, nsresult> { + QM_TRY_INSPECT(const auto& name, + MOZ_TO_RESULT_INVOKE_MEMBER_TYPED(nsAutoCString, stmt, + GetUTF8String, 0)); + QM_TRY_INSPECT(const auto& type, + MOZ_TO_RESULT_INVOKE_MEMBER_TYPED(nsAutoCString, stmt, + GetUTF8String, 1)); + QM_TRY_INSPECT(const auto& sql, + MOZ_TO_RESULT_INVOKE_MEMBER_TYPED(nsAutoCString, stmt, + GetUTF8String, 2)); + + bool foundMatch = false; + for (const auto& expect : expects) { + if (name == expect.mName) { + if (type != expect.mType) { + NS_WARNING( + nsPrintfCString("Unexpected type for Cache schema entry %s", + name.get()) + .get()); + return Err(NS_ERROR_FAILURE); + } + + if (!expect.mIgnoreSql && sql != expect.mSql) { + NS_WARNING( + nsPrintfCString("Unexpected SQL for Cache schema entry %s", + name.get()) + .get()); + return Err(NS_ERROR_FAILURE); + } + + foundMatch = true; + break; + } + } + + if (NS_WARN_IF(!foundMatch)) { + NS_WARNING( + nsPrintfCString("Unexpected schema entry %s in Cache database", + name.get()) + .get()); + return Err(NS_ERROR_FAILURE); + } + + return Ok{}; + })); +#endif + + return NS_OK; +} + +// ----- +// Schema migration code +// ----- + +using MigrationFunc = nsresult (*)(mozIStorageConnection&, bool&); +struct Migration { + int32_t mFromVersion; + MigrationFunc mFunc; +}; + +// Declare migration functions here. Each function should upgrade +// the version by a single increment. Don't skip versions. +nsresult MigrateFrom15To16(mozIStorageConnection& aConn, bool& aRewriteSchema); +nsresult MigrateFrom16To17(mozIStorageConnection& aConn, bool& aRewriteSchema); +nsresult MigrateFrom17To18(mozIStorageConnection& aConn, bool& aRewriteSchema); +nsresult MigrateFrom18To19(mozIStorageConnection& aConn, bool& aRewriteSchema); +nsresult MigrateFrom19To20(mozIStorageConnection& aConn, bool& aRewriteSchema); +nsresult MigrateFrom20To21(mozIStorageConnection& aConn, bool& aRewriteSchema); +nsresult MigrateFrom21To22(mozIStorageConnection& aConn, bool& aRewriteSchema); +nsresult MigrateFrom22To23(mozIStorageConnection& aConn, bool& aRewriteSchema); +nsresult MigrateFrom23To24(mozIStorageConnection& aConn, bool& aRewriteSchema); +nsresult MigrateFrom24To25(mozIStorageConnection& aConn, bool& aRewriteSchema); +nsresult MigrateFrom25To26(mozIStorageConnection& aConn, bool& aRewriteSchema); +nsresult MigrateFrom26To27(mozIStorageConnection& aConn, bool& aRewriteSchema); +nsresult MigrateFrom27To28(mozIStorageConnection& aConn, bool& aRewriteSchema); +// Configure migration functions to run for the given starting version. +constexpr Migration sMigrationList[] = { + Migration{15, MigrateFrom15To16}, Migration{16, MigrateFrom16To17}, + Migration{17, MigrateFrom17To18}, Migration{18, MigrateFrom18To19}, + Migration{19, MigrateFrom19To20}, Migration{20, MigrateFrom20To21}, + Migration{21, MigrateFrom21To22}, Migration{22, MigrateFrom22To23}, + Migration{23, MigrateFrom23To24}, Migration{24, MigrateFrom24To25}, + Migration{25, MigrateFrom25To26}, Migration{26, MigrateFrom26To27}, + Migration{27, MigrateFrom27To28}, +}; + +nsresult RewriteEntriesSchema(mozIStorageConnection& aConn) { + QM_TRY( + MOZ_TO_RESULT(aConn.ExecuteSimpleSQL("PRAGMA writable_schema = ON"_ns))); + + QM_TRY_INSPECT( + const auto& state, + MOZ_TO_RESULT_INVOKE_MEMBER_TYPED( + nsCOMPtr<mozIStorageStatement>, aConn, CreateStatement, + "UPDATE sqlite_master SET sql=:sql WHERE name='entries'"_ns)); + + QM_TRY(MOZ_TO_RESULT(state->BindUTF8StringByName( + "sql"_ns, nsDependentCString(kTableEntries)))); + QM_TRY(MOZ_TO_RESULT(state->Execute())); + + QM_TRY( + MOZ_TO_RESULT(aConn.ExecuteSimpleSQL("PRAGMA writable_schema = OFF"_ns))); + + return NS_OK; +} + +nsresult Migrate(mozIStorageConnection& aConn) { + MOZ_ASSERT(!NS_IsMainThread()); + + QM_TRY_UNWRAP(int32_t currentVersion, GetEffectiveSchemaVersion(aConn)); + + bool rewriteSchema = false; + + while (currentVersion < kLatestSchemaVersion) { + // Wiping old databases is handled in DBAction because it requires + // making a whole new mozIStorageConnection. Make sure we don't + // accidentally get here for one of those old databases. + MOZ_DIAGNOSTIC_ASSERT(currentVersion >= kFirstShippedSchemaVersion); + + for (const auto& migration : sMigrationList) { + if (migration.mFromVersion == currentVersion) { + bool shouldRewrite = false; + QM_TRY(MOZ_TO_RESULT(migration.mFunc(aConn, shouldRewrite))); + if (shouldRewrite) { + rewriteSchema = true; + } + break; + } + } + +#ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED + int32_t lastVersion = currentVersion; +#endif + QM_TRY_UNWRAP(currentVersion, GetEffectiveSchemaVersion(aConn)); + + MOZ_DIAGNOSTIC_ASSERT(currentVersion > lastVersion); + } + + // Don't release assert this since people do sometimes share profiles + // across schema versions. Our check in Validate() will catch it. + MOZ_ASSERT(currentVersion == kLatestSchemaVersion); + + nsresult rv = NS_OK; + if (rewriteSchema) { + // Now overwrite the master SQL for the entries table to remove the column + // default value. This is also necessary for our Validate() method to + // pass on this database. + rv = RewriteEntriesSchema(aConn); + } + + return rv; +} + +nsresult MigrateFrom15To16(mozIStorageConnection& aConn, bool& aRewriteSchema) { + MOZ_ASSERT(!NS_IsMainThread()); + + // Add the request_redirect column with a default value of "follow". Note, + // we only use a default value here because its required by ALTER TABLE and + // we need to apply the default "follow" to existing records in the table. + // We don't actually want to keep the default in the schema for future + // INSERTs. + QM_TRY(MOZ_TO_RESULT(aConn.ExecuteSimpleSQL( + "ALTER TABLE entries " + "ADD COLUMN request_redirect INTEGER NOT NULL DEFAULT 0"_ns))); + + QM_TRY(MOZ_TO_RESULT(aConn.SetSchemaVersion(16))); + + aRewriteSchema = true; + + return NS_OK; +} + +nsresult MigrateFrom16To17(mozIStorageConnection& aConn, bool& aRewriteSchema) { + MOZ_ASSERT(!NS_IsMainThread()); + + // This migration path removes the response_redirected and + // response_redirected_url columns from the entries table. sqlite doesn't + // support removing a column from a table using ALTER TABLE, so we need to + // create a new table without those columns, fill it up with the existing + // data, and then drop the original table and rename the new one to the old + // one. + + // Create a new_entries table with the new fields as of version 17. + QM_TRY(MOZ_TO_RESULT(aConn.ExecuteSimpleSQL( + "CREATE TABLE new_entries (" + "id INTEGER NOT NULL PRIMARY KEY, " + "request_method TEXT NOT NULL, " + "request_url_no_query TEXT NOT NULL, " + "request_url_no_query_hash BLOB NOT NULL, " + "request_url_query TEXT NOT NULL, " + "request_url_query_hash BLOB NOT NULL, " + "request_referrer TEXT NOT NULL, " + "request_headers_guard INTEGER NOT NULL, " + "request_mode INTEGER NOT NULL, " + "request_credentials INTEGER NOT NULL, " + "request_contentpolicytype INTEGER NOT NULL, " + "request_cache INTEGER NOT NULL, " + "request_body_id TEXT NULL, " + "response_type INTEGER NOT NULL, " + "response_url TEXT NOT NULL, " + "response_status INTEGER NOT NULL, " + "response_status_text TEXT NOT NULL, " + "response_headers_guard INTEGER NOT NULL, " + "response_body_id TEXT NULL, " + "response_security_info_id INTEGER NULL REFERENCES security_info(id), " + "response_principal_info TEXT NOT NULL, " + "cache_id INTEGER NOT NULL REFERENCES caches(id) ON DELETE CASCADE, " + "request_redirect INTEGER NOT NULL" + ")"_ns))); + + // Copy all of the data to the newly created table. + QM_TRY( + MOZ_TO_RESULT(aConn.ExecuteSimpleSQL("INSERT INTO new_entries (" + "id, " + "request_method, " + "request_url_no_query, " + "request_url_no_query_hash, " + "request_url_query, " + "request_url_query_hash, " + "request_referrer, " + "request_headers_guard, " + "request_mode, " + "request_credentials, " + "request_contentpolicytype, " + "request_cache, " + "request_redirect, " + "request_body_id, " + "response_type, " + "response_url, " + "response_status, " + "response_status_text, " + "response_headers_guard, " + "response_body_id, " + "response_security_info_id, " + "response_principal_info, " + "cache_id " + ") SELECT " + "id, " + "request_method, " + "request_url_no_query, " + "request_url_no_query_hash, " + "request_url_query, " + "request_url_query_hash, " + "request_referrer, " + "request_headers_guard, " + "request_mode, " + "request_credentials, " + "request_contentpolicytype, " + "request_cache, " + "request_redirect, " + "request_body_id, " + "response_type, " + "response_url, " + "response_status, " + "response_status_text, " + "response_headers_guard, " + "response_body_id, " + "response_security_info_id, " + "response_principal_info, " + "cache_id " + "FROM entries;"_ns))); + + // Remove the old table. + QM_TRY(MOZ_TO_RESULT(aConn.ExecuteSimpleSQL("DROP TABLE entries;"_ns))); + + // Rename new_entries to entries. + QM_TRY(MOZ_TO_RESULT( + aConn.ExecuteSimpleSQL("ALTER TABLE new_entries RENAME to entries;"_ns))); + + // Now, recreate our indices. + QM_TRY(MOZ_TO_RESULT( + aConn.ExecuteSimpleSQL(nsDependentCString(kIndexEntriesRequest)))); + + // Revalidate the foreign key constraints, and ensure that there are no + // violations. + QM_TRY_INSPECT(const bool& hasResult, + quota::CreateAndExecuteSingleStepStatement< + quota::SingleStepResult::ReturnNullIfNoResult>( + aConn, "PRAGMA foreign_key_check;"_ns)); + + QM_TRY(OkIf(!hasResult), NS_ERROR_FAILURE); + + QM_TRY(MOZ_TO_RESULT(aConn.SetSchemaVersion(17))); + + return NS_OK; +} + +nsresult MigrateFrom17To18(mozIStorageConnection& aConn, bool& aRewriteSchema) { + MOZ_ASSERT(!NS_IsMainThread()); + + // This migration is needed in order to remove "only-if-cached" RequestCache + // values from the database. This enum value was removed from the spec in + // https://github.com/whatwg/fetch/issues/39 but we unfortunately happily + // accepted this value in the Request constructor. + // + // There is no good value to upgrade this to, so we just stick to "default". + + static_assert(int(RequestCache::Default) == 0, + "This is where the 0 below comes from!"); + QM_TRY(MOZ_TO_RESULT( + aConn.ExecuteSimpleSQL("UPDATE entries SET request_cache = 0 " + "WHERE request_cache = 5;"_ns))); + + QM_TRY(MOZ_TO_RESULT(aConn.SetSchemaVersion(18))); + + return NS_OK; +} + +nsresult MigrateFrom18To19(mozIStorageConnection& aConn, bool& aRewriteSchema) { + MOZ_ASSERT(!NS_IsMainThread()); + + // This migration is needed in order to update the RequestMode values for + // Request objects corresponding to a navigation content policy type to + // "navigate". + + static_assert(int(nsIContentPolicy::TYPE_DOCUMENT) == 6 && + int(nsIContentPolicy::TYPE_SUBDOCUMENT) == 7 && + int(nsIContentPolicy::TYPE_INTERNAL_FRAME) == 28 && + int(nsIContentPolicy::TYPE_INTERNAL_IFRAME) == 29 && + int(RequestMode::Navigate) == 3, + "This is where the numbers below come from!"); + // 8 is former TYPE_REFRESH. + + QM_TRY(MOZ_TO_RESULT(aConn.ExecuteSimpleSQL( + "UPDATE entries SET request_mode = 3 " + "WHERE request_contentpolicytype IN (6, 7, 28, 29, 8);"_ns))); + + QM_TRY(MOZ_TO_RESULT(aConn.SetSchemaVersion(19))); + + return NS_OK; +} + +nsresult MigrateFrom19To20(mozIStorageConnection& aConn, bool& aRewriteSchema) { + MOZ_ASSERT(!NS_IsMainThread()); + + // Add the request_referrer_policy column with a default value of + // "no-referrer-when-downgrade". Note, we only use a default value here + // because its required by ALTER TABLE and we need to apply the default + // "no-referrer-when-downgrade" to existing records in the table. We don't + // actually want to keep the default in the schema for future INSERTs. + QM_TRY(MOZ_TO_RESULT(aConn.ExecuteSimpleSQL( + "ALTER TABLE entries " + "ADD COLUMN request_referrer_policy INTEGER NOT NULL DEFAULT 2"_ns))); + + QM_TRY(MOZ_TO_RESULT(aConn.SetSchemaVersion(20))); + + aRewriteSchema = true; + + return NS_OK; +} + +nsresult MigrateFrom20To21(mozIStorageConnection& aConn, bool& aRewriteSchema) { + MOZ_ASSERT(!NS_IsMainThread()); + + // This migration creates response_url_list table to store response_url and + // removes the response_url column from the entries table. + // sqlite doesn't support removing a column from a table using ALTER TABLE, + // so we need to create a new table without those columns, fill it up with the + // existing data, and then drop the original table and rename the new one to + // the old one. + + // Create a new_entries table with the new fields as of version 21. + QM_TRY(MOZ_TO_RESULT(aConn.ExecuteSimpleSQL( + "CREATE TABLE new_entries (" + "id INTEGER NOT NULL PRIMARY KEY, " + "request_method TEXT NOT NULL, " + "request_url_no_query TEXT NOT NULL, " + "request_url_no_query_hash BLOB NOT NULL, " + "request_url_query TEXT NOT NULL, " + "request_url_query_hash BLOB NOT NULL, " + "request_referrer TEXT NOT NULL, " + "request_headers_guard INTEGER NOT NULL, " + "request_mode INTEGER NOT NULL, " + "request_credentials INTEGER NOT NULL, " + "request_contentpolicytype INTEGER NOT NULL, " + "request_cache INTEGER NOT NULL, " + "request_body_id TEXT NULL, " + "response_type INTEGER NOT NULL, " + "response_status INTEGER NOT NULL, " + "response_status_text TEXT NOT NULL, " + "response_headers_guard INTEGER NOT NULL, " + "response_body_id TEXT NULL, " + "response_security_info_id INTEGER NULL REFERENCES security_info(id), " + "response_principal_info TEXT NOT NULL, " + "cache_id INTEGER NOT NULL REFERENCES caches(id) ON DELETE CASCADE, " + "request_redirect INTEGER NOT NULL, " + "request_referrer_policy INTEGER NOT NULL" + ")"_ns))); + + // Create a response_url_list table with the new fields as of version 21. + QM_TRY(MOZ_TO_RESULT(aConn.ExecuteSimpleSQL( + "CREATE TABLE response_url_list (" + "url TEXT NOT NULL, " + "entry_id INTEGER NOT NULL REFERENCES entries(id) ON DELETE CASCADE" + ")"_ns))); + + // Copy all of the data to the newly created entries table. + QM_TRY( + MOZ_TO_RESULT(aConn.ExecuteSimpleSQL("INSERT INTO new_entries (" + "id, " + "request_method, " + "request_url_no_query, " + "request_url_no_query_hash, " + "request_url_query, " + "request_url_query_hash, " + "request_referrer, " + "request_headers_guard, " + "request_mode, " + "request_credentials, " + "request_contentpolicytype, " + "request_cache, " + "request_redirect, " + "request_referrer_policy, " + "request_body_id, " + "response_type, " + "response_status, " + "response_status_text, " + "response_headers_guard, " + "response_body_id, " + "response_security_info_id, " + "response_principal_info, " + "cache_id " + ") SELECT " + "id, " + "request_method, " + "request_url_no_query, " + "request_url_no_query_hash, " + "request_url_query, " + "request_url_query_hash, " + "request_referrer, " + "request_headers_guard, " + "request_mode, " + "request_credentials, " + "request_contentpolicytype, " + "request_cache, " + "request_redirect, " + "request_referrer_policy, " + "request_body_id, " + "response_type, " + "response_status, " + "response_status_text, " + "response_headers_guard, " + "response_body_id, " + "response_security_info_id, " + "response_principal_info, " + "cache_id " + "FROM entries;"_ns))); + + // Copy reponse_url to the newly created response_url_list table. + QM_TRY( + MOZ_TO_RESULT(aConn.ExecuteSimpleSQL("INSERT INTO response_url_list (" + "url, " + "entry_id " + ") SELECT " + "response_url, " + "id " + "FROM entries;"_ns))); + + // Remove the old table. + QM_TRY(MOZ_TO_RESULT(aConn.ExecuteSimpleSQL("DROP TABLE entries;"_ns))); + + // Rename new_entries to entries. + QM_TRY(MOZ_TO_RESULT( + aConn.ExecuteSimpleSQL("ALTER TABLE new_entries RENAME to entries;"_ns))); + + // Now, recreate our indices. + QM_TRY(MOZ_TO_RESULT( + aConn.ExecuteSimpleSQL(nsLiteralCString(kIndexEntriesRequest)))); + + // Revalidate the foreign key constraints, and ensure that there are no + // violations. + QM_TRY_INSPECT(const bool& hasResult, + quota::CreateAndExecuteSingleStepStatement< + quota::SingleStepResult::ReturnNullIfNoResult>( + aConn, "PRAGMA foreign_key_check;"_ns)); + + QM_TRY(OkIf(!hasResult), NS_ERROR_FAILURE); + + QM_TRY(MOZ_TO_RESULT(aConn.SetSchemaVersion(21))); + + aRewriteSchema = true; + + return NS_OK; +} + +nsresult MigrateFrom21To22(mozIStorageConnection& aConn, bool& aRewriteSchema) { + MOZ_ASSERT(!NS_IsMainThread()); + + // Add the request_integrity column. + QM_TRY(MOZ_TO_RESULT(aConn.ExecuteSimpleSQL( + "ALTER TABLE entries " + "ADD COLUMN request_integrity TEXT NOT NULL DEFAULT '';"_ns))); + + QM_TRY(MOZ_TO_RESULT( + aConn.ExecuteSimpleSQL("UPDATE entries SET request_integrity = '';"_ns))); + + QM_TRY(MOZ_TO_RESULT(aConn.SetSchemaVersion(22))); + + aRewriteSchema = true; + + return NS_OK; +} + +nsresult MigrateFrom22To23(mozIStorageConnection& aConn, bool& aRewriteSchema) { + MOZ_ASSERT(!NS_IsMainThread()); + + // The only change between 22 and 23 was a different snappy compression + // format, but it's backwards-compatible. + QM_TRY(MOZ_TO_RESULT(aConn.SetSchemaVersion(23))); + + return NS_OK; +} + +nsresult MigrateFrom23To24(mozIStorageConnection& aConn, bool& aRewriteSchema) { + MOZ_ASSERT(!NS_IsMainThread()); + + // Add the request_url_fragment column. + QM_TRY(MOZ_TO_RESULT(aConn.ExecuteSimpleSQL( + "ALTER TABLE entries " + "ADD COLUMN request_url_fragment TEXT NOT NULL DEFAULT ''"_ns))); + + QM_TRY(MOZ_TO_RESULT(aConn.SetSchemaVersion(24))); + + aRewriteSchema = true; + + return NS_OK; +} + +nsresult MigrateFrom24To25(mozIStorageConnection& aConn, bool& aRewriteSchema) { + MOZ_ASSERT(!NS_IsMainThread()); + + // The only change between 24 and 25 was a new nsIContentPolicy type. + QM_TRY(MOZ_TO_RESULT(aConn.SetSchemaVersion(25))); + + return NS_OK; +} + +nsresult MigrateFrom25To26(mozIStorageConnection& aConn, bool& aRewriteSchema) { + MOZ_ASSERT(!NS_IsMainThread()); + + // Add the response_padding_size column. + // Note: only opaque repsonse should be non-null interger. + QM_TRY(MOZ_TO_RESULT(aConn.ExecuteSimpleSQL( + "ALTER TABLE entries " + "ADD COLUMN response_padding_size INTEGER NULL "_ns))); + + QM_TRY(MOZ_TO_RESULT( + aConn.ExecuteSimpleSQL("UPDATE entries SET response_padding_size = 0 " + "WHERE response_type = 4"_ns // opaque response + ))); + + QM_TRY(MOZ_TO_RESULT(aConn.SetSchemaVersion(26))); + + aRewriteSchema = true; + + return NS_OK; +} + +nsresult MigrateFrom26To27(mozIStorageConnection& aConn, bool& aRewriteSchema) { + MOZ_ASSERT(!NS_IsMainThread()); + + QM_TRY(MOZ_TO_RESULT(aConn.SetSchemaVersion(kHackyDowngradeSchemaVersion))); + + return NS_OK; +} + +nsresult MigrateFrom27To28(mozIStorageConnection& aConn, bool& aRewriteSchema) { + MOZ_ASSERT(!NS_IsMainThread()); + + // In Bug 1264178, we added a column request_integrity into table entries. + // However, at that time, the default value for the existing rows is NULL + // which against the statement in kTableEntries. Thus, we need to have another + // upgrade to update these values to an empty string. + QM_TRY(MOZ_TO_RESULT( + aConn.ExecuteSimpleSQL("UPDATE entries SET request_integrity = '' " + "WHERE request_integrity is NULL;"_ns))); + + QM_TRY(MOZ_TO_RESULT(aConn.SetSchemaVersion(28))); + + return NS_OK; +} + +} // anonymous namespace +} // namespace mozilla::dom::cache::db diff --git a/dom/cache/DBSchema.h b/dom/cache/DBSchema.h new file mode 100644 index 0000000000..e8b864f3c9 --- /dev/null +++ b/dom/cache/DBSchema.h @@ -0,0 +1,97 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_DBSchema_h +#define mozilla_dom_cache_DBSchema_h + +#include "mozilla/Attributes.h" +#include "mozilla/dom/cache/Types.h" +#include "nsError.h" +#include "nsString.h" +#include "nsTArrayForwardDeclare.h" + +class mozIStorageConnection; +struct nsID; + +namespace mozilla::dom::cache { + +class CacheQueryParams; +class CacheRequest; +class CacheResponse; +struct SavedRequest; +struct SavedResponse; + +namespace db { + +// Note, this cannot be executed within a transaction. +nsresult CreateOrMigrateSchema(mozIStorageConnection& aConn); + +// Note, this cannot be executed within a transaction. +nsresult InitializeConnection(mozIStorageConnection& aConn); + +Result<CacheId, nsresult> CreateCacheId(mozIStorageConnection& aConn); + +Result<DeletionInfo, nsresult> DeleteCacheId(mozIStorageConnection& aConn, + CacheId aCacheId); + +Result<AutoTArray<CacheId, 8>, nsresult> FindOrphanedCacheIds( + mozIStorageConnection& aConn); + +Result<int64_t, nsresult> FindOverallPaddingSize(mozIStorageConnection& aConn); + +Result<nsTArray<nsID>, nsresult> GetKnownBodyIds(mozIStorageConnection& aConn); + +Result<Maybe<SavedResponse>, nsresult> CacheMatch( + mozIStorageConnection& aConn, CacheId aCacheId, + const CacheRequest& aRequest, const CacheQueryParams& aParams); + +Result<nsTArray<SavedResponse>, nsresult> CacheMatchAll( + mozIStorageConnection& aConn, CacheId aCacheId, + const Maybe<CacheRequest>& aMaybeRequest, const CacheQueryParams& aParams); + +Result<DeletionInfo, nsresult> CachePut(mozIStorageConnection& aConn, + CacheId aCacheId, + const CacheRequest& aRequest, + const nsID* aRequestBodyId, + const CacheResponse& aResponse, + const nsID* aResponseBodyId); + +Result<Maybe<DeletionInfo>, nsresult> CacheDelete( + mozIStorageConnection& aConn, CacheId aCacheId, + const CacheRequest& aRequest, const CacheQueryParams& aParams); + +Result<nsTArray<SavedRequest>, nsresult> CacheKeys( + mozIStorageConnection& aConn, CacheId aCacheId, + const Maybe<CacheRequest>& aMaybeRequest, const CacheQueryParams& aParams); + +Result<Maybe<SavedResponse>, nsresult> StorageMatch( + mozIStorageConnection& aConn, Namespace aNamespace, + const CacheRequest& aRequest, const CacheQueryParams& aParams); + +Result<Maybe<CacheId>, nsresult> StorageGetCacheId(mozIStorageConnection& aConn, + Namespace aNamespace, + const nsAString& aKey); + +nsresult StoragePutCache(mozIStorageConnection& aConn, Namespace aNamespace, + const nsAString& aKey, CacheId aCacheId); + +nsresult StorageForgetCache(mozIStorageConnection& aConn, Namespace aNamespace, + const nsAString& aKey); + +Result<nsTArray<nsString>, nsresult> StorageGetKeys( + mozIStorageConnection& aConn, Namespace aNamespace); + +// Note, this works best when its NOT executed within a transaction. +nsresult IncrementalVacuum(mozIStorageConnection& aConn); + +// We will wipe out databases with a schema versions less than this. Newer +// versions will be migrated on open to the latest schema version. +extern const int32_t kFirstShippedSchemaVersion; + +} // namespace db +} // namespace mozilla::dom::cache + +#endif // mozilla_dom_cache_DBSchema_h diff --git a/dom/cache/FileUtils.cpp b/dom/cache/FileUtils.cpp new file mode 100644 index 0000000000..9c6b1f60e6 --- /dev/null +++ b/dom/cache/FileUtils.cpp @@ -0,0 +1,778 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "FileUtilsImpl.h" + +#include "DBSchema.h" +#include "mozilla/dom/InternalResponse.h" +#include "mozilla/dom/quota/FileStreams.h" +#include "mozilla/dom/quota/QuotaManager.h" +#include "mozilla/dom/quota/QuotaObject.h" +#include "mozilla/dom/quota/ResultExtensions.h" +#include "mozilla/ScopeExit.h" +#include "mozilla/SnappyCompressOutputStream.h" +#include "mozilla/Unused.h" +#include "nsIObjectInputStream.h" +#include "nsIObjectOutputStream.h" +#include "nsIFile.h" +#include "nsIUUIDGenerator.h" +#include "nsNetCID.h" +#include "nsNetUtil.h" +#include "nsServiceManagerUtils.h" +#include "nsString.h" +#include "nsThreadUtils.h" +#include "snappy/snappy.h" + +namespace mozilla::dom::cache { + +static_assert(SNAPPY_VERSION == 0x010109); + +using mozilla::dom::quota::Client; +using mozilla::dom::quota::CloneFileAndAppend; +using mozilla::dom::quota::FileInputStream; +using mozilla::dom::quota::FileOutputStream; +using mozilla::dom::quota::GetDirEntryKind; +using mozilla::dom::quota::nsIFileKind; +using mozilla::dom::quota::PERSISTENCE_TYPE_DEFAULT; +using mozilla::dom::quota::QuotaManager; +using mozilla::dom::quota::QuotaObject; + +namespace { + +// Const variable for generate padding size. +// XXX This will be tweaked to something more meaningful in Bug 1383656. +const int64_t kRoundUpNumber = 20480; + +enum BodyFileType { BODY_FILE_FINAL, BODY_FILE_TMP }; + +Result<NotNull<nsCOMPtr<nsIFile>>, nsresult> BodyIdToFile(nsIFile& aBaseDir, + const nsID& aId, + BodyFileType aType); + +int64_t RoundUp(int64_t aX, int64_t aY); + +// The alogrithm for generating padding refers to the mitigation approach in +// https://github.com/whatwg/storage/issues/31. +// First, generate a random number between 0 and 100kB. +// Next, round up the sum of random number and response size to the nearest +// 20kB. +// Finally, the virtual padding size will be the result minus the response size. +int64_t BodyGeneratePadding(int64_t aBodyFileSize, uint32_t aPaddingInfo); + +nsresult DirectoryPaddingWrite(nsIFile& aBaseDir, + DirPaddingFile aPaddingFileType, + int64_t aPaddingSize); + +const auto kMorgueDirectory = u"morgue"_ns; + +bool IsFileNotFoundError(const nsresult aRv) { + return aRv == NS_ERROR_FILE_NOT_FOUND; +} + +Result<NotNull<nsCOMPtr<nsIFile>>, nsresult> BodyGetCacheDir(nsIFile& aBaseDir, + const nsID& aId) { + QM_TRY_UNWRAP(auto cacheDir, CloneFileAndAppend(aBaseDir, kMorgueDirectory)); + + // Some file systems have poor performance when there are too many files + // in a single directory. Mitigate this issue by spreading the body + // files out into sub-directories. We use the last byte of the ID for + // the name of the sub-directory. + QM_TRY(MOZ_TO_RESULT(cacheDir->Append(IntToString(aId.m3[7])))); + + // Callers call this function without checking if the directory already + // exists (idempotent usage). QM_OR_ELSE_WARN_IF is not used here since we + // just want to log NS_ERROR_FILE_ALREADY_EXISTS result and not spam the + // reports. + QM_TRY(QM_OR_ELSE_LOG_VERBOSE_IF( + // Expression. + MOZ_TO_RESULT(cacheDir->Create(nsIFile::DIRECTORY_TYPE, 0755)), + // Predicate. + IsSpecificError<NS_ERROR_FILE_ALREADY_EXISTS>, + // Fallback. + ErrToDefaultOk<>)); + + return WrapNotNullUnchecked(std::move(cacheDir)); +} + +} // namespace + +nsresult BodyCreateDir(nsIFile& aBaseDir) { + QM_TRY_INSPECT(const auto& bodyDir, + CloneFileAndAppend(aBaseDir, kMorgueDirectory)); + + // Callers call this function without checking if the directory already + // exists (idempotent usage). QM_OR_ELSE_WARN_IF is not used here since we + // just want to log NS_ERROR_FILE_ALREADY_EXISTS result and not spam the + // reports. + QM_TRY(QM_OR_ELSE_LOG_VERBOSE_IF( + // Expression. + MOZ_TO_RESULT(bodyDir->Create(nsIFile::DIRECTORY_TYPE, 0755)), + // Predicate. + IsSpecificError<NS_ERROR_FILE_ALREADY_EXISTS>, + // Fallback. + ErrToDefaultOk<>)); + + return NS_OK; +} + +nsresult BodyDeleteDir(const CacheDirectoryMetadata& aDirectoryMetadata, + nsIFile& aBaseDir) { + QM_TRY_INSPECT(const auto& bodyDir, + CloneFileAndAppend(aBaseDir, kMorgueDirectory)); + + QM_TRY(MOZ_TO_RESULT(RemoveNsIFileRecursively(aDirectoryMetadata, *bodyDir))); + + return NS_OK; +} + +Result<std::pair<nsID, nsCOMPtr<nsISupports>>, nsresult> BodyStartWriteStream( + const CacheDirectoryMetadata& aDirectoryMetadata, nsIFile& aBaseDir, + nsIInputStream& aSource, void* aClosure, nsAsyncCopyCallbackFun aCallback) { + MOZ_DIAGNOSTIC_ASSERT(aClosure); + MOZ_DIAGNOSTIC_ASSERT(aCallback); + + QM_TRY_INSPECT(const auto& idGen, + MOZ_TO_RESULT_GET_TYPED(nsCOMPtr<nsIUUIDGenerator>, + MOZ_SELECT_OVERLOAD(do_GetService), + "@mozilla.org/uuid-generator;1")); + + nsID id; + QM_TRY(MOZ_TO_RESULT(idGen->GenerateUUIDInPlace(&id))); + + QM_TRY_INSPECT(const auto& finalFile, + BodyIdToFile(aBaseDir, id, BODY_FILE_FINAL)); + + { + QM_TRY_INSPECT(const bool& exists, + MOZ_TO_RESULT_INVOKE_MEMBER(*finalFile, Exists)); + + QM_TRY(OkIf(!exists), Err(NS_ERROR_FILE_ALREADY_EXISTS)); + } + + QM_TRY_INSPECT(const auto& tmpFile, + BodyIdToFile(aBaseDir, id, BODY_FILE_TMP)); + + QM_TRY_UNWRAP( + nsCOMPtr<nsIOutputStream> fileStream, + CreateFileOutputStream(PERSISTENCE_TYPE_DEFAULT, aDirectoryMetadata, + Client::DOMCACHE, tmpFile.get())); + + const auto compressed = MakeRefPtr<SnappyCompressOutputStream>(fileStream); + + const nsCOMPtr<nsIEventTarget> target = + do_GetService(NS_STREAMTRANSPORTSERVICE_CONTRACTID); + + nsCOMPtr<nsISupports> copyContext; + QM_TRY(MOZ_TO_RESULT( + NS_AsyncCopy(&aSource, compressed, target, NS_ASYNCCOPY_VIA_WRITESEGMENTS, + compressed->BlockSize(), aCallback, aClosure, true, + true, // close streams + getter_AddRefs(copyContext)))); + + return std::make_pair(id, std::move(copyContext)); +} + +void BodyCancelWrite(nsISupports& aCopyContext) { + QM_WARNONLY_TRY( + QM_TO_RESULT(NS_CancelAsyncCopy(&aCopyContext, NS_ERROR_ABORT))); + + // TODO The partially written file must be cleaned up after the async copy + // makes its callback. +} + +nsresult BodyFinalizeWrite(nsIFile& aBaseDir, const nsID& aId) { + QM_TRY_INSPECT(const auto& tmpFile, + BodyIdToFile(aBaseDir, aId, BODY_FILE_TMP)); + + QM_TRY_INSPECT(const auto& finalFile, + BodyIdToFile(aBaseDir, aId, BODY_FILE_FINAL)); + + nsAutoString finalFileName; + QM_TRY(MOZ_TO_RESULT(finalFile->GetLeafName(finalFileName))); + + // It's fine to not notify the QuotaManager that the path has been changed, + // because its path will be updated and its size will be recalculated when + // opening file next time. + QM_TRY(MOZ_TO_RESULT(tmpFile->RenameTo(nullptr, finalFileName))); + + return NS_OK; +} + +Result<MovingNotNull<nsCOMPtr<nsIInputStream>>, nsresult> BodyOpen( + const CacheDirectoryMetadata& aDirectoryMetadata, nsIFile& aBaseDir, + const nsID& aId) { + QM_TRY_INSPECT(const auto& finalFile, + BodyIdToFile(aBaseDir, aId, BODY_FILE_FINAL)); + + QM_TRY_RETURN(CreateFileInputStream(PERSISTENCE_TYPE_DEFAULT, + aDirectoryMetadata, Client::DOMCACHE, + finalFile.get())); +} + +nsresult BodyMaybeUpdatePaddingSize( + const CacheDirectoryMetadata& aDirectoryMetadata, nsIFile& aBaseDir, + const nsID& aId, const uint32_t aPaddingInfo, int64_t* aPaddingSizeInOut) { + MOZ_DIAGNOSTIC_ASSERT(aPaddingSizeInOut); + + QM_TRY_INSPECT(const auto& bodyFile, + BodyIdToFile(aBaseDir, aId, BODY_FILE_TMP)); + + QuotaManager* quotaManager = QuotaManager::Get(); + MOZ_DIAGNOSTIC_ASSERT(quotaManager); + + int64_t fileSize = 0; + RefPtr<QuotaObject> quotaObject = quotaManager->GetQuotaObject( + PERSISTENCE_TYPE_DEFAULT, aDirectoryMetadata, Client::DOMCACHE, + bodyFile.get(), -1, &fileSize); + MOZ_DIAGNOSTIC_ASSERT(quotaObject); + MOZ_DIAGNOSTIC_ASSERT(fileSize >= 0); + // XXXtt: bug: https://bugzilla.mozilla.org/show_bug.cgi?id=1422815 + if (!quotaObject) { + return NS_ERROR_UNEXPECTED; + } + + if (*aPaddingSizeInOut == InternalResponse::UNKNOWN_PADDING_SIZE) { + *aPaddingSizeInOut = BodyGeneratePadding(fileSize, aPaddingInfo); + } + + MOZ_DIAGNOSTIC_ASSERT(*aPaddingSizeInOut >= 0); + + if (!quotaObject->IncreaseSize(*aPaddingSizeInOut)) { + return NS_ERROR_FILE_NO_DEVICE_SPACE; + } + + return NS_OK; +} + +nsresult BodyDeleteFiles(const CacheDirectoryMetadata& aDirectoryMetadata, + nsIFile& aBaseDir, const nsTArray<nsID>& aIdList) { + for (const auto id : aIdList) { + QM_TRY_INSPECT(const auto& bodyDir, BodyGetCacheDir(aBaseDir, id)); + + const auto removeFileForId = + [&aDirectoryMetadata, &id]( + nsIFile& bodyFile, + const nsACString& leafName) -> Result<bool, nsresult> { + nsID fileId; + QM_TRY(OkIf(fileId.Parse(leafName.BeginReading())), true, + ([&aDirectoryMetadata, &bodyFile](const auto) { + DebugOnly<nsresult> result = RemoveNsIFile( + aDirectoryMetadata, bodyFile, /* aTrackQuota */ false); + MOZ_ASSERT(NS_SUCCEEDED(result)); + })); + + if (id.Equals(fileId)) { + DebugOnly<nsresult> result = + RemoveNsIFile(aDirectoryMetadata, bodyFile); + MOZ_ASSERT(NS_SUCCEEDED(result)); + return true; + } + + return false; + }; + QM_TRY(MOZ_TO_RESULT(BodyTraverseFiles(aDirectoryMetadata, *bodyDir, + removeFileForId, + /* aCanRemoveFiles */ false, + /* aTrackQuota */ true))); + } + + return NS_OK; +} + +namespace { + +Result<NotNull<nsCOMPtr<nsIFile>>, nsresult> BodyIdToFile( + nsIFile& aBaseDir, const nsID& aId, const BodyFileType aType) { + QM_TRY_UNWRAP(auto bodyFile, BodyGetCacheDir(aBaseDir, aId)); + + char idString[NSID_LENGTH]; + aId.ToProvidedString(idString); + + NS_ConvertASCIItoUTF16 fileName(idString); + + if (aType == BODY_FILE_FINAL) { + fileName.AppendLiteral(".final"); + } else { + fileName.AppendLiteral(".tmp"); + } + + QM_TRY(MOZ_TO_RESULT(bodyFile->Append(fileName))); + + return bodyFile; +} + +int64_t RoundUp(const int64_t aX, const int64_t aY) { + MOZ_DIAGNOSTIC_ASSERT(aX >= 0); + MOZ_DIAGNOSTIC_ASSERT(aY > 0); + + MOZ_DIAGNOSTIC_ASSERT(INT64_MAX - ((aX - 1) / aY) * aY >= aY); + return aY + ((aX - 1) / aY) * aY; +} + +int64_t BodyGeneratePadding(const int64_t aBodyFileSize, + const uint32_t aPaddingInfo) { + // Generate padding + int64_t randomSize = static_cast<int64_t>(aPaddingInfo); + MOZ_DIAGNOSTIC_ASSERT(INT64_MAX - aBodyFileSize >= randomSize); + randomSize += aBodyFileSize; + + return RoundUp(randomSize, kRoundUpNumber) - aBodyFileSize; +} + +nsresult DirectoryPaddingWrite(nsIFile& aBaseDir, + DirPaddingFile aPaddingFileType, + int64_t aPaddingSize) { + MOZ_DIAGNOSTIC_ASSERT(aPaddingSize >= 0); + + QM_TRY_INSPECT( + const auto& file, + CloneFileAndAppend(aBaseDir, aPaddingFileType == DirPaddingFile::TMP_FILE + ? nsLiteralString(PADDING_TMP_FILE_NAME) + : nsLiteralString(PADDING_FILE_NAME))); + + QM_TRY_INSPECT(const auto& outputStream, NS_NewLocalFileOutputStream(file)); + + nsCOMPtr<nsIObjectOutputStream> objectStream = + NS_NewObjectOutputStream(outputStream); + + QM_TRY(MOZ_TO_RESULT(objectStream->Write64(aPaddingSize))); + + return NS_OK; +} + +} // namespace + +nsresult BodyDeleteOrphanedFiles( + const CacheDirectoryMetadata& aDirectoryMetadata, nsIFile& aBaseDir, + const nsTArray<nsID>& aKnownBodyIdList) { + // body files are stored in a directory structure like: + // + // /morgue/01/{01fdddb2-884d-4c3d-95ba-0c8062f6c325}.final + // /morgue/02/{02fdddb2-884d-4c3d-95ba-0c8062f6c325}.tmp + + QM_TRY_INSPECT(const auto& dir, + CloneFileAndAppend(aBaseDir, kMorgueDirectory)); + + // Iterate over all the intermediate morgue subdirs + QM_TRY(quota::CollectEachFile( + *dir, + [&aDirectoryMetadata, &aKnownBodyIdList]( + const nsCOMPtr<nsIFile>& subdir) -> Result<Ok, nsresult> { + QM_TRY_INSPECT(const auto& dirEntryKind, GetDirEntryKind(*subdir)); + + switch (dirEntryKind) { + case nsIFileKind::ExistsAsDirectory: { + const auto removeOrphanedFiles = + [&aDirectoryMetadata, &aKnownBodyIdList]( + nsIFile& bodyFile, + const nsACString& leafName) -> Result<bool, nsresult> { + // Finally, parse the uuid out of the name. If it fails to parse, + // then ignore the file. + auto cleanup = MakeScopeExit([&aDirectoryMetadata, &bodyFile] { + DebugOnly<nsresult> result = + RemoveNsIFile(aDirectoryMetadata, bodyFile); + MOZ_ASSERT(NS_SUCCEEDED(result)); + }); + + nsID id; + QM_TRY(OkIf(id.Parse(leafName.BeginReading())), true); + + if (!aKnownBodyIdList.Contains(id)) { + return true; + } + + cleanup.release(); + + return false; + }; + + // QM_OR_ELSE_WARN_IF is not used here since we just want to log + // NS_ERROR_FILE_FS_CORRUPTED result and not spam the reports (even + // a warning in the reports is not desired). + QM_TRY(QM_OR_ELSE_LOG_VERBOSE_IF( + // Expression. + MOZ_TO_RESULT(BodyTraverseFiles(aDirectoryMetadata, *subdir, + removeOrphanedFiles, + /* aCanRemoveFiles */ true, + /* aTrackQuota */ true)), + // Predicate. + IsSpecificError<NS_ERROR_FILE_FS_CORRUPTED>, + // Fallback. We treat NS_ERROR_FILE_FS_CORRUPTED as if the + // directory did not exist at all. + ErrToDefaultOk<>)); + break; + } + + case nsIFileKind::ExistsAsFile: { + // If a file got in here somehow, try to remove it and move on + DebugOnly<nsresult> result = + RemoveNsIFile(aDirectoryMetadata, *subdir, + /* aTrackQuota */ false); + MOZ_ASSERT(NS_SUCCEEDED(result)); + break; + } + + case nsIFileKind::DoesNotExist: + // Ignore files that got removed externally while iterating. + break; + } + + return Ok{}; + })); + + return NS_OK; +} + +namespace { + +Result<nsCOMPtr<nsIFile>, nsresult> GetMarkerFileHandle( + const CacheDirectoryMetadata& aDirectoryMetadata) { + QM_TRY_UNWRAP(auto marker, + CloneFileAndAppend(*aDirectoryMetadata.mDir, u"cache"_ns)); + + QM_TRY(MOZ_TO_RESULT(marker->Append(u"context_open.marker"_ns))); + + return marker; +} + +} // namespace + +nsresult CreateMarkerFile(const CacheDirectoryMetadata& aDirectoryMetadata) { + QM_TRY_INSPECT(const auto& marker, GetMarkerFileHandle(aDirectoryMetadata)); + + // Callers call this function without checking if the file already exists + // (idempotent usage). QM_OR_ELSE_WARN_IF is not used here since we just want + // to log NS_ERROR_FILE_ALREADY_EXISTS result and not spam the reports. + // + // TODO: In theory if this file exists, then Context::~Context should have + // cleaned it up, but obviously we can crash and not clean it up, which is + // the whole point of the marker file. In that case, we'll realize the marker + // file exists in SetupAction::RunSyncWithDBOnTarget and do some cleanup, but + // we won't delete the marker file, so if we see this marker file, it is part + // of our standard operating procedure to redundantly try and create the + // marker here. We currently treat this as idempotent usage, but we could + // make sure to delete the marker file when handling the existing marker + // file in SetupAction::RunSyncWithDBOnTarget and change + // QM_OR_ELSE_LOG_VERBOSE_IF to QM_OR_ELSE_WARN_IF in the end. + QM_TRY(QM_OR_ELSE_LOG_VERBOSE_IF( + // Expression. + MOZ_TO_RESULT(marker->Create(nsIFile::NORMAL_FILE_TYPE, 0644)), + // Predicate. + IsSpecificError<NS_ERROR_FILE_ALREADY_EXISTS>, + // Fallback. + ErrToDefaultOk<>)); + + // Note, we don't need to fsync here. We only care about actually + // writing the marker if later modifications to the Cache are + // actually flushed to the disk. If the OS crashes before the marker + // is written then we are ensured no other changes to the Cache were + // flushed either. + + return NS_OK; +} + +nsresult DeleteMarkerFile(const CacheDirectoryMetadata& aDirectoryMetadata) { + QM_TRY_INSPECT(const auto& marker, GetMarkerFileHandle(aDirectoryMetadata)); + + DebugOnly<nsresult> result = + RemoveNsIFile(aDirectoryMetadata, *marker, /* aTrackQuota */ false); + MOZ_ASSERT(NS_SUCCEEDED(result)); + + // Again, no fsync is necessary. If the OS crashes before the file + // removal is flushed, then the Cache will search for stale data on + // startup. This will cause the next Cache access to be a bit slow, but + // it seems appropriate after an OS crash. + + return NS_OK; +} + +bool MarkerFileExists(const CacheDirectoryMetadata& aDirectoryMetadata) { + QM_TRY_INSPECT(const auto& marker, GetMarkerFileHandle(aDirectoryMetadata), + false); + + QM_TRY_RETURN(MOZ_TO_RESULT_INVOKE_MEMBER(marker, Exists), false); +} + +nsresult RemoveNsIFileRecursively( + const Maybe<CacheDirectoryMetadata>& aDirectoryMetadata, nsIFile& aFile, + const bool aTrackQuota) { + // XXX This assertion proves that we can remove aTrackQuota and just check + // aClientMetadata + MOZ_DIAGNOSTIC_ASSERT_IF(aTrackQuota, aDirectoryMetadata); + + QM_TRY_INSPECT(const auto& dirEntryKind, GetDirEntryKind(aFile)); + + switch (dirEntryKind) { + case nsIFileKind::ExistsAsDirectory: + // Unfortunately, we need to traverse all the entries and delete files one + // by + // one to update their usages to the QuotaManager. + QM_TRY(quota::CollectEachFile( + aFile, + [&aDirectoryMetadata, &aTrackQuota]( + const nsCOMPtr<nsIFile>& file) -> Result<Ok, nsresult> { + QM_TRY(MOZ_TO_RESULT(RemoveNsIFileRecursively(aDirectoryMetadata, + *file, aTrackQuota))); + + return Ok{}; + })); + + // In the end, remove the folder + QM_TRY(MOZ_TO_RESULT(aFile.Remove(/* recursive */ false))); + + break; + + case nsIFileKind::ExistsAsFile: + return RemoveNsIFile(aDirectoryMetadata, aFile, aTrackQuota); + + case nsIFileKind::DoesNotExist: + // Ignore files that got removed externally while iterating. + break; + } + + return NS_OK; +} + +nsresult RemoveNsIFile(const Maybe<CacheDirectoryMetadata>& aDirectoryMetadata, + nsIFile& aFile, const bool aTrackQuota) { + // XXX This assertion proves that we can remove aTrackQuota and just check + // aClientMetadata + MOZ_DIAGNOSTIC_ASSERT_IF(aTrackQuota, aDirectoryMetadata); + + int64_t fileSize = 0; + if (aTrackQuota) { + QM_TRY_INSPECT( + const auto& maybeFileSize, + QM_OR_ELSE_WARN_IF( + // Expression. + MOZ_TO_RESULT_INVOKE_MEMBER(aFile, GetFileSize).map(Some<int64_t>), + // Predicate. + IsFileNotFoundError, + // Fallback. + ErrToDefaultOk<Maybe<int64_t>>)); + + if (!maybeFileSize) { + return NS_OK; + } + + fileSize = *maybeFileSize; + } + + QM_TRY(QM_OR_ELSE_WARN_IF( + // Expression. + MOZ_TO_RESULT(aFile.Remove(/* recursive */ false)), + // Predicate. + IsFileNotFoundError, + // Fallback. + ErrToDefaultOk<>)); + + if (fileSize > 0) { + MOZ_ASSERT(aTrackQuota); + DecreaseUsageForDirectoryMetadata(*aDirectoryMetadata, fileSize); + } + + return NS_OK; +} + +void DecreaseUsageForDirectoryMetadata( + const CacheDirectoryMetadata& aDirectoryMetadata, + const int64_t aUpdatingSize) { + MOZ_DIAGNOSTIC_ASSERT(aUpdatingSize > 0); + + QuotaManager* quotaManager = QuotaManager::Get(); + MOZ_DIAGNOSTIC_ASSERT(quotaManager); + + quotaManager->DecreaseUsageForClient( + quota::ClientMetadata{aDirectoryMetadata, Client::DOMCACHE}, + aUpdatingSize); +} + +bool DirectoryPaddingFileExists(nsIFile& aBaseDir, + DirPaddingFile aPaddingFileType) { + QM_TRY_INSPECT( + const auto& file, + CloneFileAndAppend(aBaseDir, aPaddingFileType == DirPaddingFile::TMP_FILE + ? nsLiteralString(PADDING_TMP_FILE_NAME) + : nsLiteralString(PADDING_FILE_NAME)), + false); + + QM_TRY_RETURN(MOZ_TO_RESULT_INVOKE_MEMBER(file, Exists), false); +} + +Result<int64_t, nsresult> DirectoryPaddingGet(nsIFile& aBaseDir) { + MOZ_DIAGNOSTIC_ASSERT( + !DirectoryPaddingFileExists(aBaseDir, DirPaddingFile::TMP_FILE)); + + QM_TRY_INSPECT( + const auto& file, + CloneFileAndAppend(aBaseDir, nsLiteralString(PADDING_FILE_NAME))); + + QM_TRY_UNWRAP(auto stream, NS_NewLocalFileInputStream(file)); + + QM_TRY_INSPECT(const auto& bufferedStream, + NS_NewBufferedInputStream(stream.forget(), 512)); + + const nsCOMPtr<nsIObjectInputStream> objectStream = + NS_NewObjectInputStream(bufferedStream); + + QM_TRY_RETURN(MOZ_TO_RESULT_INVOKE_MEMBER(objectStream, Read64) + .map([](const uint64_t val) { return int64_t(val); })); +} + +nsresult DirectoryPaddingInit(nsIFile& aBaseDir) { + QM_TRY( + MOZ_TO_RESULT(DirectoryPaddingWrite(aBaseDir, DirPaddingFile::FILE, 0))); + + return NS_OK; +} + +nsresult UpdateDirectoryPaddingFile(nsIFile& aBaseDir, + mozIStorageConnection& aConn, + const int64_t aIncreaseSize, + const int64_t aDecreaseSize, + const bool aTemporaryFileExist) { + MOZ_DIAGNOSTIC_ASSERT(aIncreaseSize >= 0); + MOZ_DIAGNOSTIC_ASSERT(aDecreaseSize >= 0); + + const auto directoryPaddingGetResult = + aTemporaryFileExist ? Maybe<int64_t>{} : [&aBaseDir] { + QM_TRY_RETURN(QM_OR_ELSE_WARN_IF( + // Expression. + DirectoryPaddingGet(aBaseDir).map(Some<int64_t>), + // Predicate. + IsFileNotFoundError, + // Fallback. + ErrToDefaultOk<Maybe<int64_t>>), + Maybe<int64_t>{}); + }(); + + QM_TRY_INSPECT( + const int64_t& currentPaddingSize, + ([directoryPaddingGetResult, &aBaseDir, &aConn, aIncreaseSize, + aDecreaseSize]() -> Result<int64_t, nsresult> { + if (!directoryPaddingGetResult) { + // Fail to read padding size from the dir padding file, so try to + // restore. + + // Not delete the temporary padding file here, because we're going + // to overwrite it below anyway. + QM_TRY(MOZ_TO_RESULT( + DirectoryPaddingDeleteFile(aBaseDir, DirPaddingFile::FILE))); + + // We don't need to add the aIncreaseSize or aDecreaseSize here, + // because it's already encompassed within the database. + QM_TRY_RETURN(db::FindOverallPaddingSize(aConn)); + } + + int64_t currentPaddingSize = directoryPaddingGetResult.value(); + bool shouldRevise = false; + + if (aIncreaseSize > 0) { + if (INT64_MAX - currentPaddingSize < aDecreaseSize) { + shouldRevise = true; + } else { + currentPaddingSize += aIncreaseSize; + } + } + + if (aDecreaseSize > 0) { + if (currentPaddingSize < aDecreaseSize) { + shouldRevise = true; + } else if (!shouldRevise) { + currentPaddingSize -= aDecreaseSize; + } + } + + if (shouldRevise) { + // If somehow runing into this condition, the tracking padding size is + // incorrect. + // Delete padding file to indicate the padding size is incorrect for + // avoiding error happening in the following lines. + QM_TRY(MOZ_TO_RESULT( + DirectoryPaddingDeleteFile(aBaseDir, DirPaddingFile::FILE))); + + QM_TRY_UNWRAP(currentPaddingSize, db::FindOverallPaddingSize(aConn)); + + // XXXtt: we should have an easy way to update (increase or + // recalulate) padding size in the QM. For now, only correct the + // padding size in padding file and make QM be able to get the correct + // size in the next QM initialization. We still want to catch this in + // the debug build. + MOZ_ASSERT(false, "The padding size is unsync with QM"); + } + +#ifdef DEBUG + const int64_t lastPaddingSize = currentPaddingSize; + QM_TRY_UNWRAP(currentPaddingSize, db::FindOverallPaddingSize(aConn)); + + MOZ_DIAGNOSTIC_ASSERT(currentPaddingSize == lastPaddingSize); +#endif // DEBUG + + return currentPaddingSize; + }())); + + MOZ_DIAGNOSTIC_ASSERT(currentPaddingSize >= 0); + + QM_TRY(MOZ_TO_RESULT(DirectoryPaddingWrite(aBaseDir, DirPaddingFile::TMP_FILE, + currentPaddingSize))); + + return NS_OK; +} + +nsresult DirectoryPaddingFinalizeWrite(nsIFile& aBaseDir) { + MOZ_DIAGNOSTIC_ASSERT( + DirectoryPaddingFileExists(aBaseDir, DirPaddingFile::TMP_FILE)); + + QM_TRY_INSPECT( + const auto& file, + CloneFileAndAppend(aBaseDir, nsLiteralString(PADDING_TMP_FILE_NAME))); + + QM_TRY(MOZ_TO_RESULT( + file->RenameTo(nullptr, nsLiteralString(PADDING_FILE_NAME)))); + + return NS_OK; +} + +Result<int64_t, nsresult> DirectoryPaddingRestore(nsIFile& aBaseDir, + mozIStorageConnection& aConn, + const bool aMustRestore) { + // The content of padding file is untrusted, so remove it here. + QM_TRY(MOZ_TO_RESULT( + DirectoryPaddingDeleteFile(aBaseDir, DirPaddingFile::FILE))); + + QM_TRY_INSPECT(const int64_t& paddingSize, db::FindOverallPaddingSize(aConn)); + MOZ_DIAGNOSTIC_ASSERT(paddingSize >= 0); + + QM_TRY(MOZ_TO_RESULT(DirectoryPaddingWrite(aBaseDir, DirPaddingFile::FILE, + paddingSize)), + (aMustRestore ? Err(tryTempError) + : Result<int64_t, nsresult>{paddingSize})); + + QM_TRY(MOZ_TO_RESULT( + DirectoryPaddingDeleteFile(aBaseDir, DirPaddingFile::TMP_FILE))); + + return paddingSize; +} + +nsresult DirectoryPaddingDeleteFile(nsIFile& aBaseDir, + DirPaddingFile aPaddingFileType) { + QM_TRY_INSPECT( + const auto& file, + CloneFileAndAppend(aBaseDir, aPaddingFileType == DirPaddingFile::TMP_FILE + ? nsLiteralString(PADDING_TMP_FILE_NAME) + : nsLiteralString(PADDING_FILE_NAME))); + + QM_TRY(QM_OR_ELSE_WARN_IF( + // Expression. + MOZ_TO_RESULT(file->Remove(/* recursive */ false)), + // Predicate. + IsFileNotFoundError, + // Fallback. + ErrToDefaultOk<>)); + + return NS_OK; +} +} // namespace mozilla::dom::cache diff --git a/dom/cache/FileUtils.h b/dom/cache/FileUtils.h new file mode 100644 index 0000000000..e22a08908d --- /dev/null +++ b/dom/cache/FileUtils.h @@ -0,0 +1,153 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_FileUtils_h +#define mozilla_dom_cache_FileUtils_h + +#include "mozilla/Attributes.h" +#include "mozilla/dom/cache/Types.h" +#include "CacheCommon.h" +#include "mozIStorageConnection.h" +#include "nsStreamUtils.h" +#include "nsTArrayForwardDeclare.h" + +struct nsID; +class nsIFile; + +namespace mozilla::dom::cache { + +#define PADDING_FILE_NAME u".padding" +#define PADDING_TMP_FILE_NAME u".padding-tmp" + +enum class DirPaddingFile { FILE, TMP_FILE }; + +nsresult BodyCreateDir(nsIFile& aBaseDir); + +// Note that this function can only be used during the initialization of the +// database. We're unlikely to be able to delete the DB successfully past +// that point due to the file being in use. +nsresult BodyDeleteDir(const CacheDirectoryMetadata& aDirectoryMetadata, + nsIFile& aBaseDir); + +// Returns a Result with a success value with the body id and, optionally, the +// copy context. +Result<std::pair<nsID, nsCOMPtr<nsISupports>>, nsresult> BodyStartWriteStream( + const CacheDirectoryMetadata& aDirectoryMetadata, nsIFile& aBaseDir, + nsIInputStream& aSource, void* aClosure, nsAsyncCopyCallbackFun aCallback); + +void BodyCancelWrite(nsISupports& aCopyContext); + +nsresult BodyFinalizeWrite(nsIFile& aBaseDir, const nsID& aId); + +Result<MovingNotNull<nsCOMPtr<nsIInputStream>>, nsresult> BodyOpen( + const CacheDirectoryMetadata& aDirectoryMetadata, nsIFile& aBaseDir, + const nsID& aId); + +nsresult BodyMaybeUpdatePaddingSize( + const CacheDirectoryMetadata& aDirectoryMetadata, nsIFile& aBaseDir, + const nsID& aId, uint32_t aPaddingInfo, int64_t* aPaddingSizeInOut); + +nsresult BodyDeleteFiles(const CacheDirectoryMetadata& aDirectoryMetadata, + nsIFile& aBaseDir, const nsTArray<nsID>& aIdList); + +nsresult BodyDeleteOrphanedFiles( + const CacheDirectoryMetadata& aDirectoryMetadata, nsIFile& aBaseDir, + const nsTArray<nsID>& aKnownBodyIdList); + +// If aCanRemoveFiles is true, that means we are safe to touch the files which +// can be accessed in other threads. +// If it's not, that means we cannot remove the files which are possible to +// created by other threads. Note that if the files are not expected, we should +// be safe to remove them in any case. +template <typename Func> +nsresult BodyTraverseFiles( + const Maybe<CacheDirectoryMetadata>& aDirectoryMetadata, nsIFile& aBodyDir, + const Func& aHandleFileFunc, bool aCanRemoveFiles, bool aTrackQuota = true); + +// XXX Remove this method when all callers properly wrap aClientMetadata with +// Some/Nothing +template <typename Func> +nsresult BodyTraverseFiles(const CacheDirectoryMetadata& aDirectoryMetadata, + nsIFile& aBodyDir, const Func& aHandleFileFunc, + bool aCanRemoveFiles, bool aTrackQuota = true) { + return BodyTraverseFiles(Some(aDirectoryMetadata), aBodyDir, aHandleFileFunc, + aCanRemoveFiles, aTrackQuota); +} + +nsresult CreateMarkerFile(const CacheDirectoryMetadata& aDirectoryMetadata); + +nsresult DeleteMarkerFile(const CacheDirectoryMetadata& aDirectoryMetadata); + +bool MarkerFileExists(const CacheDirectoryMetadata& aDirectoryMetadata); + +nsresult RemoveNsIFileRecursively( + const Maybe<CacheDirectoryMetadata>& aDirectoryMetadata, nsIFile& aFile, + bool aTrackQuota = true); + +// XXX Remove this method when all callers properly wrap aClientMetadata with +// Some/Nothing +inline nsresult RemoveNsIFileRecursively( + const CacheDirectoryMetadata& aDirectoryMetadata, nsIFile& aFile, + bool aTrackQuota = true) { + return RemoveNsIFileRecursively(Some(aDirectoryMetadata), aFile, aTrackQuota); +} + +// Delete a file that you think exists. If the file doesn't exist, an error +// will not be returned, but warning telemetry will be generated! So only call +// this on files that you know exist (idempotent usage, but it's not +// recommended). +nsresult RemoveNsIFile(const Maybe<CacheDirectoryMetadata>& aDirectoryMetadata, + nsIFile& aFile, bool aTrackQuota = true); + +// XXX Remove this method when all callers properly wrap aClientMetadata with +// Some/Nothing +inline nsresult RemoveNsIFile(const CacheDirectoryMetadata& aDirectoryMetadata, + nsIFile& aFile, bool aTrackQuota = true) { + return RemoveNsIFile(Some(aDirectoryMetadata), aFile, aTrackQuota); +} + +void DecreaseUsageForDirectoryMetadata( + const CacheDirectoryMetadata& aDirectoryMetadata, int64_t aUpdatingSize); + +/** + * This function is used to check if the directory padding file is existed. + */ +bool DirectoryPaddingFileExists(nsIFile& aBaseDir, + DirPaddingFile aPaddingFileType); + +/** + * + * The functions below are used to read/write/delete the directory padding file + * after acquiring the mutex lock. The mutex lock is held by + * CacheQuotaClient to prevent multi-thread accessing issue. To avoid deadlock, + * these functions should only access by static functions in + * dom/cache/QuotaClient.cpp. + * + */ + +// Returns a Result with a success value denoting the padding size. +Result<int64_t, nsresult> DirectoryPaddingGet(nsIFile& aBaseDir); + +nsresult DirectoryPaddingInit(nsIFile& aBaseDir); + +nsresult UpdateDirectoryPaddingFile(nsIFile& aBaseDir, + mozIStorageConnection& aConn, + int64_t aIncreaseSize, + int64_t aDecreaseSize, + bool aTemporaryFileExist); + +nsresult DirectoryPaddingFinalizeWrite(nsIFile& aBaseDir); + +// Returns a Result with a success value denoting the padding size. +Result<int64_t, nsresult> DirectoryPaddingRestore(nsIFile& aBaseDir, + mozIStorageConnection& aConn, + bool aMustRestore); + +nsresult DirectoryPaddingDeleteFile(nsIFile& aBaseDir, + DirPaddingFile aPaddingFileType); +} // namespace mozilla::dom::cache + +#endif // mozilla_dom_cache_FileUtils_h diff --git a/dom/cache/FileUtilsImpl.h b/dom/cache/FileUtilsImpl.h new file mode 100644 index 0000000000..3f00757b64 --- /dev/null +++ b/dom/cache/FileUtilsImpl.h @@ -0,0 +1,113 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_FileUtilsImpl_h +#define mozilla_dom_cache_FileUtilsImpl_h + +#include "mozilla/dom/FlippedOnce.h" +#include "mozilla/dom/cache/FileUtils.h" +#include "mozilla/dom/quota/ResultExtensions.h" + +namespace mozilla::dom::cache { + +template <typename Func> +nsresult BodyTraverseFiles( + const Maybe<CacheDirectoryMetadata>& aDirectoryMetadata, nsIFile& aBodyDir, + const Func& aHandleFileFunc, const bool aCanRemoveFiles, + const bool aTrackQuota) { + // XXX This assertion proves that we can remove aTrackQuota and just check + // aClientMetadata.isSome() + MOZ_DIAGNOSTIC_ASSERT_IF(aTrackQuota, aDirectoryMetadata); + +#ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED + { + nsCOMPtr<nsIFile> parentFile; + nsresult rv = aBodyDir.GetParent(getter_AddRefs(parentFile)); + MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); + MOZ_DIAGNOSTIC_ASSERT(parentFile); + + nsAutoCString nativeLeafName; + rv = parentFile->GetNativeLeafName(nativeLeafName); + MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); + + MOZ_DIAGNOSTIC_ASSERT(StringEndsWith(nativeLeafName, "morgue"_ns)); + } +#endif + + FlippedOnce<true> isEmpty; + QM_TRY(quota::CollectEachFile( + aBodyDir, + [&isEmpty, &aDirectoryMetadata, aTrackQuota, &aHandleFileFunc, + aCanRemoveFiles](const nsCOMPtr<nsIFile>& file) -> Result<Ok, nsresult> { + QM_TRY_INSPECT(const auto& dirEntryKind, quota::GetDirEntryKind(*file)); + + switch (dirEntryKind) { + case quota::nsIFileKind::ExistsAsDirectory: { + // If it's a directory somehow, try to remove it and move on + DebugOnly<nsresult> result = RemoveNsIFileRecursively( + aDirectoryMetadata, *file, /* aTrackQuota */ false); + MOZ_ASSERT(NS_SUCCEEDED(result)); + break; + } + + case quota::nsIFileKind::ExistsAsFile: { + nsAutoCString leafName; + QM_TRY(MOZ_TO_RESULT(file->GetNativeLeafName(leafName))); + + // Delete all tmp files regardless of known bodies. These are all + // considered orphans. + if (StringEndsWith(leafName, ".tmp"_ns)) { + if (aCanRemoveFiles) { + DebugOnly<nsresult> result = + RemoveNsIFile(aDirectoryMetadata, *file, aTrackQuota); + MOZ_ASSERT(NS_SUCCEEDED(result)); + return Ok{}; + } + } else { + // Otherwise, it must be a .final file. + QM_WARNONLY_TRY_UNWRAP( + const auto maybeEndingOk, + OkIf(StringEndsWith(leafName, ".final"_ns))); + + // If its not, try to remove it and move on. + if (!maybeEndingOk) { + DebugOnly<nsresult> result = RemoveNsIFile( + aDirectoryMetadata, *file, /* aTrackQuota */ false); + MOZ_ASSERT(NS_SUCCEEDED(result)); + return Ok{}; + } + } + + QM_TRY_INSPECT(const bool& fileDeleted, + aHandleFileFunc(*file, leafName)); + if (fileDeleted) { + return Ok{}; + } + + isEmpty.EnsureFlipped(); + break; + } + + case quota::nsIFileKind::DoesNotExist: + // Ignore files that got removed externally while iterating. + break; + } + + return Ok{}; + })); + + if (isEmpty && aCanRemoveFiles) { + DebugOnly<nsresult> result = RemoveNsIFileRecursively( + aDirectoryMetadata, aBodyDir, /* aTrackQuota */ false); + MOZ_ASSERT(NS_SUCCEEDED(result)); + } + + return NS_OK; +} + +} // namespace mozilla::dom::cache + +#endif // mozilla_dom_cache_FileUtilsImpl_h diff --git a/dom/cache/IPCUtils.h b/dom/cache/IPCUtils.h new file mode 100644 index 0000000000..684521f2cf --- /dev/null +++ b/dom/cache/IPCUtils.h @@ -0,0 +1,30 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_IPCUtils_h +#define mozilla_dom_cache_IPCUtils_h + +#include "ipc/EnumSerializer.h" + +#include "mozilla/dom/cache/Types.h" + +namespace IPC { +template <> +struct ParamTraits<mozilla::dom::cache::Namespace> + : public ContiguousEnumSerializer< + mozilla::dom::cache::Namespace, + mozilla::dom::cache::DEFAULT_NAMESPACE, + mozilla::dom::cache::NUMBER_OF_NAMESPACES> {}; + +template <> +struct ParamTraits<mozilla::dom::cache::OpenMode> + : public ContiguousEnumSerializer<mozilla::dom::cache::OpenMode, + mozilla::dom::cache::OpenMode::Eager, + mozilla::dom::cache::OpenMode::NumTypes> { +}; +} // namespace IPC + +#endif // mozilla_dom_cache_IPCUtils_h diff --git a/dom/cache/Manager.cpp b/dom/cache/Manager.cpp new file mode 100644 index 0000000000..ab9b878ae0 --- /dev/null +++ b/dom/cache/Manager.cpp @@ -0,0 +1,2127 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/Manager.h" + +#include "mozilla/AppShutdown.h" +#include "mozilla/AutoRestore.h" +#include "mozilla/Mutex.h" +#include "mozilla/StaticMutex.h" +#include "mozilla/StaticPtr.h" +#include "mozilla/Unused.h" +#include "mozilla/dom/cache/Context.h" +#include "mozilla/dom/cache/DBAction.h" +#include "mozilla/dom/cache/DBSchema.h" +#include "mozilla/dom/cache/FileUtils.h" +#include "mozilla/dom/cache/ManagerId.h" +#include "mozilla/dom/cache/CacheTypes.h" +#include "mozilla/dom/cache/SavedTypes.h" +#include "mozilla/dom/cache/StreamList.h" +#include "mozilla/dom/cache/Types.h" +#include "mozilla/dom/quota/Client.h" +#include "mozilla/dom/quota/ClientImpl.h" +#include "mozilla/dom/quota/QuotaManager.h" +#include "mozilla/ipc/BackgroundParent.h" +#include "mozStorageHelper.h" +#include "nsIInputStream.h" +#include "nsID.h" +#include "nsIFile.h" +#include "nsIThread.h" +#include "nsThreadUtils.h" +#include "nsTObserverArray.h" +#include "QuotaClientImpl.h" + +namespace mozilla::dom::cache { + +using mozilla::dom::quota::Client; +using mozilla::dom::quota::CloneFileAndAppend; +using mozilla::dom::quota::DirectoryLock; + +namespace { + +/** + * Note: The aCommitHook argument will be invoked while a lock is held. Callers + * should be careful not to pass a hook that might lock on something else and + * trigger a deadlock. + */ +template <typename Callable> +nsresult MaybeUpdatePaddingFile(nsIFile* aBaseDir, mozIStorageConnection* aConn, + const int64_t aIncreaseSize, + const int64_t aDecreaseSize, + Callable aCommitHook) { + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aBaseDir); + MOZ_DIAGNOSTIC_ASSERT(aConn); + MOZ_DIAGNOSTIC_ASSERT(aIncreaseSize >= 0); + MOZ_DIAGNOSTIC_ASSERT(aDecreaseSize >= 0); + + RefPtr<CacheQuotaClient> cacheQuotaClient = CacheQuotaClient::Get(); + MOZ_DIAGNOSTIC_ASSERT(cacheQuotaClient); + + QM_TRY(MOZ_TO_RESULT(cacheQuotaClient->MaybeUpdatePaddingFileInternal( + *aBaseDir, *aConn, aIncreaseSize, aDecreaseSize, aCommitHook))); + + return NS_OK; +} + +// An Action that is executed when a Context is first created. It ensures that +// the directory and database are setup properly. This lets other actions +// not worry about these details. +class SetupAction final : public SyncDBAction { + public: + SetupAction() : SyncDBAction(DBAction::Create) {} + + virtual nsresult RunSyncWithDBOnTarget( + const CacheDirectoryMetadata& aDirectoryMetadata, nsIFile* aDBDir, + mozIStorageConnection* aConn) override { + MOZ_DIAGNOSTIC_ASSERT(aDBDir); + + QM_TRY(MOZ_TO_RESULT(BodyCreateDir(*aDBDir))); + + // executes in its own transaction + QM_TRY(MOZ_TO_RESULT(db::CreateOrMigrateSchema(*aConn))); + + // If the Context marker file exists, then the last session was + // not cleanly shutdown. In these cases sqlite will ensure that + // the database is valid, but we might still orphan data. Both + // Cache objects and body files can be referenced by DOM objects + // after they are "removed" from their parent. So we need to + // look and see if any of these late access objects have been + // orphaned. + // + // Note, this must be done after any schema version updates to + // ensure our DBSchema methods work correctly. + if (MarkerFileExists(aDirectoryMetadata)) { + NS_WARNING("Cache not shutdown cleanly! Cleaning up stale data..."); + mozStorageTransaction trans(aConn, false, + mozIStorageConnection::TRANSACTION_IMMEDIATE); + + QM_TRY(MOZ_TO_RESULT(trans.Start())); + + // Clean up orphaned Cache objects + QM_TRY_INSPECT(const auto& orphanedCacheIdList, + db::FindOrphanedCacheIds(*aConn)); + + QM_TRY_INSPECT( + const CheckedInt64& overallDeletedPaddingSize, + Reduce( + orphanedCacheIdList, CheckedInt64(0), + [aConn, &aDirectoryMetadata, &aDBDir]( + CheckedInt64 oldValue, const Maybe<const CacheId&>& element) + -> Result<CheckedInt64, nsresult> { + QM_TRY_INSPECT(const auto& deletionInfo, + db::DeleteCacheId(*aConn, *element)); + + QM_TRY(MOZ_TO_RESULT( + BodyDeleteFiles(aDirectoryMetadata, *aDBDir, + deletionInfo.mDeletedBodyIdList))); + + if (deletionInfo.mDeletedPaddingSize > 0) { + DecreaseUsageForDirectoryMetadata( + aDirectoryMetadata, deletionInfo.mDeletedPaddingSize); + } + + return oldValue + deletionInfo.mDeletedPaddingSize; + })); + + // Clean up orphaned body objects + QM_TRY_INSPECT(const auto& knownBodyIdList, db::GetKnownBodyIds(*aConn)); + + QM_TRY(MOZ_TO_RESULT(BodyDeleteOrphanedFiles(aDirectoryMetadata, *aDBDir, + knownBodyIdList))); + + // Commit() explicitly here, because we want to ensure the padding file + // has the correct content. + // We'll restore padding file below, so just warn here if failure happens. + // + // XXX Before, if MaybeUpdatePaddingFile failed but we didn't enter the if + // body below, we would have propagated the MaybeUpdatePaddingFile + // failure, but if we entered it and RestorePaddingFile succeeded, we + // would have returned NS_OK. Now, we will never propagate a + // MaybeUpdatePaddingFile failure. + QM_WARNONLY_TRY(QM_TO_RESULT( + MaybeUpdatePaddingFile(aDBDir, aConn, /* aIncreaceSize */ 0, + overallDeletedPaddingSize.value(), + [&trans]() { return trans.Commit(); }))); + } + + if (DirectoryPaddingFileExists(*aDBDir, DirPaddingFile::TMP_FILE) || + !DirectoryPaddingFileExists(*aDBDir, DirPaddingFile::FILE)) { + QM_TRY(MOZ_TO_RESULT(RestorePaddingFile(aDBDir, aConn))); + } + + return NS_OK; + } +}; + +// ---------------------------------------------------------------------------- + +// Action that is executed when we determine that content has stopped using +// a body file that has been orphaned. +class DeleteOrphanedBodyAction final : public Action { + public: + using DeletedBodyIdList = AutoTArray<nsID, 64>; + + explicit DeleteOrphanedBodyAction(DeletedBodyIdList&& aDeletedBodyIdList) + : mDeletedBodyIdList(std::move(aDeletedBodyIdList)) {} + + explicit DeleteOrphanedBodyAction(const nsID& aBodyId) + : mDeletedBodyIdList{aBodyId} {} + + void RunOnTarget(SafeRefPtr<Resolver> aResolver, + const Maybe<CacheDirectoryMetadata>& aDirectoryMetadata, + Data*) override { + MOZ_DIAGNOSTIC_ASSERT(aResolver); + MOZ_DIAGNOSTIC_ASSERT(aDirectoryMetadata); + MOZ_DIAGNOSTIC_ASSERT(aDirectoryMetadata->mDir); + + // Note that since DeleteOrphanedBodyAction isn't used while the context is + // being initialized, we don't need to check for cancellation here. + + const auto resolve = [&aResolver](const nsresult rv) { + aResolver->Resolve(rv); + }; + + QM_TRY_INSPECT(const auto& dbDir, + CloneFileAndAppend(*aDirectoryMetadata->mDir, u"cache"_ns), + QM_VOID, resolve); + + QM_TRY(MOZ_TO_RESULT(BodyDeleteFiles(*aDirectoryMetadata, *dbDir, + mDeletedBodyIdList)), + QM_VOID, resolve); + + aResolver->Resolve(NS_OK); + } + + private: + DeletedBodyIdList mDeletedBodyIdList; +}; + +bool IsHeadRequest(const CacheRequest& aRequest, + const CacheQueryParams& aParams) { + return !aParams.ignoreMethod() && + aRequest.method().LowerCaseEqualsLiteral("head"); +} + +bool IsHeadRequest(const Maybe<CacheRequest>& aRequest, + const CacheQueryParams& aParams) { + if (aRequest.isSome()) { + return !aParams.ignoreMethod() && + aRequest.ref().method().LowerCaseEqualsLiteral("head"); + } + return false; +} + +auto MatchByCacheId(CacheId aCacheId) { + return [aCacheId](const auto& entry) { return entry.mCacheId == aCacheId; }; +} + +auto MatchByBodyId(const nsID& aBodyId) { + return [&aBodyId](const auto& entry) { return entry.mBodyId == aBodyId; }; +} + +} // namespace + +// ---------------------------------------------------------------------------- + +// Singleton class to track Manager instances and ensure there is only +// one for each unique ManagerId. +class Manager::Factory { + public: + friend class StaticAutoPtr<Manager::Factory>; + + static Result<SafeRefPtr<Manager>, nsresult> AcquireCreateIfNonExistent( + const SafeRefPtr<ManagerId>& aManagerId) { + mozilla::ipc::AssertIsOnBackgroundThread(); + + // If we get here during/after quota manager shutdown, we bail out. + MOZ_ASSERT(AppShutdown::GetCurrentShutdownPhase() < + ShutdownPhase::AppShutdownQM); + if (AppShutdown::GetCurrentShutdownPhase() >= + ShutdownPhase::AppShutdownQM) { + NS_WARNING( + "Attempt to AcquireCreateIfNonExistent a Manager during QM " + "shutdown."); + return Err(NS_ERROR_ILLEGAL_DURING_SHUTDOWN); + } + + // Ensure there is a factory instance. This forces the Acquire() call + // below to use the same factory. + QM_TRY(MOZ_TO_RESULT(MaybeCreateInstance())); + + SafeRefPtr<Manager> ref = Acquire(*aManagerId); + if (!ref) { + // TODO: replace this with a thread pool (bug 1119864) + // XXX Can't use QM_TRY_INSPECT because that causes a clang-plugin + // error of the NoNewThreadsChecker. + nsCOMPtr<nsIThread> ioThread; + QM_TRY(MOZ_TO_RESULT( + NS_NewNamedThread("DOMCacheThread", getter_AddRefs(ioThread)))); + + ref = MakeSafeRefPtr<Manager>(aManagerId.clonePtr(), ioThread, + ConstructorGuard{}); + + // There may be an old manager for this origin in the process of + // cleaning up. We need to tell the new manager about this so + // that it won't actually start until the old manager is done. + const SafeRefPtr<Manager> oldManager = Acquire(*aManagerId, Closing); + ref->Init(oldManager.maybeDeref()); + + MOZ_ASSERT(!sFactory->mManagerList.Contains(ref)); + sFactory->mManagerList.AppendElement( + WrapNotNullUnchecked(ref.unsafeGetRawPtr())); + } + + return ref; + } + + static void Remove(Manager& aManager) { + mozilla::ipc::AssertIsOnBackgroundThread(); + MOZ_DIAGNOSTIC_ASSERT(sFactory); + + MOZ_ALWAYS_TRUE(sFactory->mManagerList.RemoveElement(&aManager)); + + // This might both happen in late shutdown such that this event + // is executed even after the QuotaManager singleton passed away + // or if the QuotaManager has not yet been created. + quota::QuotaManager::SafeMaybeRecordQuotaClientShutdownStep( + quota::Client::DOMCACHE, "Manager removed"_ns); + + // clean up the factory singleton if there are no more managers + MaybeDestroyInstance(); + } + + static void Abort(const Client::DirectoryLockIdTable& aDirectoryLockIds) { + mozilla::ipc::AssertIsOnBackgroundThread(); + + AbortMatching([&aDirectoryLockIds](const auto& manager) { + // Check if the Manager holds an acquired DirectoryLock. Origin clearing + // can't be blocked by this Manager if there is no acquired DirectoryLock. + // If there is an acquired DirectoryLock, check if the table contains the + // lock for the Manager. + return Client::IsLockForObjectAcquiredAndContainedInLockTable( + manager, aDirectoryLockIds); + }); + } + + static void AbortAll() { + mozilla::ipc::AssertIsOnBackgroundThread(); + + AbortMatching([](const auto&) { return true; }); + } + + static void ShutdownAll() { + mozilla::ipc::AssertIsOnBackgroundThread(); + + if (!sFactory) { + return; + } + + MOZ_DIAGNOSTIC_ASSERT(!sFactory->mManagerList.IsEmpty()); + + { + // Note that we are synchronously calling shutdown code here. If any + // of the shutdown code synchronously decides to delete the Factory + // we need to delay that delete until the end of this method. + AutoRestore<bool> restore(sFactory->mInSyncAbortOrShutdown); + sFactory->mInSyncAbortOrShutdown = true; + + for (const auto& manager : sFactory->mManagerList.ForwardRange()) { + auto pinnedManager = + SafeRefPtr{manager.get(), AcquireStrongRefFromRawPtr{}}; + pinnedManager->Shutdown(); + } + } + + MaybeDestroyInstance(); + } + + static bool IsShutdownAllComplete() { + mozilla::ipc::AssertIsOnBackgroundThread(); + return !sFactory; + } + +#ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED + static void RecordMayNotDeleteCSCP(int32_t aCacheStreamControlParentId) { + if (sFactory) { + sFactory->mPotentiallyUnreleasedCSCP.AppendElement( + aCacheStreamControlParentId); + } + } + + static void RecordHaveDeletedCSCP(int32_t aCacheStreamControlParentId) { + if (sFactory) { + sFactory->mPotentiallyUnreleasedCSCP.RemoveElement( + aCacheStreamControlParentId); + } + } +#endif + static nsCString GetShutdownStatus() { + mozilla::ipc::AssertIsOnBackgroundThread(); + + nsCString data; + + if (sFactory && !sFactory->mManagerList.IsEmpty()) { + data.Append( + "Managers: "_ns + + IntToCString(static_cast<uint64_t>(sFactory->mManagerList.Length())) + + " ("_ns); + + for (const auto& manager : sFactory->mManagerList.NonObservingRange()) { + data.Append(quota::AnonymizedOriginString( + manager->GetManagerId().QuotaOrigin())); + + data.AppendLiteral(": "); + + data.Append(manager->GetState() == State::Open ? "Open"_ns + : "Closing"_ns); + + data.AppendLiteral(", "); + } + + data.AppendLiteral(" ) "); + if (sFactory->mPotentiallyUnreleasedCSCP.Length() > 0) { + data.Append( + "There have been CSCP instances whose" + "Send__delete__ might not have freed them."); + } + } + + return data; + } + + private: + Factory() : mInSyncAbortOrShutdown(false) { + MOZ_COUNT_CTOR(cache::Manager::Factory); + } + + ~Factory() { + MOZ_COUNT_DTOR(cache::Manager::Factory); + MOZ_DIAGNOSTIC_ASSERT(mManagerList.IsEmpty()); + MOZ_DIAGNOSTIC_ASSERT(!mInSyncAbortOrShutdown); + } + + static nsresult MaybeCreateInstance() { + mozilla::ipc::AssertIsOnBackgroundThread(); + + if (!sFactory) { + // We cannot use ClearOnShutdown() here because we're not on the main + // thread. Instead, we delete sFactory in Factory::Remove() after the + // last manager is removed. ShutdownObserver ensures this happens + // before shutdown. + sFactory = new Factory(); + } + + // Never return sFactory to code outside Factory. We need to delete it + // out from under ourselves just before we return from Remove(). This + // would be (even more) dangerous if other code had a pointer to the + // factory itself. + + return NS_OK; + } + + static void MaybeDestroyInstance() { + mozilla::ipc::AssertIsOnBackgroundThread(); + MOZ_DIAGNOSTIC_ASSERT(sFactory); + + // If the factory is is still in use then we cannot delete yet. This + // could be due to managers still existing or because we are in the + // middle of aborting or shutting down. We need to be careful not to delete + // ourself synchronously during shutdown. + if (!sFactory->mManagerList.IsEmpty() || sFactory->mInSyncAbortOrShutdown) { + return; + } + + sFactory = nullptr; + } + + static SafeRefPtr<Manager> Acquire(const ManagerId& aManagerId, + State aState = Open) { + mozilla::ipc::AssertIsOnBackgroundThread(); + + QM_TRY(MOZ_TO_RESULT(MaybeCreateInstance()), nullptr); + + // Iterate in reverse to find the most recent, matching Manager. This + // is important when looking for a Closing Manager. If a new Manager + // chains to an old Manager we want it to be the most recent one. + const auto range = Reversed(sFactory->mManagerList.NonObservingRange()); + const auto foundIt = std::find_if( + range.begin(), range.end(), [aState, &aManagerId](const auto& manager) { + return aState == manager->GetState() && + *manager->mManagerId == aManagerId; + }); + return foundIt != range.end() + ? SafeRefPtr{foundIt->get(), AcquireStrongRefFromRawPtr{}} + : nullptr; + } + + template <typename Condition> + static void AbortMatching(const Condition& aCondition) { + mozilla::ipc::AssertIsOnBackgroundThread(); + + if (!sFactory) { + return; + } + + MOZ_DIAGNOSTIC_ASSERT(!sFactory->mManagerList.IsEmpty()); + + { + // Note that we are synchronously calling abort code here. If any + // of the shutdown code synchronously decides to delete the Factory + // we need to delay that delete until the end of this method. + AutoRestore<bool> restore(sFactory->mInSyncAbortOrShutdown); + sFactory->mInSyncAbortOrShutdown = true; + + for (const auto& manager : sFactory->mManagerList.ForwardRange()) { + if (aCondition(*manager)) { + auto pinnedManager = + SafeRefPtr{manager.get(), AcquireStrongRefFromRawPtr{}}; + pinnedManager->Abort(); + } + } + } + + MaybeDestroyInstance(); + } + + // Singleton created on demand and deleted when last Manager is cleared + // in Remove(). + // PBackground thread only. + static StaticAutoPtr<Factory> sFactory; + + // Weak references as we don't want to keep Manager objects alive forever. + // When a Manager is destroyed it calls Factory::Remove() to clear itself. + // PBackground thread only. + nsTObserverArray<NotNull<Manager*>> mManagerList; + + // This flag is set when we are looping through the list and calling Abort() + // or Shutdown() on each Manager. We need to be careful not to synchronously + // trigger the deletion of the factory while still executing this loop. + bool mInSyncAbortOrShutdown; + + nsTArray<int32_t> mPotentiallyUnreleasedCSCP; +}; + +// static +StaticAutoPtr<Manager::Factory> Manager::Factory::sFactory; + +// ---------------------------------------------------------------------------- + +// Abstract class to help implement the various Actions. The vast majority +// of Actions are synchronous and need to report back to a Listener on the +// Manager. +class Manager::BaseAction : public SyncDBAction { + protected: + BaseAction(SafeRefPtr<Manager> aManager, ListenerId aListenerId) + : SyncDBAction(DBAction::Existing), + mManager(std::move(aManager)), + mListenerId(aListenerId) {} + + virtual void Complete(Listener* aListener, ErrorResult&& aRv) = 0; + + virtual void CompleteOnInitiatingThread(nsresult aRv) override { + NS_ASSERT_OWNINGTHREAD(Manager::BaseAction); + Listener* listener = mManager->GetListener(mListenerId); + if (listener) { + Complete(listener, ErrorResult(aRv)); + } + + // ensure we release the manager on the initiating thread + mManager = nullptr; + } + + SafeRefPtr<Manager> mManager; + const ListenerId mListenerId; +}; + +// ---------------------------------------------------------------------------- + +// Action that is executed when we determine that content has stopped using +// a Cache object that has been orphaned. +class Manager::DeleteOrphanedCacheAction final : public SyncDBAction { + public: + DeleteOrphanedCacheAction(SafeRefPtr<Manager> aManager, CacheId aCacheId) + : SyncDBAction(DBAction::Existing), + mManager(std::move(aManager)), + mCacheId(aCacheId) {} + + virtual nsresult RunSyncWithDBOnTarget( + const CacheDirectoryMetadata& aDirectoryMetadata, nsIFile* aDBDir, + mozIStorageConnection* aConn) override { + mDirectoryMetadata.emplace(aDirectoryMetadata); + + mozStorageTransaction trans(aConn, false, + mozIStorageConnection::TRANSACTION_IMMEDIATE); + + QM_TRY(MOZ_TO_RESULT(trans.Start())); + + QM_TRY_UNWRAP(mDeletionInfo, db::DeleteCacheId(*aConn, mCacheId)); + + QM_TRY(MOZ_TO_RESULT(MaybeUpdatePaddingFile( + aDBDir, aConn, /* aIncreaceSize */ 0, mDeletionInfo.mDeletedPaddingSize, + [&trans]() mutable { return trans.Commit(); }))); + + return NS_OK; + } + + virtual void CompleteOnInitiatingThread(nsresult aRv) override { + // If the transaction fails, we shouldn't delete the body files and decrease + // their padding size. + if (NS_FAILED(aRv)) { + mDeletionInfo.mDeletedBodyIdList.Clear(); + mDeletionInfo.mDeletedPaddingSize = 0; + } + + mManager->NoteOrphanedBodyIdList(mDeletionInfo.mDeletedBodyIdList); + + if (mDeletionInfo.mDeletedPaddingSize > 0) { + DecreaseUsageForDirectoryMetadata(*mDirectoryMetadata, + mDeletionInfo.mDeletedPaddingSize); + } + + // ensure we release the manager on the initiating thread + mManager = nullptr; + } + + private: + SafeRefPtr<Manager> mManager; + const CacheId mCacheId; + DeletionInfo mDeletionInfo; + Maybe<CacheDirectoryMetadata> mDirectoryMetadata; +}; + +// ---------------------------------------------------------------------------- + +class Manager::CacheMatchAction final : public Manager::BaseAction { + public: + CacheMatchAction(SafeRefPtr<Manager> aManager, ListenerId aListenerId, + CacheId aCacheId, const CacheMatchArgs& aArgs, + SafeRefPtr<StreamList> aStreamList) + : BaseAction(std::move(aManager), aListenerId), + mCacheId(aCacheId), + mArgs(aArgs), + mStreamList(std::move(aStreamList)), + mFoundResponse(false) {} + + virtual nsresult RunSyncWithDBOnTarget( + const CacheDirectoryMetadata& aDirectoryMetadata, nsIFile* aDBDir, + mozIStorageConnection* aConn) override { + MOZ_DIAGNOSTIC_ASSERT(aDBDir); + + QM_TRY_INSPECT( + const auto& maybeResponse, + db::CacheMatch(*aConn, mCacheId, mArgs.request(), mArgs.params())); + + mFoundResponse = maybeResponse.isSome(); + if (mFoundResponse) { + mResponse = std::move(maybeResponse.ref()); + } + + if (!mFoundResponse || !mResponse.mHasBodyId || + IsHeadRequest(mArgs.request(), mArgs.params())) { + mResponse.mHasBodyId = false; + return NS_OK; + } + + nsCOMPtr<nsIInputStream> stream; + if (mArgs.openMode() == OpenMode::Eager) { + QM_TRY_UNWRAP(stream, + BodyOpen(aDirectoryMetadata, *aDBDir, mResponse.mBodyId)); + } + + mStreamList->Add(mResponse.mBodyId, std::move(stream)); + + return NS_OK; + } + + virtual void Complete(Listener* aListener, ErrorResult&& aRv) override { + if (!mFoundResponse) { + aListener->OnOpComplete(std::move(aRv), CacheMatchResult(Nothing())); + } else { + mStreamList->Activate(mCacheId); + aListener->OnOpComplete(std::move(aRv), CacheMatchResult(Nothing()), + mResponse, *mStreamList); + } + mStreamList = nullptr; + } + + virtual bool MatchesCacheId(CacheId aCacheId) const override { + return aCacheId == mCacheId; + } + + private: + const CacheId mCacheId; + const CacheMatchArgs mArgs; + SafeRefPtr<StreamList> mStreamList; + bool mFoundResponse; + SavedResponse mResponse; +}; + +// ---------------------------------------------------------------------------- + +class Manager::CacheMatchAllAction final : public Manager::BaseAction { + public: + CacheMatchAllAction(SafeRefPtr<Manager> aManager, ListenerId aListenerId, + CacheId aCacheId, const CacheMatchAllArgs& aArgs, + SafeRefPtr<StreamList> aStreamList) + : BaseAction(std::move(aManager), aListenerId), + mCacheId(aCacheId), + mArgs(aArgs), + mStreamList(std::move(aStreamList)) {} + + virtual nsresult RunSyncWithDBOnTarget( + const CacheDirectoryMetadata& aDirectoryMetadata, nsIFile* aDBDir, + mozIStorageConnection* aConn) override { + MOZ_DIAGNOSTIC_ASSERT(aDBDir); + + QM_TRY_UNWRAP(mSavedResponses, + db::CacheMatchAll(*aConn, mCacheId, mArgs.maybeRequest(), + mArgs.params())); + + for (uint32_t i = 0; i < mSavedResponses.Length(); ++i) { + if (!mSavedResponses[i].mHasBodyId || + IsHeadRequest(mArgs.maybeRequest(), mArgs.params())) { + mSavedResponses[i].mHasBodyId = false; + continue; + } + + nsCOMPtr<nsIInputStream> stream; + if (mArgs.openMode() == OpenMode::Eager) { + QM_TRY_UNWRAP(stream, BodyOpen(aDirectoryMetadata, *aDBDir, + mSavedResponses[i].mBodyId)); + } + + mStreamList->Add(mSavedResponses[i].mBodyId, std::move(stream)); + } + + return NS_OK; + } + + virtual void Complete(Listener* aListener, ErrorResult&& aRv) override { + mStreamList->Activate(mCacheId); + aListener->OnOpComplete(std::move(aRv), CacheMatchAllResult(), + mSavedResponses, *mStreamList); + mStreamList = nullptr; + } + + virtual bool MatchesCacheId(CacheId aCacheId) const override { + return aCacheId == mCacheId; + } + + private: + const CacheId mCacheId; + const CacheMatchAllArgs mArgs; + SafeRefPtr<StreamList> mStreamList; + nsTArray<SavedResponse> mSavedResponses; +}; + +// ---------------------------------------------------------------------------- + +// This is the most complex Action. It puts a request/response pair into the +// Cache. It does not complete until all of the body data has been saved to +// disk. This means its an asynchronous Action. +class Manager::CachePutAllAction final : public DBAction { + public: + CachePutAllAction( + SafeRefPtr<Manager> aManager, ListenerId aListenerId, CacheId aCacheId, + const nsTArray<CacheRequestResponse>& aPutList, + const nsTArray<nsCOMPtr<nsIInputStream>>& aRequestStreamList, + const nsTArray<nsCOMPtr<nsIInputStream>>& aResponseStreamList) + : DBAction(DBAction::Existing), + mManager(std::move(aManager)), + mListenerId(aListenerId), + mCacheId(aCacheId), + mList(aPutList.Length()), + mExpectedAsyncCopyCompletions(1), + mAsyncResult(NS_OK), + mMutex("cache::Manager::CachePutAllAction"), + mUpdatedPaddingSize(0), + mDeletedPaddingSize(0) { + MOZ_DIAGNOSTIC_ASSERT(!aPutList.IsEmpty()); + MOZ_DIAGNOSTIC_ASSERT(aPutList.Length() == aRequestStreamList.Length()); + MOZ_DIAGNOSTIC_ASSERT(aPutList.Length() == aResponseStreamList.Length()); + + for (uint32_t i = 0; i < aPutList.Length(); ++i) { + Entry* entry = mList.AppendElement(); + entry->mRequest = aPutList[i].request(); + entry->mRequestStream = aRequestStreamList[i]; + entry->mResponse = aPutList[i].response(); + entry->mResponseStream = aResponseStreamList[i]; + } + } + + private: + ~CachePutAllAction() = default; + + virtual void RunWithDBOnTarget( + SafeRefPtr<Resolver> aResolver, + const CacheDirectoryMetadata& aDirectoryMetadata, nsIFile* aDBDir, + mozIStorageConnection* aConn) override { + MOZ_DIAGNOSTIC_ASSERT(aResolver); + MOZ_DIAGNOSTIC_ASSERT(aDBDir); + MOZ_DIAGNOSTIC_ASSERT(aConn); + MOZ_DIAGNOSTIC_ASSERT(!mResolver); + MOZ_DIAGNOSTIC_ASSERT(!mDBDir); + MOZ_DIAGNOSTIC_ASSERT(!mConn); + + MOZ_DIAGNOSTIC_ASSERT(!mTarget); + mTarget = GetCurrentSerialEventTarget(); + MOZ_DIAGNOSTIC_ASSERT(mTarget); + + // We should be pre-initialized to expect one async completion. This is + // the "manual" completion we call at the end of this method in all + // cases. + MOZ_DIAGNOSTIC_ASSERT(mExpectedAsyncCopyCompletions == 1); + + mResolver = std::move(aResolver); + mDBDir = aDBDir; + mConn = aConn; + mDirectoryMetadata.emplace(aDirectoryMetadata); + + // File bodies are streamed to disk via asynchronous copying. Start + // this copying now. Each copy will eventually result in a call + // to OnAsyncCopyComplete(). + const nsresult rv = [this, &aDirectoryMetadata]() -> nsresult { + QM_TRY(CollectEachInRange( + mList, [this, &aDirectoryMetadata](auto& entry) -> nsresult { + QM_TRY(MOZ_TO_RESULT( + StartStreamCopy(aDirectoryMetadata, entry, RequestStream, + &mExpectedAsyncCopyCompletions))); + + QM_TRY(MOZ_TO_RESULT( + StartStreamCopy(aDirectoryMetadata, entry, ResponseStream, + &mExpectedAsyncCopyCompletions))); + + return NS_OK; + })); + + return NS_OK; + }(); + + // Always call OnAsyncCopyComplete() manually here. This covers the + // case where there is no async copying and also reports any startup + // errors correctly. If we hit an error, then OnAsyncCopyComplete() + // will cancel any async copying. + OnAsyncCopyComplete(rv); + } + + // Called once for each asynchronous file copy whether it succeeds or + // fails. If a file copy is canceled, it still calls this method with + // an error code. + void OnAsyncCopyComplete(nsresult aRv) { + MOZ_ASSERT(mTarget->IsOnCurrentThread()); + MOZ_DIAGNOSTIC_ASSERT(mConn); + MOZ_DIAGNOSTIC_ASSERT(mResolver); + MOZ_DIAGNOSTIC_ASSERT(mExpectedAsyncCopyCompletions > 0); + + // Explicitly check for cancellation here to catch a race condition. + // Consider: + // + // 1) NS_AsyncCopy() executes on IO thread, but has not saved its + // copy context yet. + // 2) CancelAllStreamCopying() occurs on PBackground thread + // 3) Copy context from (1) is saved on IO thread. + // + // Checking for cancellation here catches this condition when we + // first call OnAsyncCopyComplete() manually from RunWithDBOnTarget(). + // + // This explicit cancellation check also handles the case where we + // are canceled just after all stream copying completes. We should + // abort the synchronous DB operations in this case if we have not + // started them yet. + if (NS_SUCCEEDED(aRv) && IsCanceled()) { + aRv = NS_ERROR_ABORT; + } + + // If any of the async copies fail, we need to still wait for them all to + // complete. Cancel any other streams still working and remember the + // error. All canceled streams will call OnAsyncCopyComplete(). + if (NS_FAILED(aRv) && NS_SUCCEEDED(mAsyncResult)) { + CancelAllStreamCopying(); + mAsyncResult = aRv; + } + + // Check to see if async copying is still on-going. If so, then simply + // return for now. We must wait for a later OnAsyncCopyComplete() call. + mExpectedAsyncCopyCompletions -= 1; + if (mExpectedAsyncCopyCompletions > 0) { + return; + } + + // We have finished with all async copying. Indicate this by clearing all + // our copy contexts. + { + MutexAutoLock lock(mMutex); + mCopyContextList.Clear(); + } + + // An error occurred while async copying. Terminate the Action. + // DoResolve() will clean up any files we may have written. + if (NS_FAILED(mAsyncResult)) { + DoResolve(mAsyncResult); + return; + } + + mozStorageTransaction trans(mConn, false, + mozIStorageConnection::TRANSACTION_IMMEDIATE); + + QM_TRY(MOZ_TO_RESULT(trans.Start()), QM_VOID); + + const nsresult rv = [this, &trans]() -> nsresult { + QM_TRY(CollectEachInRange(mList, [this](Entry& e) -> nsresult { + if (e.mRequestStream) { + QM_TRY(MOZ_TO_RESULT(BodyFinalizeWrite(*mDBDir, e.mRequestBodyId))); + } + if (e.mResponseStream) { + // Gerenate padding size for opaque response if needed. + if (e.mResponse.type() == ResponseType::Opaque) { + // It'll generate padding if we've not set it yet. + QM_TRY(MOZ_TO_RESULT(BodyMaybeUpdatePaddingSize( + *mDirectoryMetadata, *mDBDir, e.mResponseBodyId, + e.mResponse.paddingInfo(), &e.mResponse.paddingSize()))); + + MOZ_DIAGNOSTIC_ASSERT(INT64_MAX - e.mResponse.paddingSize() >= + mUpdatedPaddingSize); + mUpdatedPaddingSize += e.mResponse.paddingSize(); + } + + QM_TRY(MOZ_TO_RESULT(BodyFinalizeWrite(*mDBDir, e.mResponseBodyId))); + } + + QM_TRY_UNWRAP( + auto deletionInfo, + db::CachePut(*mConn, mCacheId, e.mRequest, + e.mRequestStream ? &e.mRequestBodyId : nullptr, + e.mResponse, + e.mResponseStream ? &e.mResponseBodyId : nullptr)); + + const int64_t deletedPaddingSize = deletionInfo.mDeletedPaddingSize; + mDeletedBodyIdList = std::move(deletionInfo.mDeletedBodyIdList); + + MOZ_DIAGNOSTIC_ASSERT(INT64_MAX - mDeletedPaddingSize >= + deletedPaddingSize); + mDeletedPaddingSize += deletedPaddingSize; + + return NS_OK; + })); + + // Update padding file when it's necessary + QM_TRY(MOZ_TO_RESULT(MaybeUpdatePaddingFile( + mDBDir, mConn, mUpdatedPaddingSize, mDeletedPaddingSize, + [&trans]() mutable { return trans.Commit(); }))); + + return NS_OK; + }(); + + DoResolve(rv); + } + + virtual void CompleteOnInitiatingThread(nsresult aRv) override { + NS_ASSERT_OWNINGTHREAD(Action); + + for (uint32_t i = 0; i < mList.Length(); ++i) { + mList[i].mRequestStream = nullptr; + mList[i].mResponseStream = nullptr; + } + + // If the transaction fails, we shouldn't delete the body files and decrease + // their padding size. + if (NS_FAILED(aRv)) { + mDeletedBodyIdList.Clear(); + mDeletedPaddingSize = 0; + } + + mManager->NoteOrphanedBodyIdList(mDeletedBodyIdList); + + if (mDeletedPaddingSize > 0) { + DecreaseUsageForDirectoryMetadata(*mDirectoryMetadata, + mDeletedPaddingSize); + } + + Listener* listener = mManager->GetListener(mListenerId); + mManager = nullptr; + if (listener) { + listener->OnOpComplete(ErrorResult(aRv), CachePutAllResult()); + } + } + + virtual void CancelOnInitiatingThread() override { + NS_ASSERT_OWNINGTHREAD(Action); + Action::CancelOnInitiatingThread(); + CancelAllStreamCopying(); + } + + virtual bool MatchesCacheId(CacheId aCacheId) const override { + NS_ASSERT_OWNINGTHREAD(Action); + return aCacheId == mCacheId; + } + + struct Entry { + CacheRequest mRequest; + nsCOMPtr<nsIInputStream> mRequestStream; + nsID mRequestBodyId; + nsCOMPtr<nsISupports> mRequestCopyContext; + + CacheResponse mResponse; + nsCOMPtr<nsIInputStream> mResponseStream; + nsID mResponseBodyId; + nsCOMPtr<nsISupports> mResponseCopyContext; + }; + + enum StreamId { RequestStream, ResponseStream }; + + nsresult StartStreamCopy(const CacheDirectoryMetadata& aDirectoryMetadata, + Entry& aEntry, StreamId aStreamId, + uint32_t* aCopyCountOut) { + MOZ_ASSERT(mTarget->IsOnCurrentThread()); + MOZ_DIAGNOSTIC_ASSERT(aCopyCountOut); + + if (IsCanceled()) { + return NS_ERROR_ABORT; + } + + MOZ_DIAGNOSTIC_ASSERT(aStreamId == RequestStream || + aStreamId == ResponseStream); + + const auto& source = aStreamId == RequestStream ? aEntry.mRequestStream + : aEntry.mResponseStream; + + if (!source) { + return NS_OK; + } + + QM_TRY_INSPECT((const auto& [bodyId, copyContext]), + BodyStartWriteStream(aDirectoryMetadata, *mDBDir, *source, + this, AsyncCopyCompleteFunc)); + + if (aStreamId == RequestStream) { + aEntry.mRequestBodyId = bodyId; + } else { + aEntry.mResponseBodyId = bodyId; + } + + mBodyIdWrittenList.AppendElement(bodyId); + + if (copyContext) { + MutexAutoLock lock(mMutex); + mCopyContextList.AppendElement(copyContext); + } + + *aCopyCountOut += 1; + + return NS_OK; + } + + void CancelAllStreamCopying() { + // May occur on either owning thread or target thread + MutexAutoLock lock(mMutex); + for (uint32_t i = 0; i < mCopyContextList.Length(); ++i) { + MOZ_DIAGNOSTIC_ASSERT(mCopyContextList[i]); + BodyCancelWrite(*mCopyContextList[i]); + } + mCopyContextList.Clear(); + } + + static void AsyncCopyCompleteFunc(void* aClosure, nsresult aRv) { + // May be on any thread, including STS event target. + MOZ_DIAGNOSTIC_ASSERT(aClosure); + // Weak ref as we are guaranteed to the action is alive until + // CompleteOnInitiatingThread is called. + CachePutAllAction* action = static_cast<CachePutAllAction*>(aClosure); + action->CallOnAsyncCopyCompleteOnTargetThread(aRv); + } + + void CallOnAsyncCopyCompleteOnTargetThread(nsresult aRv) { + // May be on any thread, including STS event target. Non-owning runnable + // here since we are guaranteed the Action will survive until + // CompleteOnInitiatingThread is called. + nsCOMPtr<nsIRunnable> runnable = NewNonOwningRunnableMethod<nsresult>( + "dom::cache::Manager::CachePutAllAction::OnAsyncCopyComplete", this, + &CachePutAllAction::OnAsyncCopyComplete, aRv); + MOZ_ALWAYS_SUCCEEDS( + mTarget->Dispatch(runnable.forget(), nsIThread::DISPATCH_NORMAL)); + } + + void DoResolve(nsresult aRv) { + MOZ_ASSERT(mTarget->IsOnCurrentThread()); + + // DoResolve() must not be called until all async copying has completed. +#ifdef DEBUG + { + MutexAutoLock lock(mMutex); + MOZ_ASSERT(mCopyContextList.IsEmpty()); + } +#endif + + // Clean up any files we might have written before hitting the error. + if (NS_FAILED(aRv)) { + BodyDeleteFiles(*mDirectoryMetadata, *mDBDir, mBodyIdWrittenList); + if (mUpdatedPaddingSize > 0) { + DecreaseUsageForDirectoryMetadata(*mDirectoryMetadata, + mUpdatedPaddingSize); + } + } + + // Must be released on the target thread where it was opened. + mConn = nullptr; + + // Drop our ref to the target thread as we are done with this thread. + // Also makes our thread assertions catch any incorrect method calls + // after resolve. + mTarget = nullptr; + + // Make sure to de-ref the resolver per the Action API contract. + SafeRefPtr<Action::Resolver> resolver = std::move(mResolver); + resolver->Resolve(aRv); + } + + // initiating thread only + SafeRefPtr<Manager> mManager; + const ListenerId mListenerId; + + // Set on initiating thread, read on target thread. State machine guarantees + // these are not modified while being read by the target thread. + const CacheId mCacheId; + nsTArray<Entry> mList; + uint32_t mExpectedAsyncCopyCompletions; + + // target thread only + SafeRefPtr<Resolver> mResolver; + nsCOMPtr<nsIFile> mDBDir; + nsCOMPtr<mozIStorageConnection> mConn; + nsCOMPtr<nsISerialEventTarget> mTarget; + nsresult mAsyncResult; + nsTArray<nsID> mBodyIdWrittenList; + + // Written to on target thread, accessed on initiating thread after target + // thread activity is guaranteed complete + nsTArray<nsID> mDeletedBodyIdList; + + // accessed from any thread while mMutex locked + Mutex mMutex MOZ_UNANNOTATED; + nsTArray<nsCOMPtr<nsISupports>> mCopyContextList; + + Maybe<CacheDirectoryMetadata> mDirectoryMetadata; + // Track how much pad amount has been added for new entries so that it can be + // removed if an error occurs. + int64_t mUpdatedPaddingSize; + // Track any pad amount associated with overwritten entries. + int64_t mDeletedPaddingSize; +}; + +// ---------------------------------------------------------------------------- + +class Manager::CacheDeleteAction final : public Manager::BaseAction { + public: + CacheDeleteAction(SafeRefPtr<Manager> aManager, ListenerId aListenerId, + CacheId aCacheId, const CacheDeleteArgs& aArgs) + : BaseAction(std::move(aManager), aListenerId), + mCacheId(aCacheId), + mArgs(aArgs), + mSuccess(false) {} + + virtual nsresult RunSyncWithDBOnTarget( + const CacheDirectoryMetadata& aDirectoryMetadata, nsIFile* aDBDir, + mozIStorageConnection* aConn) override { + mDirectoryMetadata.emplace(aDirectoryMetadata); + + mozStorageTransaction trans(aConn, false, + mozIStorageConnection::TRANSACTION_IMMEDIATE); + + QM_TRY(MOZ_TO_RESULT(trans.Start())); + + QM_TRY_UNWRAP( + auto maybeDeletionInfo, + db::CacheDelete(*aConn, mCacheId, mArgs.request(), mArgs.params())); + + mSuccess = maybeDeletionInfo.isSome(); + if (mSuccess) { + mDeletionInfo = std::move(maybeDeletionInfo.ref()); + } + + QM_TRY(MOZ_TO_RESULT(MaybeUpdatePaddingFile( + aDBDir, aConn, /* aIncreaceSize */ 0, + mDeletionInfo.mDeletedPaddingSize, + [&trans]() mutable { return trans.Commit(); })), + QM_PROPAGATE, [this](const nsresult) { mSuccess = false; }); + + return NS_OK; + } + + virtual void Complete(Listener* aListener, ErrorResult&& aRv) override { + // If the transaction fails, we shouldn't delete the body files and decrease + // their padding size. + if (aRv.Failed()) { + mDeletionInfo.mDeletedBodyIdList.Clear(); + mDeletionInfo.mDeletedPaddingSize = 0; + } + + mManager->NoteOrphanedBodyIdList(mDeletionInfo.mDeletedBodyIdList); + + if (mDeletionInfo.mDeletedPaddingSize > 0) { + DecreaseUsageForDirectoryMetadata(*mDirectoryMetadata, + mDeletionInfo.mDeletedPaddingSize); + } + + aListener->OnOpComplete(std::move(aRv), CacheDeleteResult(mSuccess)); + } + + virtual bool MatchesCacheId(CacheId aCacheId) const override { + return aCacheId == mCacheId; + } + + private: + const CacheId mCacheId; + const CacheDeleteArgs mArgs; + bool mSuccess; + DeletionInfo mDeletionInfo; + Maybe<CacheDirectoryMetadata> mDirectoryMetadata; +}; + +// ---------------------------------------------------------------------------- + +class Manager::CacheKeysAction final : public Manager::BaseAction { + public: + CacheKeysAction(SafeRefPtr<Manager> aManager, ListenerId aListenerId, + CacheId aCacheId, const CacheKeysArgs& aArgs, + SafeRefPtr<StreamList> aStreamList) + : BaseAction(std::move(aManager), aListenerId), + mCacheId(aCacheId), + mArgs(aArgs), + mStreamList(std::move(aStreamList)) {} + + virtual nsresult RunSyncWithDBOnTarget( + const CacheDirectoryMetadata& aDirectoryMetadata, nsIFile* aDBDir, + mozIStorageConnection* aConn) override { + MOZ_DIAGNOSTIC_ASSERT(aDBDir); + + QM_TRY_UNWRAP( + mSavedRequests, + db::CacheKeys(*aConn, mCacheId, mArgs.maybeRequest(), mArgs.params())); + + for (uint32_t i = 0; i < mSavedRequests.Length(); ++i) { + if (!mSavedRequests[i].mHasBodyId || + IsHeadRequest(mArgs.maybeRequest(), mArgs.params())) { + mSavedRequests[i].mHasBodyId = false; + continue; + } + + nsCOMPtr<nsIInputStream> stream; + if (mArgs.openMode() == OpenMode::Eager) { + QM_TRY_UNWRAP(stream, BodyOpen(aDirectoryMetadata, *aDBDir, + mSavedRequests[i].mBodyId)); + } + + mStreamList->Add(mSavedRequests[i].mBodyId, std::move(stream)); + } + + return NS_OK; + } + + virtual void Complete(Listener* aListener, ErrorResult&& aRv) override { + mStreamList->Activate(mCacheId); + aListener->OnOpComplete(std::move(aRv), CacheKeysResult(), mSavedRequests, + *mStreamList); + mStreamList = nullptr; + } + + virtual bool MatchesCacheId(CacheId aCacheId) const override { + return aCacheId == mCacheId; + } + + private: + const CacheId mCacheId; + const CacheKeysArgs mArgs; + SafeRefPtr<StreamList> mStreamList; + nsTArray<SavedRequest> mSavedRequests; +}; + +// ---------------------------------------------------------------------------- + +class Manager::StorageMatchAction final : public Manager::BaseAction { + public: + StorageMatchAction(SafeRefPtr<Manager> aManager, ListenerId aListenerId, + Namespace aNamespace, const StorageMatchArgs& aArgs, + SafeRefPtr<StreamList> aStreamList) + : BaseAction(std::move(aManager), aListenerId), + mNamespace(aNamespace), + mArgs(aArgs), + mStreamList(std::move(aStreamList)), + mFoundResponse(false) {} + + virtual nsresult RunSyncWithDBOnTarget( + const CacheDirectoryMetadata& aDirectoryMetadata, nsIFile* aDBDir, + mozIStorageConnection* aConn) override { + MOZ_DIAGNOSTIC_ASSERT(aDBDir); + + auto maybeResponse = + db::StorageMatch(*aConn, mNamespace, mArgs.request(), mArgs.params()); + if (NS_WARN_IF(maybeResponse.isErr())) { + return maybeResponse.unwrapErr(); + } + + mFoundResponse = maybeResponse.inspect().isSome(); + if (mFoundResponse) { + mSavedResponse = maybeResponse.unwrap().ref(); + } + + if (!mFoundResponse || !mSavedResponse.mHasBodyId || + IsHeadRequest(mArgs.request(), mArgs.params())) { + mSavedResponse.mHasBodyId = false; + return NS_OK; + } + + nsCOMPtr<nsIInputStream> stream; + if (mArgs.openMode() == OpenMode::Eager) { + QM_TRY_UNWRAP(stream, BodyOpen(aDirectoryMetadata, *aDBDir, + mSavedResponse.mBodyId)); + } + + mStreamList->Add(mSavedResponse.mBodyId, std::move(stream)); + + return NS_OK; + } + + virtual void Complete(Listener* aListener, ErrorResult&& aRv) override { + if (!mFoundResponse) { + aListener->OnOpComplete(std::move(aRv), StorageMatchResult(Nothing())); + } else { + mStreamList->Activate(mSavedResponse.mCacheId); + aListener->OnOpComplete(std::move(aRv), StorageMatchResult(Nothing()), + mSavedResponse, *mStreamList); + } + mStreamList = nullptr; + } + + private: + const Namespace mNamespace; + const StorageMatchArgs mArgs; + SafeRefPtr<StreamList> mStreamList; + bool mFoundResponse; + SavedResponse mSavedResponse; +}; + +// ---------------------------------------------------------------------------- + +class Manager::StorageHasAction final : public Manager::BaseAction { + public: + StorageHasAction(SafeRefPtr<Manager> aManager, ListenerId aListenerId, + Namespace aNamespace, const StorageHasArgs& aArgs) + : BaseAction(std::move(aManager), aListenerId), + mNamespace(aNamespace), + mArgs(aArgs), + mCacheFound(false) {} + + virtual nsresult RunSyncWithDBOnTarget( + const CacheDirectoryMetadata& aDirectoryMetadata, nsIFile* aDBDir, + mozIStorageConnection* aConn) override { + QM_TRY_INSPECT(const auto& maybeCacheId, + db::StorageGetCacheId(*aConn, mNamespace, mArgs.key())); + + mCacheFound = maybeCacheId.isSome(); + + return NS_OK; + } + + virtual void Complete(Listener* aListener, ErrorResult&& aRv) override { + aListener->OnOpComplete(std::move(aRv), StorageHasResult(mCacheFound)); + } + + private: + const Namespace mNamespace; + const StorageHasArgs mArgs; + bool mCacheFound; +}; + +// ---------------------------------------------------------------------------- + +class Manager::StorageOpenAction final : public Manager::BaseAction { + public: + StorageOpenAction(SafeRefPtr<Manager> aManager, ListenerId aListenerId, + Namespace aNamespace, const StorageOpenArgs& aArgs) + : BaseAction(std::move(aManager), aListenerId), + mNamespace(aNamespace), + mArgs(aArgs), + mCacheId(INVALID_CACHE_ID) {} + + virtual nsresult RunSyncWithDBOnTarget( + const CacheDirectoryMetadata& aDirectoryMetadata, nsIFile* aDBDir, + mozIStorageConnection* aConn) override { + // Cache does not exist, create it instead + mozStorageTransaction trans(aConn, false, + mozIStorageConnection::TRANSACTION_IMMEDIATE); + + QM_TRY(MOZ_TO_RESULT(trans.Start())); + + // Look for existing cache + QM_TRY_INSPECT(const auto& maybeCacheId, + db::StorageGetCacheId(*aConn, mNamespace, mArgs.key())); + + if (maybeCacheId.isSome()) { + mCacheId = maybeCacheId.ref(); + MOZ_DIAGNOSTIC_ASSERT(mCacheId != INVALID_CACHE_ID); + return NS_OK; + } + + QM_TRY_UNWRAP(mCacheId, db::CreateCacheId(*aConn)); + + QM_TRY(MOZ_TO_RESULT( + db::StoragePutCache(*aConn, mNamespace, mArgs.key(), mCacheId))); + + QM_TRY(MOZ_TO_RESULT(trans.Commit())); + + MOZ_DIAGNOSTIC_ASSERT(mCacheId != INVALID_CACHE_ID); + return NS_OK; + } + + virtual void Complete(Listener* aListener, ErrorResult&& aRv) override { + MOZ_DIAGNOSTIC_ASSERT(aRv.Failed() || mCacheId != INVALID_CACHE_ID); + aListener->OnOpComplete( + std::move(aRv), StorageOpenResult((PCacheParent*)nullptr, mNamespace), + mCacheId); + } + + private: + const Namespace mNamespace; + const StorageOpenArgs mArgs; + CacheId mCacheId; +}; + +// ---------------------------------------------------------------------------- + +class Manager::StorageDeleteAction final : public Manager::BaseAction { + public: + StorageDeleteAction(SafeRefPtr<Manager> aManager, ListenerId aListenerId, + Namespace aNamespace, const StorageDeleteArgs& aArgs) + : BaseAction(std::move(aManager), aListenerId), + mNamespace(aNamespace), + mArgs(aArgs), + mCacheDeleted(false), + mCacheId(INVALID_CACHE_ID) {} + + virtual nsresult RunSyncWithDBOnTarget( + const CacheDirectoryMetadata& aDirectoryMetadata, nsIFile* aDBDir, + mozIStorageConnection* aConn) override { + mozStorageTransaction trans(aConn, false, + mozIStorageConnection::TRANSACTION_IMMEDIATE); + + QM_TRY(MOZ_TO_RESULT(trans.Start())); + + QM_TRY_INSPECT(const auto& maybeCacheId, + db::StorageGetCacheId(*aConn, mNamespace, mArgs.key())); + + if (maybeCacheId.isNothing()) { + mCacheDeleted = false; + return NS_OK; + } + mCacheId = maybeCacheId.ref(); + + // Don't delete the removing padding size here, we'll delete it on + // DeleteOrphanedCacheAction. + QM_TRY( + MOZ_TO_RESULT(db::StorageForgetCache(*aConn, mNamespace, mArgs.key()))); + + QM_TRY(MOZ_TO_RESULT(trans.Commit())); + + mCacheDeleted = true; + return NS_OK; + } + + virtual void Complete(Listener* aListener, ErrorResult&& aRv) override { + if (mCacheDeleted) { + // If content is referencing this cache, mark it orphaned to be + // deleted later. + if (!mManager->SetCacheIdOrphanedIfRefed(mCacheId)) { + // no outstanding references, delete immediately + const auto pinnedContext = + SafeRefPtr{mManager->mContext, AcquireStrongRefFromRawPtr{}}; + + if (pinnedContext->IsCanceled()) { + pinnedContext->NoteOrphanedData(); + } else { + pinnedContext->CancelForCacheId(mCacheId); + pinnedContext->Dispatch(MakeSafeRefPtr<DeleteOrphanedCacheAction>( + mManager.clonePtr(), mCacheId)); + } + } + } + + aListener->OnOpComplete(std::move(aRv), StorageDeleteResult(mCacheDeleted)); + } + + private: + const Namespace mNamespace; + const StorageDeleteArgs mArgs; + bool mCacheDeleted; + CacheId mCacheId; +}; + +// ---------------------------------------------------------------------------- + +class Manager::StorageKeysAction final : public Manager::BaseAction { + public: + StorageKeysAction(SafeRefPtr<Manager> aManager, ListenerId aListenerId, + Namespace aNamespace) + : BaseAction(std::move(aManager), aListenerId), mNamespace(aNamespace) {} + + virtual nsresult RunSyncWithDBOnTarget( + const CacheDirectoryMetadata& aDirectoryMetadata, nsIFile* aDBDir, + mozIStorageConnection* aConn) override { + QM_TRY_UNWRAP(mKeys, db::StorageGetKeys(*aConn, mNamespace)); + + return NS_OK; + } + + virtual void Complete(Listener* aListener, ErrorResult&& aRv) override { + if (aRv.Failed()) { + mKeys.Clear(); + } + aListener->OnOpComplete(std::move(aRv), StorageKeysResult(mKeys)); + } + + private: + const Namespace mNamespace; + nsTArray<nsString> mKeys; +}; + +// ---------------------------------------------------------------------------- + +class Manager::OpenStreamAction final : public Manager::BaseAction { + public: + OpenStreamAction(SafeRefPtr<Manager> aManager, ListenerId aListenerId, + InputStreamResolver&& aResolver, const nsID& aBodyId) + : BaseAction(std::move(aManager), aListenerId), + mResolver(std::move(aResolver)), + mBodyId(aBodyId) {} + + virtual nsresult RunSyncWithDBOnTarget( + const CacheDirectoryMetadata& aDirectoryMetadata, nsIFile* aDBDir, + mozIStorageConnection* aConn) override { + MOZ_DIAGNOSTIC_ASSERT(aDBDir); + + QM_TRY_UNWRAP(mBodyStream, BodyOpen(aDirectoryMetadata, *aDBDir, mBodyId)); + + return NS_OK; + } + + virtual void Complete(Listener* aListener, ErrorResult&& aRv) override { + if (aRv.Failed()) { + // Ignore the reason for fail and just pass a null input stream to let it + // fail. + aRv.SuppressException(); + mResolver(nullptr); + } else { + mResolver(std::move(mBodyStream)); + } + + mResolver = nullptr; + } + + private: + InputStreamResolver mResolver; + const nsID mBodyId; + nsCOMPtr<nsIInputStream> mBodyStream; +}; + +// ---------------------------------------------------------------------------- + +// static +Manager::ListenerId Manager::sNextListenerId = 0; + +void Manager::Listener::OnOpComplete(ErrorResult&& aRv, + const CacheOpResult& aResult) { + OnOpComplete(std::move(aRv), aResult, INVALID_CACHE_ID, Nothing()); +} + +void Manager::Listener::OnOpComplete(ErrorResult&& aRv, + const CacheOpResult& aResult, + CacheId aOpenedCacheId) { + OnOpComplete(std::move(aRv), aResult, aOpenedCacheId, Nothing()); +} + +void Manager::Listener::OnOpComplete(ErrorResult&& aRv, + const CacheOpResult& aResult, + const SavedResponse& aSavedResponse, + StreamList& aStreamList) { + AutoTArray<SavedResponse, 1> responseList; + responseList.AppendElement(aSavedResponse); + OnOpComplete( + std::move(aRv), aResult, INVALID_CACHE_ID, + Some(StreamInfo{responseList, nsTArray<SavedRequest>(), aStreamList})); +} + +void Manager::Listener::OnOpComplete( + ErrorResult&& aRv, const CacheOpResult& aResult, + const nsTArray<SavedResponse>& aSavedResponseList, + StreamList& aStreamList) { + OnOpComplete(std::move(aRv), aResult, INVALID_CACHE_ID, + Some(StreamInfo{aSavedResponseList, nsTArray<SavedRequest>(), + aStreamList})); +} + +void Manager::Listener::OnOpComplete( + ErrorResult&& aRv, const CacheOpResult& aResult, + const nsTArray<SavedRequest>& aSavedRequestList, StreamList& aStreamList) { + OnOpComplete(std::move(aRv), aResult, INVALID_CACHE_ID, + Some(StreamInfo{nsTArray<SavedResponse>(), aSavedRequestList, + aStreamList})); +} + +// static +Result<SafeRefPtr<Manager>, nsresult> Manager::AcquireCreateIfNonExistent( + const SafeRefPtr<ManagerId>& aManagerId) { + mozilla::ipc::AssertIsOnBackgroundThread(); + return Factory::AcquireCreateIfNonExistent(aManagerId); +} + +// static +void Manager::InitiateShutdown() { + mozilla::ipc::AssertIsOnBackgroundThread(); + + Factory::AbortAll(); + + Factory::ShutdownAll(); +} + +// static +bool Manager::IsShutdownAllComplete() { + mozilla::ipc::AssertIsOnBackgroundThread(); + + return Factory::IsShutdownAllComplete(); +} + +#ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED +void Manager::RecordMayNotDeleteCSCP(int32_t aCacheStreamControlParentId) { + Factory::RecordMayNotDeleteCSCP(aCacheStreamControlParentId); +} + +void Manager::RecordHaveDeletedCSCP(int32_t aCacheStreamControlParentId) { + Factory::RecordHaveDeletedCSCP(aCacheStreamControlParentId); +} +#endif + +// static +nsCString Manager::GetShutdownStatus() { + mozilla::ipc::AssertIsOnBackgroundThread(); + + return Factory::GetShutdownStatus(); +} + +// static +void Manager::Abort(const Client::DirectoryLockIdTable& aDirectoryLockIds) { + mozilla::ipc::AssertIsOnBackgroundThread(); + + Factory::Abort(aDirectoryLockIds); +} + +// static +void Manager::AbortAll() { + mozilla::ipc::AssertIsOnBackgroundThread(); + + Factory::AbortAll(); +} + +void Manager::RemoveListener(Listener* aListener) { + NS_ASSERT_OWNINGTHREAD(Manager); + // There may not be a listener here in the case where an actor is killed + // before it can perform any actual async requests on Manager. + mListeners.RemoveElement(aListener, ListenerEntryListenerComparator()); + MOZ_ASSERT( + !mListeners.Contains(aListener, ListenerEntryListenerComparator())); + MaybeAllowContextToClose(); +} + +void Manager::RemoveContext(Context& aContext) { + NS_ASSERT_OWNINGTHREAD(Manager); + MOZ_DIAGNOSTIC_ASSERT(mContext); + MOZ_DIAGNOSTIC_ASSERT(mContext == &aContext); + + // Whether the Context destruction was triggered from the Manager going + // idle or the underlying storage being invalidated, we should know we + // are closing before the Context is destroyed. + MOZ_DIAGNOSTIC_ASSERT(mState == Closing); + + // Before forgetting the Context, check to see if we have any outstanding + // cache or body objects waiting for deletion. If so, note that we've + // orphaned data so it will be cleaned up on the next open. + if (std::any_of( + mCacheIdRefs.cbegin(), mCacheIdRefs.cend(), + [](const auto& cacheIdRef) { return cacheIdRef.mOrphaned; }) || + std::any_of(mBodyIdRefs.cbegin(), mBodyIdRefs.cend(), + [](const auto& bodyIdRef) { return bodyIdRef.mOrphaned; })) { + aContext.NoteOrphanedData(); + } + + mContext = nullptr; + + // Once the context is gone, we can immediately remove ourself from the + // Factory list. We don't need to block shutdown by staying in the list + // any more. + Factory::Remove(*this); +} + +void Manager::NoteClosing() { + NS_ASSERT_OWNINGTHREAD(Manager); + // This can be called more than once legitimately through different paths. + mState = Closing; +} + +Manager::State Manager::GetState() const { + NS_ASSERT_OWNINGTHREAD(Manager); + return mState; +} + +void Manager::AddRefCacheId(CacheId aCacheId) { + NS_ASSERT_OWNINGTHREAD(Manager); + + const auto end = mCacheIdRefs.end(); + const auto foundIt = + std::find_if(mCacheIdRefs.begin(), end, MatchByCacheId(aCacheId)); + if (foundIt != end) { + foundIt->mCount += 1; + return; + } + + mCacheIdRefs.AppendElement(CacheIdRefCounter{aCacheId, 1, false}); +} + +void Manager::ReleaseCacheId(CacheId aCacheId) { + NS_ASSERT_OWNINGTHREAD(Manager); + + const auto end = mCacheIdRefs.end(); + const auto foundIt = + std::find_if(mCacheIdRefs.begin(), end, MatchByCacheId(aCacheId)); + if (foundIt != end) { +#ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED + const uint32_t oldRef = foundIt->mCount; +#endif + foundIt->mCount -= 1; + MOZ_DIAGNOSTIC_ASSERT(foundIt->mCount < oldRef); + if (foundIt->mCount == 0) { + const bool orphaned = foundIt->mOrphaned; + mCacheIdRefs.RemoveElementAt(foundIt); + const auto pinnedContext = + SafeRefPtr{mContext, AcquireStrongRefFromRawPtr{}}; + // If the context is already gone, then orphan flag should have been + // set in RemoveContext(). + if (orphaned && pinnedContext) { + if (pinnedContext->IsCanceled()) { + pinnedContext->NoteOrphanedData(); + } else { + pinnedContext->CancelForCacheId(aCacheId); + pinnedContext->Dispatch(MakeSafeRefPtr<DeleteOrphanedCacheAction>( + SafeRefPtrFromThis(), aCacheId)); + } + } + } + MaybeAllowContextToClose(); + return; + } + + MOZ_ASSERT_UNREACHABLE("Attempt to release CacheId that is not referenced!"); +} + +void Manager::AddRefBodyId(const nsID& aBodyId) { + NS_ASSERT_OWNINGTHREAD(Manager); + + const auto end = mBodyIdRefs.end(); + const auto foundIt = + std::find_if(mBodyIdRefs.begin(), end, MatchByBodyId(aBodyId)); + if (foundIt != end) { + foundIt->mCount += 1; + return; + } + + mBodyIdRefs.AppendElement(BodyIdRefCounter{aBodyId, 1, false}); +} + +void Manager::ReleaseBodyId(const nsID& aBodyId) { + NS_ASSERT_OWNINGTHREAD(Manager); + + const auto end = mBodyIdRefs.end(); + const auto foundIt = + std::find_if(mBodyIdRefs.begin(), end, MatchByBodyId(aBodyId)); + if (foundIt != end) { +#ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED + const uint32_t oldRef = foundIt->mCount; +#endif + foundIt->mCount -= 1; + MOZ_DIAGNOSTIC_ASSERT(foundIt->mCount < oldRef); + if (foundIt->mCount < 1) { + const bool orphaned = foundIt->mOrphaned; + mBodyIdRefs.RemoveElementAt(foundIt); + const auto pinnedContext = + SafeRefPtr{mContext, AcquireStrongRefFromRawPtr{}}; + // If the context is already gone, then orphan flag should have been + // set in RemoveContext(). + if (orphaned && pinnedContext) { + if (pinnedContext->IsCanceled()) { + pinnedContext->NoteOrphanedData(); + } else { + pinnedContext->Dispatch( + MakeSafeRefPtr<DeleteOrphanedBodyAction>(aBodyId)); + } + } + } + MaybeAllowContextToClose(); + return; + } + + MOZ_ASSERT_UNREACHABLE("Attempt to release BodyId that is not referenced!"); +} + +const ManagerId& Manager::GetManagerId() const { return *mManagerId; } + +void Manager::AddStreamList(StreamList& aStreamList) { + NS_ASSERT_OWNINGTHREAD(Manager); + mStreamLists.AppendElement(WrapNotNullUnchecked(&aStreamList)); +} + +void Manager::RemoveStreamList(StreamList& aStreamList) { + NS_ASSERT_OWNINGTHREAD(Manager); + mStreamLists.RemoveElement(&aStreamList); +} + +void Manager::ExecuteCacheOp(Listener* aListener, CacheId aCacheId, + const CacheOpArgs& aOpArgs) { + NS_ASSERT_OWNINGTHREAD(Manager); + MOZ_DIAGNOSTIC_ASSERT(aListener); + MOZ_DIAGNOSTIC_ASSERT(aOpArgs.type() != CacheOpArgs::TCachePutAllArgs); + + if (NS_WARN_IF(mState == Closing)) { + aListener->OnOpComplete(ErrorResult(NS_ERROR_FAILURE), void_t()); + return; + } + + const auto pinnedContext = SafeRefPtr{mContext, AcquireStrongRefFromRawPtr{}}; + MOZ_DIAGNOSTIC_ASSERT(!pinnedContext->IsCanceled()); + + auto action = [this, aListener, aCacheId, &aOpArgs, + &pinnedContext]() -> SafeRefPtr<Action> { + const ListenerId listenerId = SaveListener(aListener); + + if (CacheOpArgs::TCacheDeleteArgs == aOpArgs.type()) { + return MakeSafeRefPtr<CacheDeleteAction>(SafeRefPtrFromThis(), listenerId, + aCacheId, + aOpArgs.get_CacheDeleteArgs()); + } + + auto streamList = MakeSafeRefPtr<StreamList>(SafeRefPtrFromThis(), + pinnedContext.clonePtr()); + + switch (aOpArgs.type()) { + case CacheOpArgs::TCacheMatchArgs: + return MakeSafeRefPtr<CacheMatchAction>( + SafeRefPtrFromThis(), listenerId, aCacheId, + aOpArgs.get_CacheMatchArgs(), std::move(streamList)); + case CacheOpArgs::TCacheMatchAllArgs: + return MakeSafeRefPtr<CacheMatchAllAction>( + SafeRefPtrFromThis(), listenerId, aCacheId, + aOpArgs.get_CacheMatchAllArgs(), std::move(streamList)); + case CacheOpArgs::TCacheKeysArgs: + return MakeSafeRefPtr<CacheKeysAction>( + SafeRefPtrFromThis(), listenerId, aCacheId, + aOpArgs.get_CacheKeysArgs(), std::move(streamList)); + default: + MOZ_CRASH("Unknown Cache operation!"); + } + }(); + + pinnedContext->Dispatch(std::move(action)); +} + +void Manager::ExecuteStorageOp(Listener* aListener, Namespace aNamespace, + const CacheOpArgs& aOpArgs) { + NS_ASSERT_OWNINGTHREAD(Manager); + MOZ_DIAGNOSTIC_ASSERT(aListener); + + if (NS_WARN_IF(mState == Closing)) { + aListener->OnOpComplete(ErrorResult(NS_ERROR_FAILURE), void_t()); + return; + } + + const auto pinnedContext = SafeRefPtr{mContext, AcquireStrongRefFromRawPtr{}}; + MOZ_DIAGNOSTIC_ASSERT(!pinnedContext->IsCanceled()); + + auto action = [this, aListener, aNamespace, &aOpArgs, + &pinnedContext]() -> SafeRefPtr<Action> { + const ListenerId listenerId = SaveListener(aListener); + + switch (aOpArgs.type()) { + case CacheOpArgs::TStorageMatchArgs: + return MakeSafeRefPtr<StorageMatchAction>( + SafeRefPtrFromThis(), listenerId, aNamespace, + aOpArgs.get_StorageMatchArgs(), + MakeSafeRefPtr<StreamList>(SafeRefPtrFromThis(), + pinnedContext.clonePtr())); + case CacheOpArgs::TStorageHasArgs: + return MakeSafeRefPtr<StorageHasAction>(SafeRefPtrFromThis(), + listenerId, aNamespace, + aOpArgs.get_StorageHasArgs()); + case CacheOpArgs::TStorageOpenArgs: + return MakeSafeRefPtr<StorageOpenAction>(SafeRefPtrFromThis(), + listenerId, aNamespace, + aOpArgs.get_StorageOpenArgs()); + case CacheOpArgs::TStorageDeleteArgs: + return MakeSafeRefPtr<StorageDeleteAction>( + SafeRefPtrFromThis(), listenerId, aNamespace, + aOpArgs.get_StorageDeleteArgs()); + case CacheOpArgs::TStorageKeysArgs: + return MakeSafeRefPtr<StorageKeysAction>(SafeRefPtrFromThis(), + listenerId, aNamespace); + default: + MOZ_CRASH("Unknown CacheStorage operation!"); + } + }(); + + pinnedContext->Dispatch(std::move(action)); +} + +void Manager::ExecuteOpenStream(Listener* aListener, + InputStreamResolver&& aResolver, + const nsID& aBodyId) { + NS_ASSERT_OWNINGTHREAD(Manager); + MOZ_DIAGNOSTIC_ASSERT(aListener); + MOZ_DIAGNOSTIC_ASSERT(aResolver); + + if (NS_WARN_IF(mState == Closing)) { + aResolver(nullptr); + return; + } + + const auto pinnedContext = SafeRefPtr{mContext, AcquireStrongRefFromRawPtr{}}; + MOZ_DIAGNOSTIC_ASSERT(!pinnedContext->IsCanceled()); + + // We save the listener simply to track the existence of the caller here. + // Our returned value will really be passed to the resolver when the + // operation completes. In the future we should remove the Listener + // mechanism in favor of std::function or MozPromise. + ListenerId listenerId = SaveListener(aListener); + + pinnedContext->Dispatch(MakeSafeRefPtr<OpenStreamAction>( + SafeRefPtrFromThis(), listenerId, std::move(aResolver), aBodyId)); +} + +void Manager::ExecutePutAll( + Listener* aListener, CacheId aCacheId, + const nsTArray<CacheRequestResponse>& aPutList, + const nsTArray<nsCOMPtr<nsIInputStream>>& aRequestStreamList, + const nsTArray<nsCOMPtr<nsIInputStream>>& aResponseStreamList) { + NS_ASSERT_OWNINGTHREAD(Manager); + MOZ_DIAGNOSTIC_ASSERT(aListener); + + if (NS_WARN_IF(mState == Closing)) { + aListener->OnOpComplete(ErrorResult(NS_ERROR_FAILURE), CachePutAllResult()); + return; + } + + const auto pinnedContext = SafeRefPtr{mContext, AcquireStrongRefFromRawPtr{}}; + MOZ_DIAGNOSTIC_ASSERT(!pinnedContext->IsCanceled()); + + ListenerId listenerId = SaveListener(aListener); + pinnedContext->Dispatch(MakeSafeRefPtr<CachePutAllAction>( + SafeRefPtrFromThis(), listenerId, aCacheId, aPutList, aRequestStreamList, + aResponseStreamList)); +} + +Manager::Manager(SafeRefPtr<ManagerId> aManagerId, nsIThread* aIOThread, + const ConstructorGuard&) + : mManagerId(std::move(aManagerId)), + mIOThread(aIOThread), + mContext(nullptr), + mShuttingDown(false), + mState(Open) { + MOZ_DIAGNOSTIC_ASSERT(mManagerId); + MOZ_DIAGNOSTIC_ASSERT(mIOThread); +} + +Manager::~Manager() { + NS_ASSERT_OWNINGTHREAD(Manager); + MOZ_DIAGNOSTIC_ASSERT(mState == Closing); + MOZ_DIAGNOSTIC_ASSERT(!mContext); + + nsCOMPtr<nsIThread> ioThread; + mIOThread.swap(ioThread); + + // Don't spin the event loop in the destructor waiting for the thread to + // shutdown. Defer this to the main thread, instead. + MOZ_ALWAYS_SUCCEEDS(NS_DispatchToMainThread(NewRunnableMethod( + "nsIThread::AsyncShutdown", ioThread, &nsIThread::AsyncShutdown))); +} + +void Manager::Init(Maybe<Manager&> aOldManager) { + NS_ASSERT_OWNINGTHREAD(Manager); + + // Create the context immediately. Since there can at most be one Context + // per Manager now, this lets us cleanly call Factory::Remove() once the + // Context goes away. + SafeRefPtr<Context> ref = Context::Create( + SafeRefPtrFromThis(), mIOThread, MakeSafeRefPtr<SetupAction>(), + aOldManager ? SomeRef(*aOldManager->mContext) : Nothing()); + mContext = ref.unsafeGetRawPtr(); +} + +void Manager::Shutdown() { + NS_ASSERT_OWNINGTHREAD(Manager); + + // Ignore duplicate attempts to shutdown. This can occur when we start + // a browser initiated shutdown and then run ~Manager() which also + // calls Shutdown(). + if (mShuttingDown) { + return; + } + + mShuttingDown = true; + + // Note that we are closing to prevent any new requests from coming in and + // creating a new Context. We must ensure all Contexts and IO operations are + // complete before shutdown proceeds. + NoteClosing(); + + // If there is a context, then cancel and only note that we are done after + // its cleaned up. + if (mContext) { + const auto pinnedContext = + SafeRefPtr{mContext, AcquireStrongRefFromRawPtr{}}; + pinnedContext->CancelAll(); + return; + } +} + +Maybe<DirectoryLock&> Manager::MaybeDirectoryLockRef() const { + NS_ASSERT_OWNINGTHREAD(Manager); + MOZ_DIAGNOSTIC_ASSERT(mContext); + + return mContext->MaybeDirectoryLockRef(); +} + +void Manager::Abort() { + NS_ASSERT_OWNINGTHREAD(Manager); + MOZ_DIAGNOSTIC_ASSERT(mContext); + + // Note that we are closing to prevent any new requests from coming in and + // creating a new Context. We must ensure all Contexts and IO operations are + // complete before origin clear proceeds. + NoteClosing(); + + // Cancel and only note that we are done after the context is cleaned up. + const auto pinnedContext = SafeRefPtr{mContext, AcquireStrongRefFromRawPtr{}}; + pinnedContext->CancelAll(); +} + +Manager::ListenerId Manager::SaveListener(Listener* aListener) { + NS_ASSERT_OWNINGTHREAD(Manager); + + // Once a Listener is added, we keep a reference to it until its + // removed. Since the same Listener might make multiple requests, + // ensure we only have a single reference in our list. + ListenerList::index_type index = + mListeners.IndexOf(aListener, 0, ListenerEntryListenerComparator()); + if (index != ListenerList::NoIndex) { + return mListeners[index].mId; + } + + ListenerId id = sNextListenerId; + sNextListenerId += 1; + + mListeners.AppendElement(ListenerEntry(id, aListener)); + return id; +} + +Manager::Listener* Manager::GetListener(ListenerId aListenerId) const { + NS_ASSERT_OWNINGTHREAD(Manager); + ListenerList::index_type index = + mListeners.IndexOf(aListenerId, 0, ListenerEntryIdComparator()); + if (index != ListenerList::NoIndex) { + return mListeners[index].mListener; + } + + // This can legitimately happen if the actor is deleted while a request is + // in process. For example, the child process OOMs. + return nullptr; +} + +bool Manager::SetCacheIdOrphanedIfRefed(CacheId aCacheId) { + NS_ASSERT_OWNINGTHREAD(Manager); + + const auto end = mCacheIdRefs.end(); + const auto foundIt = + std::find_if(mCacheIdRefs.begin(), end, MatchByCacheId(aCacheId)); + if (foundIt != end) { + MOZ_DIAGNOSTIC_ASSERT(foundIt->mCount > 0); + MOZ_DIAGNOSTIC_ASSERT(!foundIt->mOrphaned); + foundIt->mOrphaned = true; + return true; + } + + return false; +} + +// TODO: provide way to set body non-orphaned if its added back to a cache (bug +// 1110479) + +bool Manager::SetBodyIdOrphanedIfRefed(const nsID& aBodyId) { + NS_ASSERT_OWNINGTHREAD(Manager); + + const auto end = mBodyIdRefs.end(); + const auto foundIt = + std::find_if(mBodyIdRefs.begin(), end, MatchByBodyId(aBodyId)); + if (foundIt != end) { + MOZ_DIAGNOSTIC_ASSERT(foundIt->mCount > 0); + MOZ_DIAGNOSTIC_ASSERT(!foundIt->mOrphaned); + foundIt->mOrphaned = true; + return true; + } + + return false; +} + +void Manager::NoteOrphanedBodyIdList(const nsTArray<nsID>& aDeletedBodyIdList) { + NS_ASSERT_OWNINGTHREAD(Manager); + + // XXX TransformIfIntoNewArray might be generalized to allow specifying the + // type of nsTArray to create, so that it can create an AutoTArray as well; an + // TransformIf (without AbortOnErr) might be added, which could be used here. + DeleteOrphanedBodyAction::DeletedBodyIdList deleteNowList; + deleteNowList.SetCapacity(aDeletedBodyIdList.Length()); + + std::copy_if(aDeletedBodyIdList.cbegin(), aDeletedBodyIdList.cend(), + MakeBackInserter(deleteNowList), + [this](const auto& deletedBodyId) { + return !SetBodyIdOrphanedIfRefed(deletedBodyId); + }); + + // TODO: note that we need to check these bodies for staleness on startup (bug + // 1110446) + const auto pinnedContext = SafeRefPtr{mContext, AcquireStrongRefFromRawPtr{}}; + if (!deleteNowList.IsEmpty() && pinnedContext && + !pinnedContext->IsCanceled()) { + pinnedContext->Dispatch( + MakeSafeRefPtr<DeleteOrphanedBodyAction>(std::move(deleteNowList))); + } +} + +void Manager::MaybeAllowContextToClose() { + NS_ASSERT_OWNINGTHREAD(Manager); + + // If we have an active context, but we have no more users of the Manager, + // then let it shut itself down. We must wait for all possible users of + // Cache state information to complete before doing this. Once we allow + // the Context to close we may not reliably get notified of storage + // invalidation. + const auto pinnedContext = SafeRefPtr{mContext, AcquireStrongRefFromRawPtr{}}; + if (pinnedContext && mListeners.IsEmpty() && mCacheIdRefs.IsEmpty() && + mBodyIdRefs.IsEmpty()) { + // Mark this Manager as invalid so that it won't get used again. We don't + // want to start any new operations once we allow the Context to close since + // it may race with the underlying storage getting invalidated. + NoteClosing(); + + pinnedContext->AllowToClose(); + } +} + +} // namespace mozilla::dom::cache diff --git a/dom/cache/Manager.h b/dom/cache/Manager.h new file mode 100644 index 0000000000..d5ebce6d48 --- /dev/null +++ b/dom/cache/Manager.h @@ -0,0 +1,310 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_Manager_h +#define mozilla_dom_cache_Manager_h + +#include "mozilla/RefPtr.h" +#include "mozilla/dom/SafeRefPtr.h" +#include "mozilla/dom/cache/Types.h" +#include "mozilla/dom/quota/Client.h" +#include "CacheCommon.h" +#include "nsCOMPtr.h" +#include "nsISupportsImpl.h" +#include "nsString.h" +#include "nsTArray.h" + +class nsIInputStream; +class nsIThread; + +namespace mozilla { + +class ErrorResult; + +namespace dom { + +namespace quota { + +class DirectoryLock; + +} // namespace quota + +namespace cache { + +class CacheOpArgs; +class CacheOpResult; +class CacheRequestResponse; +class Context; +class ManagerId; +struct SavedRequest; +struct SavedResponse; +class StreamList; + +// The Manager is class is responsible for performing all of the underlying +// work for a Cache or CacheStorage operation. The DOM objects and IPC actors +// are basically just plumbing to get the request to the right Manager object +// running in the parent process. +// +// There should be exactly one Manager object for each origin or app using the +// Cache API. This uniqueness is defined by the ManagerId equality operator. +// The uniqueness is enforced by the Manager GetOrCreate() factory method. +// +// The life cycle of Manager objects is somewhat complex. While code may +// hold a strong reference to the Manager, it will invalidate itself once it +// believes it has become completely idle. This is currently determined when +// all of the following conditions occur: +// +// 1) There are no more Manager::Listener objects registered with the Manager +// by performing a Cache or Storage operation. +// 2) There are no more CacheId references noted via Manager::AddRefCacheId(). +// 3) There are no more BodyId references noted via Manager::AddRefBodyId(). +// +// In order to keep your Manager alive you should perform an operation to set +// a Listener, call AddRefCacheId(), or call AddRefBodyId(). +// +// Even once a Manager becomes invalid, however, it may still continue to +// exist. This is allowed so that any in-progress Actions can gracefully +// complete. +// +// As an invariant, all Manager objects must cease all IO before shutdown. This +// is enforced by the Manager::Factory. If content still holds references to +// Cache DOM objects during shutdown, then all operations will begin rejecting. +class Manager final : public SafeRefCounted<Manager> { + using Client = quota::Client; + using DirectoryLock = quota::DirectoryLock; + + public: + // Callback interface implemented by clients of Manager, such as CacheParent + // and CacheStorageParent. In general, if you call a Manager method you + // should expect to receive exactly one On*() callback. For example, if + // you call Manager::CacheMatch(), then you should expect to receive + // OnCacheMatch() back in response. + // + // Listener objects are set on a per-operation basis. So you pass the + // Listener to a call like Manager::CacheMatch(). Once set in this way, + // the Manager will continue to reference the Listener until RemoveListener() + // is called. This is done to allow the same listener to be used for + // multiple operations simultaneously without having to maintain an exact + // count of operations-in-flight. + // + // Note, the Manager only holds weak references to Listener objects. + // Listeners must call Manager::RemoveListener() before they are destroyed + // to clear these weak references. + // + // All public methods should be invoked on the same thread used to create + // the Manager. + class Listener { + public: + // convenience routines + void OnOpComplete(ErrorResult&& aRv, const CacheOpResult& aResult); + + void OnOpComplete(ErrorResult&& aRv, const CacheOpResult& aResult, + CacheId aOpenedCacheId); + + void OnOpComplete(ErrorResult&& aRv, const CacheOpResult& aResult, + const SavedResponse& aSavedResponse, + StreamList& aStreamList); + + void OnOpComplete(ErrorResult&& aRv, const CacheOpResult& aResult, + const nsTArray<SavedResponse>& aSavedResponseList, + StreamList& aStreamList); + + void OnOpComplete(ErrorResult&& aRv, const CacheOpResult& aResult, + const nsTArray<SavedRequest>& aSavedRequestList, + StreamList& aStreamList); + + struct StreamInfo { + const nsTArray<SavedResponse>& mSavedResponseList; + const nsTArray<SavedRequest>& mSavedRequestList; + StreamList& mStreamList; + }; + + // interface to be implemented + virtual void OnOpComplete(ErrorResult&& aRv, const CacheOpResult& aResult, + CacheId aOpenedCacheId, + const Maybe<StreamInfo>& aStreamInfo) {} + + protected: + ~Listener() = default; + }; + + enum State { Open, Closing }; + + static Result<SafeRefPtr<Manager>, nsresult> AcquireCreateIfNonExistent( + const SafeRefPtr<ManagerId>& aManagerId); + + static void InitiateShutdown(); + + static bool IsShutdownAllComplete(); + + static nsCString GetShutdownStatus(); + + // Cancel actions for given DirectoryLock ids. + static void Abort(const Client::DirectoryLockIdTable& aDirectoryLockIds); + + // Cancel all actions. + static void AbortAll(); + + // Must be called by Listener objects before they are destroyed. + void RemoveListener(Listener* aListener); + + // Must be called by Context objects before they are destroyed. + void RemoveContext(Context& aContext); + + // Marks the Manager "invalid". Once the Context completes no new operations + // will be permitted with this Manager. New actors will get a new Manager. + void NoteClosing(); + + State GetState() const; + + // If an actor represents a long term reference to a cache or body stream, + // then they must call AddRefCacheId() or AddRefBodyId(). This will + // cause the Manager to keep the backing data store alive for the given + // object. The actor must then call ReleaseCacheId() or ReleaseBodyId() + // exactly once for every AddRef*() call it made. Any delayed deletion + // will then be performed. + void AddRefCacheId(CacheId aCacheId); + void ReleaseCacheId(CacheId aCacheId); + void AddRefBodyId(const nsID& aBodyId); + void ReleaseBodyId(const nsID& aBodyId); + + const ManagerId& GetManagerId() const; + + Maybe<DirectoryLock&> MaybeDirectoryLockRef() const; + + // Methods to allow a StreamList to register themselves with the Manager. + // StreamList objects must call RemoveStreamList() before they are destroyed. + void AddStreamList(StreamList& aStreamList); + void RemoveStreamList(StreamList& aStreamList); + + void ExecuteCacheOp(Listener* aListener, CacheId aCacheId, + const CacheOpArgs& aOpArgs); + void ExecutePutAll( + Listener* aListener, CacheId aCacheId, + const nsTArray<CacheRequestResponse>& aPutList, + const nsTArray<nsCOMPtr<nsIInputStream>>& aRequestStreamList, + const nsTArray<nsCOMPtr<nsIInputStream>>& aResponseStreamList); + + void ExecuteStorageOp(Listener* aListener, Namespace aNamespace, + const CacheOpArgs& aOpArgs); + + void ExecuteOpenStream(Listener* aListener, InputStreamResolver&& aResolver, + const nsID& aBodyId); + + void NoteStreamOpenComplete(const nsID& aBodyId, ErrorResult&& aRv, + nsCOMPtr<nsIInputStream>&& aBodyStream); + +#ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED + void RecordMayNotDeleteCSCP(int32_t aCacheStreamControlParentId); + void RecordHaveDeletedCSCP(int32_t aCacheStreamControlParentId); +#endif + + private: + class Factory; + class BaseAction; + class DeleteOrphanedCacheAction; + + class CacheMatchAction; + class CacheMatchAllAction; + class CachePutAllAction; + class CacheDeleteAction; + class CacheKeysAction; + + class StorageMatchAction; + class StorageHasAction; + class StorageOpenAction; + class StorageDeleteAction; + class StorageKeysAction; + + class OpenStreamAction; + + using ListenerId = uint64_t; + + void Init(Maybe<Manager&> aOldManager); + void Shutdown(); + + void Abort(); + + ListenerId SaveListener(Listener* aListener); + Listener* GetListener(ListenerId aListenerId) const; + + bool SetCacheIdOrphanedIfRefed(CacheId aCacheId); + bool SetBodyIdOrphanedIfRefed(const nsID& aBodyId); + void NoteOrphanedBodyIdList(const nsTArray<nsID>& aDeletedBodyIdList); + + void MaybeAllowContextToClose(); + + SafeRefPtr<ManagerId> mManagerId; + nsCOMPtr<nsIThread> mIOThread; + + // Weak reference cleared by RemoveContext() in Context destructor. + Context* MOZ_NON_OWNING_REF mContext; + + // Weak references cleared by RemoveListener() in Listener destructors. + struct ListenerEntry { + ListenerEntry() : mId(UINT64_MAX), mListener(nullptr) {} + + ListenerEntry(ListenerId aId, Listener* aListener) + : mId(aId), mListener(aListener) {} + + ListenerId mId; + Listener* mListener; + }; + + class ListenerEntryIdComparator { + public: + bool Equals(const ListenerEntry& aA, const ListenerId& aB) const { + return aA.mId == aB; + } + }; + + class ListenerEntryListenerComparator { + public: + bool Equals(const ListenerEntry& aA, const Listener* aB) const { + return aA.mListener == aB; + } + }; + + using ListenerList = nsTArray<ListenerEntry>; + ListenerList mListeners; + static ListenerId sNextListenerId; + + // Weak references cleared by RemoveStreamList() in StreamList destructors. + nsTArray<NotNull<StreamList*>> mStreamLists; + + bool mShuttingDown; + State mState; + + struct CacheIdRefCounter { + CacheId mCacheId; + MozRefCountType mCount; + bool mOrphaned; + }; + nsTArray<CacheIdRefCounter> mCacheIdRefs; + + struct BodyIdRefCounter { + nsID mBodyId; + MozRefCountType mCount; + bool mOrphaned; + }; + nsTArray<BodyIdRefCounter> mBodyIdRefs; + + struct ConstructorGuard {}; + + public: + Manager(SafeRefPtr<ManagerId> aManagerId, nsIThread* aIOThread, + const ConstructorGuard&); + ~Manager(); + + NS_DECL_OWNINGTHREAD + MOZ_DECLARE_REFCOUNTED_TYPENAME(cache::Manager) +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_cache_Manager_h diff --git a/dom/cache/ManagerId.cpp b/dom/cache/ManagerId.cpp new file mode 100644 index 0000000000..e83e9163c5 --- /dev/null +++ b/dom/cache/ManagerId.cpp @@ -0,0 +1,60 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/ManagerId.h" + +#include "CacheCommon.h" + +#include "mozilla/dom/quota/QuotaManager.h" +#include "nsIPrincipal.h" +#include "nsProxyRelease.h" +#include "mozilla/RefPtr.h" +#include "nsThreadUtils.h" + +namespace mozilla::dom::cache { + +using mozilla::dom::quota::QuotaManager; + +// static +Result<SafeRefPtr<ManagerId>, nsresult> ManagerId::Create( + nsIPrincipal* aPrincipal) { + MOZ_ASSERT(NS_IsMainThread()); + + // QuotaManager::GetOriginFromPrincipal() has special logic for system + // and about: principals. We need to use the same modified origin in + // order to interpret calls from QM correctly. + QM_TRY_INSPECT(const auto& quotaOrigin, + QuotaManager::GetOriginFromPrincipal(aPrincipal)); + + return MakeSafeRefPtr<ManagerId>(aPrincipal, quotaOrigin, ConstructorGuard{}); +} + +already_AddRefed<nsIPrincipal> ManagerId::Principal() const { + MOZ_ASSERT(NS_IsMainThread()); + nsCOMPtr<nsIPrincipal> ref = mPrincipal; + return ref.forget(); +} + +ManagerId::ManagerId(nsIPrincipal* aPrincipal, const nsACString& aQuotaOrigin, + ConstructorGuard) + : mPrincipal(aPrincipal), mQuotaOrigin(aQuotaOrigin) { + MOZ_DIAGNOSTIC_ASSERT(mPrincipal); +} + +ManagerId::~ManagerId() { + // If we're already on the main thread, then default destruction is fine + if (NS_IsMainThread()) { + return; + } + + // Otherwise we need to proxy to main thread to do the release + + // The PBackground worker thread shouldn't be running after the main thread + // is stopped. So main thread is guaranteed to exist here. + NS_ReleaseOnMainThread("ManagerId::mPrincipal", mPrincipal.forget()); +} + +} // namespace mozilla::dom::cache diff --git a/dom/cache/ManagerId.h b/dom/cache/ManagerId.h new file mode 100644 index 0000000000..60a74a632f --- /dev/null +++ b/dom/cache/ManagerId.h @@ -0,0 +1,56 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_ManagerId_h +#define mozilla_dom_cache_ManagerId_h + +#include "mozilla/Attributes.h" +#include "mozilla/dom/cache/Types.h" +#include "mozilla/dom/SafeRefPtr.h" +#include "nsCOMPtr.h" +#include "nsError.h" +#include "nsISupportsImpl.h" +#include "nsString.h" + +class nsIPrincipal; + +namespace mozilla::dom::cache { + +class ManagerId final : public AtomicSafeRefCounted<ManagerId> { + public: + // Main thread only + static Result<SafeRefPtr<ManagerId>, nsresult> Create( + nsIPrincipal* aPrincipal); + + // Main thread only + already_AddRefed<nsIPrincipal> Principal() const; + + const nsACString& QuotaOrigin() const { return mQuotaOrigin; } + + bool operator==(const ManagerId& aOther) const { + return mQuotaOrigin == aOther.mQuotaOrigin; + } + + private: + // only accessible on main thread + nsCOMPtr<nsIPrincipal> mPrincipal; + + // immutable to allow threadsfe access + const nsCString mQuotaOrigin; + + struct ConstructorGuard {}; + + public: + ManagerId(nsIPrincipal* aPrincipal, const nsACString& aOrigin, + ConstructorGuard); + ~ManagerId(); + + MOZ_DECLARE_REFCOUNTED_TYPENAME(mozilla::dom::cache::ManagerId) +}; + +} // namespace mozilla::dom::cache + +#endif // mozilla_dom_cache_ManagerId_h diff --git a/dom/cache/PCache.ipdl b/dom/cache/PCache.ipdl new file mode 100644 index 0000000000..3aa16a3ea2 --- /dev/null +++ b/dom/cache/PCache.ipdl @@ -0,0 +1,30 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +include protocol PBackground; +include protocol PCacheOp; +include protocol PCacheStreamControl; + +include CacheTypes; + +namespace mozilla { +namespace dom { +namespace cache { + +protocol PCache +{ + manager PBackground; + manages PCacheOp; + +parent: + async PCacheOp(CacheOpArgs aOpArgs); + async Teardown(); + +child: + async __delete__(); +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla diff --git a/dom/cache/PCacheOp.ipdl b/dom/cache/PCacheOp.ipdl new file mode 100644 index 0000000000..909fb2be73 --- /dev/null +++ b/dom/cache/PCacheOp.ipdl @@ -0,0 +1,29 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +include protocol PCache; +include protocol PCacheStorage; +include protocol PCacheStreamControl; + +include CacheTypes; + +include "ipc/ErrorIPCUtils.h"; + +[MoveOnly] using mozilla::ErrorResult from "mozilla/ErrorResult.h"; + +namespace mozilla { +namespace dom { +namespace cache { + +protocol PCacheOp +{ + manager PCache or PCacheStorage; + +child: + async __delete__(ErrorResult aRv, CacheOpResult aResult); +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla diff --git a/dom/cache/PCacheStorage.ipdl b/dom/cache/PCacheStorage.ipdl new file mode 100644 index 0000000000..0f1faf889e --- /dev/null +++ b/dom/cache/PCacheStorage.ipdl @@ -0,0 +1,31 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +include protocol PBackground; +include protocol PCache; +include protocol PCacheOp; +include protocol PCacheStreamControl; + +include CacheTypes; + +namespace mozilla { +namespace dom { +namespace cache { + +protocol PCacheStorage +{ + manager PBackground; + manages PCacheOp; + +parent: + async PCacheOp(CacheOpArgs aOpArgs); + async Teardown(); + +child: + async __delete__(); +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla diff --git a/dom/cache/PCacheStreamControl.ipdl b/dom/cache/PCacheStreamControl.ipdl new file mode 100644 index 0000000000..2d8a1aca05 --- /dev/null +++ b/dom/cache/PCacheStreamControl.ipdl @@ -0,0 +1,29 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +include protocol PBackground; +include IPCStream; + +using struct nsID from "nsID.h"; + +namespace mozilla { +namespace dom { +namespace cache { + +protocol PCacheStreamControl +{ + manager PBackground; + +parent: + async OpenStream(nsID aStreamId) returns(IPCStream? aStream); + async NoteClosed(nsID aStreamId); + +child: + async CloseAll(); + async __delete__(); +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla diff --git a/dom/cache/PrincipalVerifier.cpp b/dom/cache/PrincipalVerifier.cpp new file mode 100644 index 0000000000..16cc2b6470 --- /dev/null +++ b/dom/cache/PrincipalVerifier.cpp @@ -0,0 +1,176 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/PrincipalVerifier.h" + +#include "ErrorList.h" +#include "mozilla/dom/ContentParent.h" +#include "mozilla/dom/QMResult.h" +#include "mozilla/dom/cache/ManagerId.h" +#include "mozilla/dom/quota/ResultExtensions.h" +#include "mozilla/ipc/BackgroundParent.h" +#include "mozilla/ipc/PBackgroundParent.h" +#include "mozilla/ipc/BackgroundUtils.h" +#include "mozilla/BasePrincipal.h" +#include "CacheCommon.h" +#include "nsCOMPtr.h" +#include "nsContentUtils.h" +#include "nsIPrincipal.h" +#include "nsNetUtil.h" + +namespace mozilla::dom::cache { + +using mozilla::ipc::AssertIsOnBackgroundThread; +using mozilla::ipc::BackgroundParent; +using mozilla::ipc::PBackgroundParent; +using mozilla::ipc::PrincipalInfo; +using mozilla::ipc::PrincipalInfoToPrincipal; + +// static +already_AddRefed<PrincipalVerifier> PrincipalVerifier::CreateAndDispatch( + Listener& aListener, PBackgroundParent* aActor, + const PrincipalInfo& aPrincipalInfo) { + // We must get the ContentParent actor from the PBackgroundParent. This + // only works on the PBackground thread. + AssertIsOnBackgroundThread(); + + RefPtr<PrincipalVerifier> verifier = + new PrincipalVerifier(aListener, aActor, aPrincipalInfo); + + MOZ_ALWAYS_SUCCEEDS(NS_DispatchToMainThread(verifier)); + + return verifier.forget(); +} + +void PrincipalVerifier::AddListener(Listener& aListener) { + AssertIsOnBackgroundThread(); + MOZ_ASSERT(!mListenerList.Contains(&aListener)); + mListenerList.AppendElement(WrapNotNullUnchecked(&aListener)); +} + +void PrincipalVerifier::RemoveListener(Listener& aListener) { + AssertIsOnBackgroundThread(); + MOZ_ALWAYS_TRUE(mListenerList.RemoveElement(&aListener)); +} + +PrincipalVerifier::PrincipalVerifier(Listener& aListener, + PBackgroundParent* aActor, + const PrincipalInfo& aPrincipalInfo) + : Runnable("dom::cache::PrincipalVerifier"), + mHandle(BackgroundParent::GetContentParentHandle(aActor)), + mPrincipalInfo(aPrincipalInfo), + mInitiatingEventTarget(GetCurrentSerialEventTarget()), + mResult(NS_OK) { + AssertIsOnBackgroundThread(); + MOZ_DIAGNOSTIC_ASSERT(mInitiatingEventTarget); + + AddListener(aListener); +} + +PrincipalVerifier::~PrincipalVerifier() { + // Since the PrincipalVerifier is a Runnable that executes on multiple + // threads, its a race to see which thread de-refs us last. Therefore + // we cannot guarantee which thread we destruct on. + + MOZ_DIAGNOSTIC_ASSERT(mListenerList.IsEmpty()); +} + +NS_IMETHODIMP +PrincipalVerifier::Run() { + // Executed twice. First, on the main thread and then back on the + // originating thread. + + if (NS_IsMainThread()) { + VerifyOnMainThread(); + return NS_OK; + } + + CompleteOnInitiatingThread(); + return NS_OK; +} + +void PrincipalVerifier::VerifyOnMainThread() { + MOZ_ASSERT(NS_IsMainThread()); + + QM_TRY_INSPECT( + const auto& principal, PrincipalInfoToPrincipal(mPrincipalInfo), QM_VOID, + [this](const nsresult result) { DispatchToInitiatingThread(result); }); + + // We disallow null principal on the client side, but double-check here. + if (NS_WARN_IF(principal->GetIsNullPrincipal())) { + DispatchToInitiatingThread(NS_ERROR_FAILURE); + return; + } + + // Verify if a child process uses system principal, which is not allowed + // to prevent system principal is spoofed. + if (NS_WARN_IF(mHandle && principal->IsSystemPrincipal())) { + DispatchToInitiatingThread(NS_ERROR_FAILURE); + return; + } + +#ifdef DEBUG + nsresult rv = NS_OK; + // Sanity check principal origin by using it to construct a URI and security + // checking it. Don't do this for the system principal, though, as its origin + // is a synthetic [System Principal] string. + if (!principal->IsSystemPrincipal()) { + nsAutoCString origin; + rv = principal->GetOriginNoSuffix(origin); + if (NS_WARN_IF(NS_FAILED(rv))) { + DispatchToInitiatingThread(rv); + return; + } + nsCOMPtr<nsIURI> uri; + rv = NS_NewURI(getter_AddRefs(uri), origin); + if (NS_WARN_IF(NS_FAILED(rv))) { + DispatchToInitiatingThread(rv); + return; + } + rv = principal->CheckMayLoad(uri, false); + if (NS_WARN_IF(NS_FAILED(rv))) { + DispatchToInitiatingThread(rv); + return; + } + } +#endif + + auto managerIdOrErr = ManagerId::Create(principal); + if (NS_WARN_IF(managerIdOrErr.isErr())) { + DispatchToInitiatingThread(managerIdOrErr.unwrapErr()); + return; + } + mManagerId = managerIdOrErr.unwrap(); + + DispatchToInitiatingThread(NS_OK); +} + +void PrincipalVerifier::CompleteOnInitiatingThread() { + AssertIsOnBackgroundThread(); + + for (const auto& listener : mListenerList.ForwardRange()) { + listener->OnPrincipalVerified(mResult, mManagerId); + } + + // The listener must clear its reference in OnPrincipalVerified() + MOZ_DIAGNOSTIC_ASSERT(mListenerList.IsEmpty()); +} + +void PrincipalVerifier::DispatchToInitiatingThread(nsresult aRv) { + MOZ_ASSERT(NS_IsMainThread()); + + mResult = aRv; + + // The Cache ShutdownObserver does not track all principal verifiers, so we + // cannot ensure this always succeeds. Instead, simply warn on failures. + // This will result in a new CacheStorage object delaying operations until + // shutdown completes and the browser goes away. This is as graceful as + // we can get here. + QM_WARNONLY_TRY(QM_TO_RESULT( + mInitiatingEventTarget->Dispatch(this, nsIThread::DISPATCH_NORMAL))); +} + +} // namespace mozilla::dom::cache diff --git a/dom/cache/PrincipalVerifier.h b/dom/cache/PrincipalVerifier.h new file mode 100644 index 0000000000..d464bc4d01 --- /dev/null +++ b/dom/cache/PrincipalVerifier.h @@ -0,0 +1,80 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_PrincipalVerifier_h +#define mozilla_dom_cache_PrincipalVerifier_h + +#include "mozilla/dom/SafeRefPtr.h" +#include "mozilla/ipc/PBackgroundSharedTypes.h" +#include "nsThreadUtils.h" +#include "nsTObserverArray.h" + +namespace mozilla { + +namespace ipc { +class PBackgroundParent; +} // namespace ipc + +namespace dom { + +class ThreadsafeContentParentHandle; + +namespace cache { + +class ManagerId; + +class PrincipalVerifier final : public Runnable { + public: + // An interface to be implemented by code wishing to use the + // PrincipalVerifier. Note, the Listener implementation is responsible + // for calling RemoveListener() on the PrincipalVerifier to clear the + // weak reference. + class Listener { + public: + virtual void OnPrincipalVerified( + nsresult aRv, const SafeRefPtr<ManagerId>& aManagerId) = 0; + }; + + static already_AddRefed<PrincipalVerifier> CreateAndDispatch( + Listener& aListener, mozilla::ipc::PBackgroundParent* aActor, + const mozilla::ipc::PrincipalInfo& aPrincipalInfo); + + void AddListener(Listener& aListener); + + // The Listener must call RemoveListener() when OnPrincipalVerified() is + // called or when the Listener is destroyed. + void RemoveListener(Listener& aListener); + + private: + PrincipalVerifier(Listener& aListener, + mozilla::ipc::PBackgroundParent* aActor, + const mozilla::ipc::PrincipalInfo& aPrincipalInfo); + virtual ~PrincipalVerifier(); + + void VerifyOnMainThread(); + void CompleteOnInitiatingThread(); + + void DispatchToInitiatingThread(nsresult aRv); + + // Weak reference cleared by RemoveListener() + nsTObserverArray<NotNull<Listener*>> mListenerList; + + RefPtr<ThreadsafeContentParentHandle> mHandle; + + const mozilla::ipc::PrincipalInfo mPrincipalInfo; + nsCOMPtr<nsIEventTarget> mInitiatingEventTarget; + nsresult mResult; + SafeRefPtr<ManagerId> mManagerId; + + public: + NS_DECL_NSIRUNNABLE +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_cache_PrincipalVerifier_h diff --git a/dom/cache/QuotaClient.cpp b/dom/cache/QuotaClient.cpp new file mode 100644 index 0000000000..565450964f --- /dev/null +++ b/dom/cache/QuotaClient.cpp @@ -0,0 +1,541 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "QuotaClientImpl.h" + +#include "DBAction.h" +#include "FileUtilsImpl.h" +#include "mozilla/ResultExtensions.h" +#include "mozilla/Telemetry.h" +#include "mozilla/Unused.h" +#include "mozilla/dom/cache/DBSchema.h" +#include "mozilla/dom/cache/Manager.h" +#include "mozilla/dom/quota/QuotaCommon.h" +#include "mozilla/dom/quota/QuotaManager.h" +#include "mozilla/dom/quota/UsageInfo.h" +#include "mozilla/ipc/BackgroundParent.h" +#include "nsIFile.h" +#include "nsThreadUtils.h" + +namespace mozilla::dom::cache { + +using mozilla::dom::quota::AssertIsOnIOThread; +using mozilla::dom::quota::Client; +using mozilla::dom::quota::CloneFileAndAppend; +using mozilla::dom::quota::DatabaseUsageType; +using mozilla::dom::quota::GetDirEntryKind; +using mozilla::dom::quota::nsIFileKind; +using mozilla::dom::quota::OriginMetadata; +using mozilla::dom::quota::PERSISTENCE_TYPE_DEFAULT; +using mozilla::dom::quota::PersistenceType; +using mozilla::dom::quota::QuotaManager; +using mozilla::dom::quota::UsageInfo; +using mozilla::ipc::AssertIsOnBackgroundThread; + +namespace { + +template <typename StepFunc> +Result<UsageInfo, nsresult> ReduceUsageInfo(nsIFile& aDir, + const Atomic<bool>& aCanceled, + const StepFunc& aStepFunc) { + QM_TRY_RETURN(quota::ReduceEachFileAtomicCancelable( + aDir, aCanceled, UsageInfo{}, + [&aStepFunc](UsageInfo usageInfo, const nsCOMPtr<nsIFile>& bodyDir) + -> Result<UsageInfo, nsresult> { + QM_TRY(OkIf(!QuotaManager::IsShuttingDown()), Err(NS_ERROR_ABORT)); + + QM_TRY_INSPECT(const auto& stepUsageInfo, aStepFunc(bodyDir)); + + return usageInfo + stepUsageInfo; + })); +} + +Result<UsageInfo, nsresult> GetBodyUsage(nsIFile& aMorgueDir, + const Atomic<bool>& aCanceled) { + AssertIsOnIOThread(); + + QM_TRY_RETURN(ReduceUsageInfo( + aMorgueDir, aCanceled, + [](const nsCOMPtr<nsIFile>& bodyDir) -> Result<UsageInfo, nsresult> { + QM_TRY_INSPECT(const auto& dirEntryKind, GetDirEntryKind(*bodyDir)); + + if (dirEntryKind != nsIFileKind::ExistsAsDirectory) { + if (dirEntryKind == nsIFileKind::ExistsAsFile) { + const DebugOnly<nsresult> result = + RemoveNsIFile(Nothing(), *bodyDir, /* aTrackQuota */ false); + // Try to remove the unexpected files, and keep moving on even if it + // fails because it might be created by virus or the operation + // system + MOZ_ASSERT(NS_SUCCEEDED(result)); + } + + return UsageInfo{}; + } + + UsageInfo usageInfo; + const auto getUsage = + [&usageInfo](nsIFile& bodyFile, + const nsACString& leafName) -> Result<bool, nsresult> { + Unused << leafName; + + QM_TRY_INSPECT(const int64_t& fileSize, + MOZ_TO_RESULT_INVOKE_MEMBER(bodyFile, GetFileSize)); + MOZ_DIAGNOSTIC_ASSERT(fileSize >= 0); + // FIXME: Separate file usage and database usage in OriginInfo so that + // the workaround for treating body file size as database usage can be + // removed. + // + // This is needed because we want to remove the mutex lock for padding + // files. The lock is needed because the padding file is accessed on + // the QM IO thread while getting origin usage and is accessed on the + // Cache IO thread in normal Cache operations. Using the cached usage + // in QM while getting origin usage can remove the access on the QM IO + // thread and thus we can remove the mutex lock. However, QM only + // separates usage types in initialization, and the separation is gone + // after that. So, before extending the separation of usage types in + // QM, this is a workaround to avoid the file usage mismatching in our + // tests. Note that file usage hasn't been exposed to users yet. + usageInfo += DatabaseUsageType(Some(fileSize)); + + return false; + }; + + // QM_OR_ELSE_WARN_IF is not used here since we just want to log + // NS_ERROR_FILE_FS_CORRUPTED result and not spam the reports (even a + // warning in the reports is not desired). + QM_TRY(QM_OR_ELSE_LOG_VERBOSE_IF( + // Expression. + MOZ_TO_RESULT(BodyTraverseFiles(Nothing(), *bodyDir, getUsage, + /* aCanRemoveFiles */ true, + /* aTrackQuota */ false)), + // Predicate. + IsSpecificError<NS_ERROR_FILE_FS_CORRUPTED>, + // Fallback. We treat NS_ERROR_FILE_FS_CORRUPTED as if the + // directory did not exist at all. + ErrToDefaultOk<>)); + return usageInfo; + })); +} + +Result<int64_t, nsresult> GetPaddingSizeFromDB( + nsIFile& aDir, nsIFile& aDBFile, const OriginMetadata& aOriginMetadata) { + CacheDirectoryMetadata directoryMetadata(aOriginMetadata); + // directoryMetadata.mDirectoryLockId must be -1 (which is default for new + // CacheDirectoryMetadata) because this method should only be called from + // QuotaClient::InitOrigin when the temporary storage hasn't been initialized + // yet. At that time, the in-memory objects (e.g. OriginInfo) are only being + // created so it doesn't make sense to tunnel quota information to QuotaVFS + // to get corresponding QuotaObject instance for the SQLite file). + MOZ_DIAGNOSTIC_ASSERT(directoryMetadata.mDirectoryLockId == -1); + +#ifdef DEBUG + { + QM_TRY_INSPECT(const bool& exists, + MOZ_TO_RESULT_INVOKE_MEMBER(aDBFile, Exists)); + MOZ_ASSERT(exists); + } +#endif + + QM_TRY_INSPECT(const auto& conn, + OpenDBConnection(directoryMetadata, aDBFile)); + + // Make sure that the database has the latest schema before we try to read + // from it. We have to do this because GetPaddingSizeFromDB is called + // by InitOrigin. And it means that SetupAction::RunSyncWithDBOnTarget hasn't + // checked the schema for the given origin yet). + QM_TRY(MOZ_TO_RESULT(db::CreateOrMigrateSchema(*conn))); + + QM_TRY_RETURN(DirectoryPaddingRestore(aDir, *conn, + /* aMustRestore */ false)); +} + +} // namespace + +const nsLiteralString kCachesSQLiteFilename = u"caches.sqlite"_ns; +const nsLiteralString kMorgueDirectoryFilename = u"morgue"_ns; + +CacheQuotaClient::CacheQuotaClient() { + AssertIsOnBackgroundThread(); + MOZ_DIAGNOSTIC_ASSERT(!sInstance); + sInstance = this; +} + +// static +CacheQuotaClient* CacheQuotaClient::Get() { + MOZ_DIAGNOSTIC_ASSERT(sInstance); + return sInstance; +} + +CacheQuotaClient::Type CacheQuotaClient::GetType() { return DOMCACHE; } + +Result<UsageInfo, nsresult> CacheQuotaClient::InitOrigin( + PersistenceType aPersistenceType, const OriginMetadata& aOriginMetadata, + const AtomicBool& aCanceled) { + AssertIsOnIOThread(); + MOZ_ASSERT(aOriginMetadata.mPersistenceType == aPersistenceType); + + QuotaManager* const qm = QuotaManager::Get(); + MOZ_DIAGNOSTIC_ASSERT(qm); + + QM_TRY_INSPECT(const auto& dir, qm->GetOriginDirectory(aOriginMetadata)); + + QM_TRY(MOZ_TO_RESULT( + dir->Append(NS_LITERAL_STRING_FROM_CSTRING(DOMCACHE_DIRECTORY_NAME)))); + + QM_TRY_INSPECT( + const auto& cachesSQLiteFile, + ([dir]() -> Result<nsCOMPtr<nsIFile>, nsresult> { + QM_TRY_INSPECT(const auto& cachesSQLite, + CloneFileAndAppend(*dir, kCachesSQLiteFilename)); + + // IsDirectory is used to check if caches.sqlite exists or not. Another + // benefit of this is that we can test the failed cases by creating a + // directory named "caches.sqlite". + QM_TRY_INSPECT(const auto& dirEntryKind, + GetDirEntryKind(*cachesSQLite)); + if (dirEntryKind == nsIFileKind::DoesNotExist) { + // We only ensure padding files and morgue directory get removed like + // WipeDatabase in DBAction.cpp. The -wal journal file will be + // automatically deleted by sqlite when the new database is created. + // XXX Ideally, we would delete the -wal journal file as well (here + // and also in WipeDatabase). + // XXX We should have something like WipeDatabaseNoQuota for this. + // XXX Long term, we might even think about removing entire origin + // directory because missing caches.sqlite while other files exist can + // be interpreted as database corruption. + QM_TRY(MOZ_TO_RESULT(mozilla::dom::cache::DirectoryPaddingDeleteFile( + *dir, DirPaddingFile::TMP_FILE))); + + QM_TRY(MOZ_TO_RESULT(mozilla::dom::cache::DirectoryPaddingDeleteFile( + *dir, DirPaddingFile::FILE))); + + QM_TRY_INSPECT(const auto& morgueDir, + CloneFileAndAppend(*dir, kMorgueDirectoryFilename)); + + QM_TRY(MOZ_TO_RESULT(mozilla::dom::cache::RemoveNsIFileRecursively( + Nothing(), *morgueDir, + /* aTrackQuota */ false))); + + return nsCOMPtr<nsIFile>{nullptr}; + } + + QM_TRY(OkIf(dirEntryKind == nsIFileKind::ExistsAsFile), + Err(NS_ERROR_FAILURE)); + + return cachesSQLite; + }())); + + // If the caches.sqlite doesn't exist, then padding files and morgue directory + // should have been removed if they existed. We ignore the rest of known files + // because we assume that they will be removed when a new database is created. + // XXX Ensure the -wel file is removed if the caches.sqlite doesn't exist. + QM_TRY(OkIf(!!cachesSQLiteFile), UsageInfo{}); + + QM_TRY_INSPECT( + const auto& paddingSize, + ([dir, cachesSQLiteFile, + &aOriginMetadata]() -> Result<int64_t, nsresult> { + if (!DirectoryPaddingFileExists(*dir, DirPaddingFile::TMP_FILE)) { + QM_WARNONLY_TRY_UNWRAP(const auto maybePaddingSize, + DirectoryPaddingGet(*dir)); + if (maybePaddingSize) { + return maybePaddingSize.ref(); + } + } + + // If the temporary file still exists or failing to get the padding size + // from the padding file, then we need to get the padding size from the + // database and restore the padding file. + QM_TRY_RETURN( + GetPaddingSizeFromDB(*dir, *cachesSQLiteFile, aOriginMetadata)); + }())); + + QM_TRY_INSPECT( + const auto& innerUsageInfo, + ReduceUsageInfo( + *dir, aCanceled, + [&aCanceled]( + const nsCOMPtr<nsIFile>& file) -> Result<UsageInfo, nsresult> { + QM_TRY_INSPECT(const auto& leafName, + MOZ_TO_RESULT_INVOKE_MEMBER_TYPED(nsAutoString, file, + GetLeafName)); + + QM_TRY_INSPECT(const auto& dirEntryKind, GetDirEntryKind(*file)); + + switch (dirEntryKind) { + case nsIFileKind::ExistsAsDirectory: + if (leafName.EqualsLiteral("morgue")) { + QM_TRY_RETURN(GetBodyUsage(*file, aCanceled)); + } else { + NS_WARNING("Unknown Cache directory found!"); + } + + break; + + case nsIFileKind::ExistsAsFile: + // Ignore transient sqlite files and marker files + if (leafName.EqualsLiteral("caches.sqlite-journal") || + leafName.EqualsLiteral("caches.sqlite-shm") || + StringBeginsWith(leafName, u"caches.sqlite-mj"_ns) || + leafName.EqualsLiteral("context_open.marker")) { + break; + } + + if (leafName.Equals(kCachesSQLiteFilename) || + leafName.EqualsLiteral("caches.sqlite-wal")) { + QM_TRY_INSPECT( + const int64_t& fileSize, + MOZ_TO_RESULT_INVOKE_MEMBER(file, GetFileSize)); + MOZ_DIAGNOSTIC_ASSERT(fileSize >= 0); + + return UsageInfo{DatabaseUsageType(Some(fileSize))}; + } + + // Ignore directory padding file + if (leafName.EqualsLiteral(PADDING_FILE_NAME) || + leafName.EqualsLiteral(PADDING_TMP_FILE_NAME)) { + break; + } + + NS_WARNING("Unknown Cache file found!"); + + break; + + case nsIFileKind::DoesNotExist: + // Ignore files that got removed externally while iterating. + break; + } + + return UsageInfo{}; + })); + + // FIXME: Separate file usage and database usage in OriginInfo so that the + // workaround for treating padding file size as database usage can be removed. + return UsageInfo{DatabaseUsageType(Some(paddingSize))} + innerUsageInfo; +} + +nsresult CacheQuotaClient::InitOriginWithoutTracking( + PersistenceType aPersistenceType, const OriginMetadata& aOriginMetadata, + const AtomicBool& aCanceled) { + AssertIsOnIOThread(); + + // This is called when a storage/permanent/${origin}/cache directory exists. + // Even though this shouldn't happen with a "good" profile, we shouldn't + // return an error here, since that would cause origin initialization to fail. + // We just warn and otherwise ignore that. + UNKNOWN_FILE_WARNING(NS_LITERAL_STRING_FROM_CSTRING(DOMCACHE_DIRECTORY_NAME)); + return NS_OK; +} + +Result<UsageInfo, nsresult> CacheQuotaClient::GetUsageForOrigin( + PersistenceType aPersistenceType, const OriginMetadata& aOriginMetadata, + const AtomicBool& aCanceled) { + AssertIsOnIOThread(); + + // We can't open the database at this point, since it can be already used by + // the Cache IO thread. Use the cached value instead. + + QuotaManager* quotaManager = QuotaManager::Get(); + MOZ_ASSERT(quotaManager); + + return quotaManager->GetUsageForClient(PERSISTENCE_TYPE_DEFAULT, + aOriginMetadata, Client::DOMCACHE); +} + +void CacheQuotaClient::OnOriginClearCompleted(PersistenceType aPersistenceType, + const nsACString& aOrigin) { + // Nothing to do here. +} + +void CacheQuotaClient::OnRepositoryClearCompleted( + PersistenceType aPersistenceType) { + AssertIsOnIOThread(); + + // Nothing to do here. +} + +void CacheQuotaClient::ReleaseIOThreadObjects() { + // Nothing to do here as the Context handles cleaning everything up + // automatically. +} + +void CacheQuotaClient::AbortOperationsForLocks( + const DirectoryLockIdTable& aDirectoryLockIds) { + AssertIsOnBackgroundThread(); + + Manager::Abort(aDirectoryLockIds); +} + +void CacheQuotaClient::AbortOperationsForProcess( + ContentParentId aContentParentId) { + // The Cache and Context can be shared by multiple client processes. They + // are not exclusively owned by a single process. + // + // As far as I can tell this is used by QuotaManager to abort operations + // when a particular process goes away. We definitely don't want this + // since we are shared. Also, the Cache actor code already properly + // handles asynchronous actor destruction when the child process dies. + // + // Therefore, do nothing here. +} + +void CacheQuotaClient::AbortAllOperations() { + AssertIsOnBackgroundThread(); + + Manager::AbortAll(); +} + +void CacheQuotaClient::StartIdleMaintenance() {} + +void CacheQuotaClient::StopIdleMaintenance() {} + +void CacheQuotaClient::InitiateShutdown() { + AssertIsOnBackgroundThread(); + + Manager::InitiateShutdown(); +} + +bool CacheQuotaClient::IsShutdownCompleted() const { + AssertIsOnBackgroundThread(); + + return Manager::IsShutdownAllComplete(); +} + +void CacheQuotaClient::ForceKillActors() { + // Currently we don't implement killing actors (are there any to kill here?). +} + +nsCString CacheQuotaClient::GetShutdownStatus() const { + AssertIsOnBackgroundThread(); + + return Manager::GetShutdownStatus(); +} + +void CacheQuotaClient::FinalizeShutdown() { + // Nothing to do here. +} + +nsresult CacheQuotaClient::UpgradeStorageFrom2_0To2_1(nsIFile* aDirectory) { + AssertIsOnIOThread(); + MOZ_DIAGNOSTIC_ASSERT(aDirectory); + + QM_TRY(MOZ_TO_RESULT(DirectoryPaddingInit(*aDirectory))); + + return NS_OK; +} + +nsresult CacheQuotaClient::RestorePaddingFileInternal( + nsIFile* aBaseDir, mozIStorageConnection* aConn) { + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aBaseDir); + MOZ_DIAGNOSTIC_ASSERT(aConn); + + QM_TRY_INSPECT(const int64_t& dummyPaddingSize, + DirectoryPaddingRestore(*aBaseDir, *aConn, + /* aMustRestore */ true)); + Unused << dummyPaddingSize; + + return NS_OK; +} + +nsresult CacheQuotaClient::WipePaddingFileInternal( + const CacheDirectoryMetadata& aDirectoryMetadata, nsIFile* aBaseDir) { + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aBaseDir); + + MOZ_ASSERT(DirectoryPaddingFileExists(*aBaseDir, DirPaddingFile::FILE)); + + QM_TRY_INSPECT( + const int64_t& paddingSize, ([&aBaseDir]() -> Result<int64_t, nsresult> { + const bool temporaryPaddingFileExist = + DirectoryPaddingFileExists(*aBaseDir, DirPaddingFile::TMP_FILE); + + Maybe<int64_t> directoryPaddingGetResult; + if (!temporaryPaddingFileExist) { + QM_TRY_UNWRAP(directoryPaddingGetResult, + ([&aBaseDir]() -> Result<Maybe<int64_t>, nsresult> { + QM_TRY_RETURN( + DirectoryPaddingGet(*aBaseDir).map(Some<int64_t>), + Maybe<int64_t>{}); + }())); + } + + if (temporaryPaddingFileExist || !directoryPaddingGetResult) { + // XXXtt: Maybe have a method in the QuotaManager to clean the usage + // under the quota client and the origin. There is nothing we can do + // to recover the file. + NS_WARNING("Cannnot read padding size from file!"); + return 0; + } + + return *directoryPaddingGetResult; + }())); + + if (paddingSize > 0) { + DecreaseUsageForDirectoryMetadata(aDirectoryMetadata, paddingSize); + } + + QM_TRY(MOZ_TO_RESULT( + DirectoryPaddingDeleteFile(*aBaseDir, DirPaddingFile::FILE))); + + // Remove temporary file if we have one. + QM_TRY(MOZ_TO_RESULT( + DirectoryPaddingDeleteFile(*aBaseDir, DirPaddingFile::TMP_FILE))); + + QM_TRY(MOZ_TO_RESULT(DirectoryPaddingInit(*aBaseDir))); + + return NS_OK; +} + +CacheQuotaClient::~CacheQuotaClient() { + AssertIsOnBackgroundThread(); + MOZ_DIAGNOSTIC_ASSERT(sInstance == this); + + sInstance = nullptr; +} + +// static +CacheQuotaClient* CacheQuotaClient::sInstance = nullptr; + +// static +already_AddRefed<quota::Client> CreateQuotaClient() { + AssertIsOnBackgroundThread(); + + RefPtr<CacheQuotaClient> ref = new CacheQuotaClient(); + return ref.forget(); +} + +// static +nsresult RestorePaddingFile(nsIFile* aBaseDir, mozIStorageConnection* aConn) { + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aBaseDir); + MOZ_DIAGNOSTIC_ASSERT(aConn); + + RefPtr<CacheQuotaClient> cacheQuotaClient = CacheQuotaClient::Get(); + MOZ_DIAGNOSTIC_ASSERT(cacheQuotaClient); + + QM_TRY(MOZ_TO_RESULT( + cacheQuotaClient->RestorePaddingFileInternal(aBaseDir, aConn))); + + return NS_OK; +} + +// static +nsresult WipePaddingFile(const CacheDirectoryMetadata& aDirectoryMetadata, + nsIFile* aBaseDir) { + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aBaseDir); + + RefPtr<CacheQuotaClient> cacheQuotaClient = CacheQuotaClient::Get(); + MOZ_DIAGNOSTIC_ASSERT(cacheQuotaClient); + + QM_TRY(MOZ_TO_RESULT( + cacheQuotaClient->WipePaddingFileInternal(aDirectoryMetadata, aBaseDir))); + + return NS_OK; +} + +} // namespace mozilla::dom::cache diff --git a/dom/cache/QuotaClient.h b/dom/cache/QuotaClient.h new file mode 100644 index 0000000000..1bc04c1112 --- /dev/null +++ b/dom/cache/QuotaClient.h @@ -0,0 +1,47 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_QuotaClient_h +#define mozilla_dom_cache_QuotaClient_h + +#include "mozilla/Attributes.h" +#include "mozilla/dom/cache/Types.h" +#include "mozilla/dom/quota/Client.h" +#include "mozIStorageConnection.h" + +namespace mozilla::dom::cache { + +already_AddRefed<quota::Client> CreateQuotaClient(); + +/** + * The following functions are used to access the directory padding file. The + * directory padding file lives in DOM Cache base directory + * (e.g. foo.com/cache/.padding). It is used to keep the current overall padding + * size for an origin, so that the QuotaManager doesn't need to access the + * database when getting quota clients' usage. + * + * For the directory padding file, it's only accessed on Quota IO thread + * (for getting current usage) and Cache IO threads (for tracking padding size + * change). Besides, the padding file is protected by a mutex lock held by + * CacheQuotaClient. + * + * Each padding file should only take 8 bytes (int64_t) to record the overall + * padding size. Besides, we use the temporary padding file to indicate if the + * previous action is completed successfully. If the temporary file exists, it + * represents that the previous action is failed and the content of padding file + * cannot be trusted, and we need to restore the padding file from the database. + */ + +nsresult RestorePaddingFile(nsIFile* aBaseDir, mozIStorageConnection* aConn); + +nsresult WipePaddingFile(const CacheDirectoryMetadata& aDirectoryMetadata, + nsIFile* aBaseDir); + +extern const nsLiteralString kCachesSQLiteFilename; + +} // namespace mozilla::dom::cache + +#endif // mozilla_dom_cache_QuotaClient_h diff --git a/dom/cache/QuotaClientImpl.h b/dom/cache/QuotaClientImpl.h new file mode 100644 index 0000000000..62001ce8d8 --- /dev/null +++ b/dom/cache/QuotaClientImpl.h @@ -0,0 +1,136 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_QuotaClientImpl_h +#define mozilla_dom_cache_QuotaClientImpl_h + +#include "mozilla/dom/QMResult.h" +#include "mozilla/dom/cache/QuotaClient.h" +#include "mozilla/dom/cache/FileUtils.h" +#include "mozilla/dom/quota/ResultExtensions.h" + +namespace mozilla::dom::cache { + +class CacheQuotaClient final : public quota::Client { + static CacheQuotaClient* sInstance; + + public: + using OriginMetadata = quota::OriginMetadata; + using PersistenceType = quota::PersistenceType; + using UsageInfo = quota::UsageInfo; + + CacheQuotaClient(); + + static CacheQuotaClient* Get(); + + virtual Type GetType() override; + + virtual Result<UsageInfo, nsresult> InitOrigin( + PersistenceType aPersistenceType, const OriginMetadata& aOriginMetadata, + const AtomicBool& aCanceled) override; + + virtual nsresult InitOriginWithoutTracking( + PersistenceType aPersistenceType, const OriginMetadata& aOriginMetadata, + const AtomicBool& aCanceled) override; + + virtual Result<UsageInfo, nsresult> GetUsageForOrigin( + PersistenceType aPersistenceType, const OriginMetadata& aOriginMetadata, + const AtomicBool& aCanceled) override; + + virtual void OnOriginClearCompleted(PersistenceType aPersistenceType, + const nsACString& aOrigin) override; + + void OnRepositoryClearCompleted(PersistenceType aPersistenceType) override; + + virtual void ReleaseIOThreadObjects() override; + + void AbortOperationsForLocks( + const DirectoryLockIdTable& aDirectoryLockIds) override; + + virtual void AbortOperationsForProcess( + ContentParentId aContentParentId) override; + + virtual void AbortAllOperations() override; + + virtual void StartIdleMaintenance() override; + + virtual void StopIdleMaintenance() override; + + nsresult UpgradeStorageFrom2_0To2_1(nsIFile* aDirectory) override; + + template <typename Callable> + nsresult MaybeUpdatePaddingFileInternal(nsIFile& aBaseDir, + mozIStorageConnection& aConn, + const int64_t aIncreaseSize, + const int64_t aDecreaseSize, + Callable&& aCommitHook) { + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aIncreaseSize >= 0); + MOZ_DIAGNOSTIC_ASSERT(aDecreaseSize >= 0); + + // Temporary should be removed at the end of each action. If not, it means + // the failure happened. + const bool temporaryPaddingFileExist = + DirectoryPaddingFileExists(aBaseDir, DirPaddingFile::TMP_FILE); + + if (aIncreaseSize == aDecreaseSize && !temporaryPaddingFileExist) { + // Early return here, since most cache actions won't modify padding size. + QM_TRY(MOZ_TO_RESULT(aCommitHook())); + + return NS_OK; + } + + // Don't delete the temporary padding file in case of an error to force the + // next action recalculate the padding size. + QM_TRY(MOZ_TO_RESULT( + UpdateDirectoryPaddingFile(aBaseDir, aConn, aIncreaseSize, + aDecreaseSize, temporaryPaddingFileExist))); + + // Don't delete the temporary padding file in case of an error to force the + // next action recalculate the padding size. + QM_TRY(MOZ_TO_RESULT(aCommitHook())); + + QM_WARNONLY_TRY(MOZ_TO_RESULT(DirectoryPaddingFinalizeWrite(aBaseDir)), + ([&aBaseDir](const nsresult) { + // Force restore file next time. + QM_WARNONLY_TRY(QM_TO_RESULT(DirectoryPaddingDeleteFile( + aBaseDir, DirPaddingFile::FILE))); + + // Ensure that we are able to force the padding file to + // be restored. + MOZ_ASSERT(DirectoryPaddingFileExists( + aBaseDir, DirPaddingFile::TMP_FILE)); + + // Since both the body file and header have been stored + // in the file-system, just make the action be resolve + // and let the padding file be restored in the next + // action. + })); + + return NS_OK; + } + + nsresult RestorePaddingFileInternal(nsIFile* aBaseDir, + mozIStorageConnection* aConn); + + nsresult WipePaddingFileInternal( + const CacheDirectoryMetadata& aDirectoryMetadata, nsIFile* aBaseDir); + + private: + ~CacheQuotaClient(); + + void InitiateShutdown() override; + bool IsShutdownCompleted() const override; + nsCString GetShutdownStatus() const override; + void ForceKillActors() override; + void FinalizeShutdown() override; + + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(CacheQuotaClient, override) +}; + +} // namespace mozilla::dom::cache + +#endif // mozilla_dom_cache_QuotaClientImpl_h diff --git a/dom/cache/ReadStream.cpp b/dom/cache/ReadStream.cpp new file mode 100644 index 0000000000..366dd5ab13 --- /dev/null +++ b/dom/cache/ReadStream.cpp @@ -0,0 +1,620 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/ReadStream.h" + +#include "mozilla/Unused.h" +#include "mozilla/dom/cache/CacheStreamControlChild.h" +#include "mozilla/dom/cache/CacheStreamControlParent.h" +#include "mozilla/dom/cache/CacheTypes.h" +#include "mozilla/ipc/IPCStreamUtils.h" +#include "mozilla/SnappyUncompressInputStream.h" +#include "nsIAsyncInputStream.h" +#include "nsIThread.h" +#include "nsStringStream.h" +#include "nsTArray.h" + +namespace mozilla::dom::cache { + +using mozilla::Unused; + +// ---------------------------------------------------------------------------- + +// The inner stream class. This is where all of the real work is done. As +// an invariant Inner::Close() must be called before ~Inner(). This is +// guaranteed by our outer ReadStream class. +class ReadStream::Inner final : public ReadStream::Controllable { + public: + Inner(StreamControl* aControl, const nsID& aId, nsIInputStream* aStream); + + void Serialize(Maybe<CacheReadStream>* aReadStreamOut, ErrorResult& aRv); + + void Serialize(CacheReadStream* aReadStreamOut, ErrorResult& aRv); + + // ReadStream::Controllable methods + virtual void CloseStream() override; + + virtual void CloseStreamWithoutReporting() override; + + virtual bool HasEverBeenRead() const override; + + // Simulate nsIInputStream methods, but we don't actually inherit from it + nsresult Close(); + + nsresult Available(uint64_t* aNumAvailableOut); + + nsresult StreamStatus(); + + nsresult Read(char* aBuf, uint32_t aCount, uint32_t* aNumReadOut); + + nsresult ReadSegments(nsWriteSegmentFun aWriter, void* aClosure, + uint32_t aCount, uint32_t* aNumReadOut); + + nsresult IsNonBlocking(bool* aNonBlockingOut); + + NS_DECL_OWNINGTHREAD; + + ~Inner(); + + private: + class NoteClosedRunnable; + class ForgetRunnable; + + void NoteClosed(); + + void Forget(); + + void NoteClosedOnOwningThread(); + + void ForgetOnOwningThread(); + + nsIInputStream* EnsureStream(); + + void AsyncOpenStreamOnOwningThread(); + + void MaybeAbortAsyncOpenStream(); + + void OpenStreamFailed(); + + inline SafeRefPtr<Inner> SafeRefPtrFromThis() { + return Controllable::SafeRefPtrFromThis().downcast<Inner>(); + } + + // Weak ref to the stream control actor. The actor will always call either + // CloseStream() or CloseStreamWithoutReporting() before it's destroyed. The + // weak ref is cleared in the resulting NoteClosedOnOwningThread() or + // ForgetOnOwningThread() method call. + StreamControl* mControl; + + const nsID mId; + nsCOMPtr<nsISerialEventTarget> mOwningEventTarget; + + enum State { Open, Closed, NumStates }; + Atomic<State> mState; + Atomic<bool> mHasEverBeenRead; + bool mAsyncOpenStarted; + + // The wrapped stream objects may not be threadsafe. We need to be able + // to close a stream on our owning thread while an IO thread is simultaneously + // reading the same stream. Therefore, protect all access to these stream + // objects with a mutex. + Mutex mMutex MOZ_UNANNOTATED; + CondVar mCondVar; + nsCOMPtr<nsIInputStream> mStream; + nsCOMPtr<nsIInputStream> mSnappyStream; +}; + +// ---------------------------------------------------------------------------- + +// Runnable to notify actors that the ReadStream has closed. This must +// be done on the thread associated with the PBackground actor. Must be +// cancelable to execute on Worker threads (which can occur when the +// ReadStream is constructed on a child process Worker thread). +class ReadStream::Inner::NoteClosedRunnable final : public CancelableRunnable { + public: + explicit NoteClosedRunnable(SafeRefPtr<ReadStream::Inner> aStream) + : CancelableRunnable("dom::cache::ReadStream::Inner::NoteClosedRunnable"), + mStream(std::move(aStream)) {} + + NS_IMETHOD Run() override { + mStream->NoteClosedOnOwningThread(); + return NS_OK; + } + + // Note, we must proceed with the Run() method since our actor will not + // clean itself up until we note that the stream is closed. + nsresult Cancel() override { + Run(); + return NS_OK; + } + + private: + ~NoteClosedRunnable() = default; + + const SafeRefPtr<ReadStream::Inner> mStream; +}; + +// ---------------------------------------------------------------------------- + +// Runnable to clear actors without reporting that the ReadStream has +// closed. Since this can trigger actor destruction, we need to do +// it on the thread associated with the PBackground actor. Must be +// cancelable to execute on Worker threads (which can occur when the +// ReadStream is constructed on a child process Worker thread). +class ReadStream::Inner::ForgetRunnable final : public CancelableRunnable { + public: + explicit ForgetRunnable(SafeRefPtr<ReadStream::Inner> aStream) + : CancelableRunnable("dom::cache::ReadStream::Inner::ForgetRunnable"), + mStream(std::move(aStream)) {} + + NS_IMETHOD Run() override { + mStream->ForgetOnOwningThread(); + return NS_OK; + } + + // Note, we must proceed with the Run() method so that we properly + // call RemoveListener on the actor. + nsresult Cancel() override { + Run(); + return NS_OK; + } + + private: + ~ForgetRunnable() = default; + + const SafeRefPtr<ReadStream::Inner> mStream; +}; + +// ---------------------------------------------------------------------------- + +ReadStream::Inner::Inner(StreamControl* aControl, const nsID& aId, + nsIInputStream* aStream) + : mControl(aControl), + mId(aId), + mOwningEventTarget(GetCurrentSerialEventTarget()), + mState(Open), + mHasEverBeenRead(false), + mAsyncOpenStarted(false), + mMutex("dom::cache::ReadStream"), + mCondVar(mMutex, "dom::cache::ReadStream"), + mStream(aStream), + mSnappyStream(aStream ? new SnappyUncompressInputStream(aStream) + : nullptr) { + MOZ_DIAGNOSTIC_ASSERT(mControl); + mControl->AddReadStream(SafeRefPtrFromThis()); +} + +void ReadStream::Inner::Serialize(Maybe<CacheReadStream>* aReadStreamOut, + ErrorResult& aRv) { + MOZ_ASSERT(mOwningEventTarget->IsOnCurrentThread()); + MOZ_DIAGNOSTIC_ASSERT(aReadStreamOut); + aReadStreamOut->emplace(CacheReadStream()); + Serialize(&aReadStreamOut->ref(), aRv); +} + +void ReadStream::Inner::Serialize(CacheReadStream* aReadStreamOut, + ErrorResult& aRv) { + MOZ_ASSERT(mOwningEventTarget->IsOnCurrentThread()); + MOZ_DIAGNOSTIC_ASSERT(aReadStreamOut); + + if (mState != Open) { + aRv.ThrowTypeError( + "Response body is a cache file stream that has already been closed."); + return; + } + + MOZ_DIAGNOSTIC_ASSERT(mControl); + + aReadStreamOut->id() = mId; + mControl->SerializeControl(aReadStreamOut); + + { + MutexAutoLock lock(mMutex); + mControl->SerializeStream(aReadStreamOut, mStream); + } + + MOZ_DIAGNOSTIC_ASSERT(aReadStreamOut->stream().isNothing() || + aReadStreamOut->stream().ref().stream().type() != + mozilla::ipc::InputStreamParams::T__None); + + // We're passing ownership across the IPC barrier with the control, so + // do not signal that the stream is closed here. + Forget(); +} + +void ReadStream::Inner::CloseStream() { + MOZ_ASSERT(mOwningEventTarget->IsOnCurrentThread()); + MOZ_ALWAYS_SUCCEEDS(Close()); +} + +void ReadStream::Inner::CloseStreamWithoutReporting() { + MOZ_ASSERT(mOwningEventTarget->IsOnCurrentThread()); + Forget(); +} + +bool ReadStream::Inner::HasEverBeenRead() const { + MOZ_ASSERT(mOwningEventTarget->IsOnCurrentThread()); + return mHasEverBeenRead; +} + +nsresult ReadStream::Inner::Close() { + // stream ops can happen on any thread + nsresult rv = NS_OK; + { + MutexAutoLock lock(mMutex); + if (mSnappyStream) { + rv = mSnappyStream->Close(); + } + } + NoteClosed(); + return rv; +} + +nsresult ReadStream::Inner::Available(uint64_t* aNumAvailableOut) { + // stream ops can happen on any thread + nsresult rv = NS_OK; + { + MutexAutoLock lock(mMutex); + rv = EnsureStream()->Available(aNumAvailableOut); + } + + if (NS_FAILED(rv)) { + Close(); + } + + return rv; +} + +nsresult ReadStream::Inner::StreamStatus() { + // stream ops can happen on any thread + nsresult rv = NS_OK; + { + MutexAutoLock lock(mMutex); + rv = EnsureStream()->StreamStatus(); + } + + if (NS_FAILED(rv)) { + Close(); + } + + return rv; +} + +nsresult ReadStream::Inner::Read(char* aBuf, uint32_t aCount, + uint32_t* aNumReadOut) { + // stream ops can happen on any thread + MOZ_DIAGNOSTIC_ASSERT(aNumReadOut); + + nsresult rv = NS_OK; + { + MutexAutoLock lock(mMutex); + rv = EnsureStream()->Read(aBuf, aCount, aNumReadOut); + } + + if ((NS_FAILED(rv) && rv != NS_BASE_STREAM_WOULD_BLOCK) || + *aNumReadOut == 0) { + Close(); + } + + mHasEverBeenRead = true; + + return rv; +} + +nsresult ReadStream::Inner::ReadSegments(nsWriteSegmentFun aWriter, + void* aClosure, uint32_t aCount, + uint32_t* aNumReadOut) { + // stream ops can happen on any thread + MOZ_DIAGNOSTIC_ASSERT(aNumReadOut); + + if (aCount) { + mHasEverBeenRead = true; + } + + nsresult rv = NS_OK; + { + MutexAutoLock lock(mMutex); + rv = EnsureStream()->ReadSegments(aWriter, aClosure, aCount, aNumReadOut); + } + + if ((NS_FAILED(rv) && rv != NS_BASE_STREAM_WOULD_BLOCK && + rv != NS_ERROR_NOT_IMPLEMENTED) || + *aNumReadOut == 0) { + Close(); + } + + // Verify bytes were actually read before marking as being ever read. For + // example, code can test if the stream supports ReadSegments() by calling + // this method with a dummy callback which doesn't read anything. We don't + // want to trigger on that. + if (*aNumReadOut) { + mHasEverBeenRead = true; + } + + return rv; +} + +nsresult ReadStream::Inner::IsNonBlocking(bool* aNonBlockingOut) { + // stream ops can happen on any thread + MutexAutoLock lock(mMutex); + if (mSnappyStream) { + return mSnappyStream->IsNonBlocking(aNonBlockingOut); + } + *aNonBlockingOut = false; + return NS_OK; +} + +ReadStream::Inner::~Inner() { + // Any thread + MOZ_DIAGNOSTIC_ASSERT(mState == Closed); + MOZ_DIAGNOSTIC_ASSERT(!mControl); +} + +void ReadStream::Inner::NoteClosed() { + // Any thread + if (mState == Closed) { + return; + } + + if (mOwningEventTarget->IsOnCurrentThread()) { + NoteClosedOnOwningThread(); + return; + } + + nsCOMPtr<nsIRunnable> runnable = new NoteClosedRunnable(SafeRefPtrFromThis()); + MOZ_ALWAYS_SUCCEEDS(mOwningEventTarget->Dispatch(runnable.forget(), + nsIThread::DISPATCH_NORMAL)); +} + +void ReadStream::Inner::Forget() { + // Any thread + if (mState == Closed) { + return; + } + + if (mOwningEventTarget->IsOnCurrentThread()) { + ForgetOnOwningThread(); + return; + } + + nsCOMPtr<nsIRunnable> runnable = new ForgetRunnable(SafeRefPtrFromThis()); + MOZ_ALWAYS_SUCCEEDS(mOwningEventTarget->Dispatch(runnable.forget(), + nsIThread::DISPATCH_NORMAL)); +} + +void ReadStream::Inner::NoteClosedOnOwningThread() { + MOZ_ASSERT(mOwningEventTarget->IsOnCurrentThread()); + + // Mark closed and do nothing if we were already closed + if (!mState.compareExchange(Open, Closed)) { + return; + } + + MaybeAbortAsyncOpenStream(); + + MOZ_DIAGNOSTIC_ASSERT(mControl); + mControl->NoteClosed(SafeRefPtrFromThis(), mId); + mControl = nullptr; +} + +void ReadStream::Inner::ForgetOnOwningThread() { + MOZ_ASSERT(mOwningEventTarget->IsOnCurrentThread()); + + // Mark closed and do nothing if we were already closed + if (!mState.compareExchange(Open, Closed)) { + return; + } + + MaybeAbortAsyncOpenStream(); + + MOZ_DIAGNOSTIC_ASSERT(mControl); + mControl->ForgetReadStream(SafeRefPtrFromThis()); + mControl = nullptr; +} + +nsIInputStream* ReadStream::Inner::EnsureStream() { + mMutex.AssertCurrentThreadOwns(); + + // We need to block the current thread while we open the stream. We + // cannot do this safely from the main owning thread since it would + // trigger deadlock. This should be ok, though, since a blocking + // stream like this should never be read on the owning thread anyway. + if (mOwningEventTarget->IsOnCurrentThread()) { + MOZ_CRASH("Blocking read on the js/ipc owning thread!"); + } + + if (mSnappyStream) { + return mSnappyStream; + } + + nsCOMPtr<nsIRunnable> r = NewCancelableRunnableMethod( + "ReadStream::Inner::AsyncOpenStreamOnOwningThread", this, + &ReadStream::Inner::AsyncOpenStreamOnOwningThread); + nsresult rv = + mOwningEventTarget->Dispatch(r.forget(), nsIThread::DISPATCH_NORMAL); + if (NS_WARN_IF(NS_FAILED(rv))) { + OpenStreamFailed(); + return mSnappyStream; + } + + mCondVar.Wait(); + MOZ_DIAGNOSTIC_ASSERT(mSnappyStream); + + return mSnappyStream; +} + +void ReadStream::Inner::AsyncOpenStreamOnOwningThread() { + MOZ_ASSERT(mOwningEventTarget->IsOnCurrentThread()); + + if (mSnappyStream) { + // Different threads might request opening the stream at the same time. If + // the earlier request succeeded, then use the result. + mCondVar.NotifyAll(); + return; + } + + if (!mControl || mState == Closed) { + MutexAutoLock lock(mMutex); + OpenStreamFailed(); + mCondVar.NotifyAll(); + return; + } + + if (mAsyncOpenStarted) { + return; + } + mAsyncOpenStarted = true; + + RefPtr<ReadStream::Inner> self = this; + mControl->OpenStream(mId, [self](nsCOMPtr<nsIInputStream>&& aStream) { + MutexAutoLock lock(self->mMutex); + self->mAsyncOpenStarted = false; + if (!self->mStream) { + if (!aStream) { + self->OpenStreamFailed(); + } else { + self->mStream = std::move(aStream); + self->mSnappyStream = new SnappyUncompressInputStream(self->mStream); + } + } + self->mCondVar.NotifyAll(); + }); +} + +void ReadStream::Inner::MaybeAbortAsyncOpenStream() { + if (!mAsyncOpenStarted) { + return; + } + + MutexAutoLock lock(mMutex); + OpenStreamFailed(); + mCondVar.NotifyAll(); +} + +void ReadStream::Inner::OpenStreamFailed() { + MOZ_DIAGNOSTIC_ASSERT(!mStream); + MOZ_DIAGNOSTIC_ASSERT(!mSnappyStream); + mMutex.AssertCurrentThreadOwns(); + Unused << NS_NewCStringInputStream(getter_AddRefs(mStream), ""_ns); + mSnappyStream = mStream; + mStream->Close(); + NoteClosed(); +} + +// ---------------------------------------------------------------------------- + +NS_IMPL_ISUPPORTS(cache::ReadStream, nsIInputStream, ReadStream); + +// static +already_AddRefed<ReadStream> ReadStream::Create( + const Maybe<CacheReadStream>& aMaybeReadStream) { + if (aMaybeReadStream.isNothing()) { + return nullptr; + } + + return Create(aMaybeReadStream.ref()); +} + +// static +already_AddRefed<ReadStream> ReadStream::Create( + const CacheReadStream& aReadStream) { + // The parameter may or may not be for a Cache created stream. The way we + // tell is by looking at the stream control actor. If the actor exists, + // then we know the Cache created it. + if (!aReadStream.control()) { + return nullptr; + } + + MOZ_DIAGNOSTIC_ASSERT(aReadStream.stream().isNothing() || + aReadStream.stream().ref().stream().type() != + mozilla::ipc::InputStreamParams::T__None); + + // Control is guaranteed to survive this method as ActorDestroy() cannot + // run on this thread until we complete. + StreamControl* control; + if (aReadStream.control().IsChild()) { + auto actor = + static_cast<CacheStreamControlChild*>(aReadStream.control().AsChild()); + control = actor; + } else { + auto actor = static_cast<CacheStreamControlParent*>( + aReadStream.control().AsParent()); + control = actor; + } + MOZ_DIAGNOSTIC_ASSERT(control); + + nsCOMPtr<nsIInputStream> stream = DeserializeIPCStream(aReadStream.stream()); + + // Currently we expect all cache read streams to be blocking file streams. +#if defined(MOZ_DIAGNOSTIC_ASSERT_ENABLED) + if (stream) { + nsCOMPtr<nsIAsyncInputStream> asyncStream = do_QueryInterface(stream); + MOZ_DIAGNOSTIC_ASSERT(!asyncStream); + } +#endif + + return MakeAndAddRef<ReadStream>(MakeSafeRefPtr<ReadStream::Inner>( + std::move(control), aReadStream.id(), stream)); +} + +// static +already_AddRefed<ReadStream> ReadStream::Create( + PCacheStreamControlParent* aControl, const nsID& aId, + nsIInputStream* aStream) { + MOZ_DIAGNOSTIC_ASSERT(aControl); + + return MakeAndAddRef<ReadStream>(MakeSafeRefPtr<ReadStream::Inner>( + static_cast<CacheStreamControlParent*>(aControl), aId, aStream)); +} + +void ReadStream::Serialize(Maybe<CacheReadStream>* aReadStreamOut, + ErrorResult& aRv) { + mInner->Serialize(aReadStreamOut, aRv); +} + +void ReadStream::Serialize(CacheReadStream* aReadStreamOut, ErrorResult& aRv) { + mInner->Serialize(aReadStreamOut, aRv); +} + +ReadStream::ReadStream(SafeRefPtr<ReadStream::Inner> aInner) + : mInner(std::move(aInner)) { + MOZ_DIAGNOSTIC_ASSERT(mInner); +} + +ReadStream::~ReadStream() { + // Explicitly close the inner stream so that it does not have to + // deal with implicitly closing at destruction time. + mInner->Close(); +} + +NS_IMETHODIMP +ReadStream::Close() { return mInner->Close(); } + +NS_IMETHODIMP +ReadStream::Available(uint64_t* aNumAvailableOut) { + return mInner->Available(aNumAvailableOut); +} + +NS_IMETHODIMP +ReadStream::StreamStatus() { return mInner->StreamStatus(); } + +NS_IMETHODIMP +ReadStream::Read(char* aBuf, uint32_t aCount, uint32_t* aNumReadOut) { + return mInner->Read(aBuf, aCount, aNumReadOut); +} + +NS_IMETHODIMP +ReadStream::ReadSegments(nsWriteSegmentFun aWriter, void* aClosure, + uint32_t aCount, uint32_t* aNumReadOut) { + return mInner->ReadSegments(aWriter, aClosure, aCount, aNumReadOut); +} + +NS_IMETHODIMP +ReadStream::IsNonBlocking(bool* aNonBlockingOut) { + return mInner->IsNonBlocking(aNonBlockingOut); +} + +} // namespace mozilla::dom::cache diff --git a/dom/cache/ReadStream.h b/dom/cache/ReadStream.h new file mode 100644 index 0000000000..d6b4e1d979 --- /dev/null +++ b/dom/cache/ReadStream.h @@ -0,0 +1,105 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_ReadStream_h +#define mozilla_dom_cache_ReadStream_h + +#include "mozilla/dom/SafeRefPtr.h" +#include "mozilla/ipc/FileDescriptor.h" +#include "mozilla/RefPtr.h" +#include "nsCOMPtr.h" +#include "nsID.h" +#include "nsIInputStream.h" +#include "nsISupportsImpl.h" +#include "nsTArrayForwardDeclare.h" + +namespace mozilla { +class ErrorResult; + +namespace dom::cache { + +class CacheReadStream; +class PCacheStreamControlParent; + +// IID for the dom::cache::ReadStream interface +#define NS_DOM_CACHE_READSTREAM_IID \ + { \ + 0x8e5da7c9, 0x0940, 0x4f1d, { \ + 0x97, 0x25, 0x5c, 0x59, 0x38, 0xdd, 0xb9, 0x9f \ + } \ + } + +// Custom stream class for Request and Response bodies being read from +// a Cache. The main purpose of this class is to report back to the +// Cache's Manager when the stream is closed. This allows the Cache to +// accurately determine when the underlying body file can be deleted, +// etc. +// +// The ReadStream class also provides us with a convenient QI'able +// interface that we can use to pass additional meta-data with the +// stream channel. For example, Cache.put() can detect that the content +// script is passing a Cache-originated-stream back into the Cache +// again. This enables certain optimizations. +class ReadStream final : public nsIInputStream { + public: + // Interface that lets the StreamControl classes interact with + // our private inner stream. + class Controllable : public AtomicSafeRefCounted<Controllable> { + public: + virtual ~Controllable() = default; + + // Closes the stream, notifies the stream control, and then forgets + // the stream control. + virtual void CloseStream() = 0; + + // Closes the stream and then forgets the stream control. Does not + // notify. + virtual void CloseStreamWithoutReporting() = 0; + + virtual bool HasEverBeenRead() const = 0; + + MOZ_DECLARE_REFCOUNTED_TYPENAME(ReadStream::Controllable); + }; + + static already_AddRefed<ReadStream> Create( + const Maybe<CacheReadStream>& aMaybeReadStream); + + static already_AddRefed<ReadStream> Create( + const CacheReadStream& aReadStream); + + static already_AddRefed<ReadStream> Create( + PCacheStreamControlParent* aControl, const nsID& aId, + nsIInputStream* aStream); + + void Serialize(Maybe<CacheReadStream>* aReadStreamOut, ErrorResult& aRv); + void Serialize(CacheReadStream* aReadStreamOut, ErrorResult& aRv); + + private: + class Inner; + + ~ReadStream(); + + // Hold a strong ref to an inner class that actually implements the + // majority of the stream logic. Before releasing this ref the outer + // ReadStream guarantees it will call Close() on the inner stream. + // This is essential for the inner stream to avoid dealing with the + // implicit close that can happen when a stream is destroyed. + SafeRefPtr<ReadStream::Inner> mInner; + + public: + explicit ReadStream(SafeRefPtr<ReadStream::Inner> aInner); + + NS_DECLARE_STATIC_IID_ACCESSOR(NS_DOM_CACHE_READSTREAM_IID); + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSIINPUTSTREAM +}; + +NS_DEFINE_STATIC_IID_ACCESSOR(ReadStream, NS_DOM_CACHE_READSTREAM_IID); + +} // namespace dom::cache +} // namespace mozilla + +#endif // mozilla_dom_cache_ReadStream_h diff --git a/dom/cache/SavedTypes.h b/dom/cache/SavedTypes.h new file mode 100644 index 0000000000..0a1eacfb99 --- /dev/null +++ b/dom/cache/SavedTypes.h @@ -0,0 +1,50 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_SavedTypes_h +#define mozilla_dom_cache_SavedTypes_h + +// NOTE: This cannot be rolled into Types.h because the IPC dependency. +// breaks webidl unified builds. + +#include "mozilla/dom/cache/CacheTypes.h" +#include "mozilla/dom/cache/Types.h" +#include "nsCOMPtr.h" +#include "nsID.h" + +namespace mozilla::dom::cache { + +struct SavedRequest { + SavedRequest() : mHasBodyId(false), mCacheId(0) { + mBodyId.m0 = 0; + mBodyId.m1 = 0; + mBodyId.m2 = 0; + memset(mBodyId.m3, 0, sizeof(mBodyId.m3)); + mValue.body() = Nothing(); + } + CacheRequest mValue; + bool mHasBodyId; + nsID mBodyId; + CacheId mCacheId; +}; + +struct SavedResponse { + SavedResponse() : mHasBodyId(false), mCacheId(0) { + mBodyId.m0 = 0; + mBodyId.m1 = 0; + mBodyId.m2 = 0; + memset(mBodyId.m3, 0, sizeof(mBodyId.m3)); + mValue.body() = Nothing(); + } + CacheResponse mValue; + bool mHasBodyId; + nsID mBodyId; + CacheId mCacheId; +}; + +} // namespace mozilla::dom::cache + +#endif // mozilla_dom_cache_SavedTypes_h diff --git a/dom/cache/StreamControl.cpp b/dom/cache/StreamControl.cpp new file mode 100644 index 0000000000..5d6c9a9e0e --- /dev/null +++ b/dom/cache/StreamControl.cpp @@ -0,0 +1,68 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/StreamControl.h" + +namespace mozilla::dom::cache { + +void StreamControl::AddReadStream( + SafeRefPtr<ReadStream::Controllable> aReadStream) { + AssertOwningThread(); + MOZ_DIAGNOSTIC_ASSERT(aReadStream); + MOZ_ASSERT(!mReadStreamList.Contains(aReadStream)); + mReadStreamList.AppendElement(std::move(aReadStream)); +} + +void StreamControl::ForgetReadStream( + SafeRefPtr<ReadStream::Controllable> aReadStream) { + AssertOwningThread(); + MOZ_ALWAYS_TRUE(mReadStreamList.RemoveElement(aReadStream)); +} + +void StreamControl::NoteClosed(SafeRefPtr<ReadStream::Controllable> aReadStream, + const nsID& aId) { + AssertOwningThread(); + ForgetReadStream(std::move(aReadStream)); + NoteClosedAfterForget(aId); +} + +StreamControl::~StreamControl() { + // owning thread only, but can't call virtual AssertOwningThread in destructor + MOZ_DIAGNOSTIC_ASSERT(mReadStreamList.IsEmpty()); +} + +void StreamControl::CloseAllReadStreams() { + AssertOwningThread(); + + // A copy of mReadStreamList is necessary here for two reasons: + // 1. mReadStreamList is modified in StreamControl::ForgetReadStream (called + // transitively) + // 2. the this pointer is deleted by CacheStreamControlParent::Shutdown + // (called transitively) + auto readStreamList = mReadStreamList.Clone(); + for (const auto& stream : readStreamList.ForwardRange()) { + stream->CloseStream(); + } +} + +void StreamControl::CloseAllReadStreamsWithoutReporting() { + AssertOwningThread(); + + for (const auto& stream : mReadStreamList.ForwardRange()) { + // Note, we cannot trigger IPC traffic here. So use + // CloseStreamWithoutReporting(). + stream->CloseStreamWithoutReporting(); + } +} + +bool StreamControl::HasEverBeenRead() const { + const auto range = mReadStreamList.NonObservingRange(); + return std::any_of(range.begin(), range.end(), [](const auto& stream) { + return stream->HasEverBeenRead(); + }); +} + +} // namespace mozilla::dom::cache diff --git a/dom/cache/StreamControl.h b/dom/cache/StreamControl.h new file mode 100644 index 0000000000..4a5cdfc309 --- /dev/null +++ b/dom/cache/StreamControl.h @@ -0,0 +1,75 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_StreamControl_h +#define mozilla_dom_cache_StreamControl_h + +#include "mozilla/dom/cache/ReadStream.h" +#include "mozilla/dom/cache/Types.h" +#include "mozilla/RefPtr.h" +#include "nsTObserverArray.h" + +struct nsID; + +namespace mozilla::dom::cache { + +class CacheReadStream; + +// Abstract class to help implement the stream control Child and Parent actors. +// This provides an interface to partly help with serialization of IPC types, +// but also an implementation for tracking ReadStream objects. +class StreamControl { + public: + // abstract interface that must be implemented by child class + virtual void SerializeControl(CacheReadStream* aReadStreamOut) = 0; + + virtual void SerializeStream(CacheReadStream* aReadStreamOut, + nsIInputStream* aStream) = 0; + + virtual void OpenStream(const nsID& aId, InputStreamResolver&& aResolver) = 0; + + // inherited implementation of the ReadStream::Controllable list + + // Begin controlling the given ReadStream. This causes a strong ref to + // be held by the control. The ReadStream must call NoteClosed() or + // ForgetReadStream() to release this ref. + void AddReadStream(SafeRefPtr<ReadStream::Controllable> aReadStream); + + // Forget the ReadStream without notifying the actor. + void ForgetReadStream(SafeRefPtr<ReadStream::Controllable> aReadStream); + + // Forget the ReadStream and then notify the actor the stream is closed. + void NoteClosed(SafeRefPtr<ReadStream::Controllable> aReadStream, + const nsID& aId); + + protected: + ~StreamControl(); + + void CloseAllReadStreams(); + + void CloseAllReadStreamsWithoutReporting(); + + bool HasEverBeenRead() const; + + // protected parts of the abstract interface + virtual void NoteClosedAfterForget(const nsID& aId) = 0; + +#ifdef DEBUG + virtual void AssertOwningThread() = 0; +#else + void AssertOwningThread() {} +#endif + + private: + // Hold strong references to ReadStream object. When the stream is closed + // it should call NoteClosed() or ForgetReadStream() to release this ref. + using ReadStreamList = nsTObserverArray<SafeRefPtr<ReadStream::Controllable>>; + ReadStreamList mReadStreamList; +}; + +} // namespace mozilla::dom::cache + +#endif // mozilla_dom_cache_StreamControl_h diff --git a/dom/cache/StreamList.cpp b/dom/cache/StreamList.cpp new file mode 100644 index 0000000000..2e18279947 --- /dev/null +++ b/dom/cache/StreamList.cpp @@ -0,0 +1,161 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/StreamList.h" + +#include <algorithm> +#include "mozilla/dom/cache/CacheStreamControlParent.h" +#include "mozilla/dom/cache/Context.h" +#include "mozilla/dom/cache/Manager.h" +#include "nsIInputStream.h" + +namespace mozilla::dom::cache { + +namespace { + +auto MatchById(const nsID& aId) { + return [aId](const auto& entry) { return entry.mId == aId; }; +} + +} // namespace + +StreamList::StreamList(SafeRefPtr<Manager> aManager, + SafeRefPtr<Context> aContext) + : mManager(std::move(aManager)), + mContext(std::move(aContext)), + mCacheId(INVALID_CACHE_ID), + mStreamControl(nullptr), + mActivated(false) { + MOZ_DIAGNOSTIC_ASSERT(mManager); + mContext->AddActivity(*this); +} + +Manager& StreamList::GetManager() const { + MOZ_DIAGNOSTIC_ASSERT(mManager); + return *mManager; +} + +bool StreamList::ShouldOpenStreamFor(const nsID& aId) const { + NS_ASSERT_OWNINGTHREAD(StreamList); + + return std::any_of(mList.cbegin(), mList.cend(), MatchById(aId)); +} + +void StreamList::SetStreamControl(CacheStreamControlParent* aStreamControl) { + NS_ASSERT_OWNINGTHREAD(StreamList); + MOZ_DIAGNOSTIC_ASSERT(aStreamControl); + + // For cases where multiple streams are serialized for a single list + // then the control will get passed multiple times. This is ok, but + // it should be the same control each time. + if (mStreamControl) { + MOZ_DIAGNOSTIC_ASSERT(aStreamControl == mStreamControl); + return; + } + + mStreamControl = aStreamControl; + mStreamControl->SetStreamList(SafeRefPtrFromThis()); +} + +void StreamList::RemoveStreamControl(CacheStreamControlParent* aStreamControl) { + NS_ASSERT_OWNINGTHREAD(StreamList); + MOZ_DIAGNOSTIC_ASSERT(mStreamControl); + MOZ_DIAGNOSTIC_ASSERT(mStreamControl == aStreamControl); + mStreamControl = nullptr; +} + +void StreamList::Activate(CacheId aCacheId) { + NS_ASSERT_OWNINGTHREAD(StreamList); + MOZ_DIAGNOSTIC_ASSERT(!mActivated); + MOZ_DIAGNOSTIC_ASSERT(mCacheId == INVALID_CACHE_ID); + mActivated = true; + mCacheId = aCacheId; + mManager->AddRefCacheId(mCacheId); + mManager->AddStreamList(*this); + + for (uint32_t i = 0; i < mList.Length(); ++i) { + mManager->AddRefBodyId(mList[i].mId); + } +} + +void StreamList::Add(const nsID& aId, nsCOMPtr<nsIInputStream>&& aStream) { + // All streams should be added on IO thread before we set the stream + // control on the owning IPC thread. + MOZ_DIAGNOSTIC_ASSERT(!mStreamControl); + mList.EmplaceBack(aId, std::move(aStream)); +} + +already_AddRefed<nsIInputStream> StreamList::Extract(const nsID& aId) { + NS_ASSERT_OWNINGTHREAD(StreamList); + + const auto it = std::find_if(mList.begin(), mList.end(), MatchById(aId)); + + return it != mList.end() ? it->mStream.forget() : nullptr; +} + +void StreamList::NoteClosed(const nsID& aId) { + NS_ASSERT_OWNINGTHREAD(StreamList); + + const auto it = std::find_if(mList.begin(), mList.end(), MatchById(aId)); + if (it != mList.end()) { + mList.RemoveElementAt(it); + mManager->ReleaseBodyId(aId); + } + + if (mList.IsEmpty() && mStreamControl) { + mStreamControl->Shutdown(); + } +} + +void StreamList::NoteClosedAll() { + NS_ASSERT_OWNINGTHREAD(StreamList); + for (uint32_t i = 0; i < mList.Length(); ++i) { + mManager->ReleaseBodyId(mList[i].mId); + } + mList.Clear(); + + if (mStreamControl) { + mStreamControl->Shutdown(); + } +} + +void StreamList::CloseAll() { + NS_ASSERT_OWNINGTHREAD(StreamList); + if (mStreamControl) { + auto* streamControl = std::exchange(mStreamControl, nullptr); + + streamControl->CloseAll(); + + mStreamControl = std::exchange(streamControl, nullptr); + + mStreamControl->Shutdown(); + } +} + +void StreamList::Cancel() { + NS_ASSERT_OWNINGTHREAD(StreamList); + CloseAll(); +} + +bool StreamList::MatchesCacheId(CacheId aCacheId) const { + NS_ASSERT_OWNINGTHREAD(StreamList); + return aCacheId == mCacheId; +} + +StreamList::~StreamList() { + NS_ASSERT_OWNINGTHREAD(StreamList); + MOZ_DIAGNOSTIC_ASSERT(!mStreamControl); + if (mActivated) { + mManager->RemoveStreamList(*this); + for (uint32_t i = 0; i < mList.Length(); ++i) { + mManager->ReleaseBodyId(mList[i].mId); + } + mManager->ReleaseCacheId(mCacheId); + } + mContext->RemoveActivity(*this); +} + +} // namespace mozilla::dom::cache diff --git a/dom/cache/StreamList.h b/dom/cache/StreamList.h new file mode 100644 index 0000000000..2fe6bb9809 --- /dev/null +++ b/dom/cache/StreamList.h @@ -0,0 +1,70 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_StreamList_h +#define mozilla_dom_cache_StreamList_h + +#include "mozilla/dom/SafeRefPtr.h" +#include "mozilla/dom/cache/Context.h" +#include "mozilla/dom/cache/Types.h" +#include "nsTArray.h" + +class nsIInputStream; + +namespace mozilla::dom::cache { + +class CacheStreamControlParent; +class Manager; + +class StreamList final : public Context::Activity, + public SafeRefCounted<StreamList> { + public: + StreamList(SafeRefPtr<Manager> aManager, SafeRefPtr<Context> aContext); + + Manager& GetManager() const; + bool ShouldOpenStreamFor(const nsID& aId) const; + + void SetStreamControl(CacheStreamControlParent* aStreamControl); + void RemoveStreamControl(CacheStreamControlParent* aStreamControl); + + void Activate(CacheId aCacheId); + + void Add(const nsID& aId, nsCOMPtr<nsIInputStream>&& aStream); + already_AddRefed<nsIInputStream> Extract(const nsID& aId); + + void NoteClosed(const nsID& aId); + void NoteClosedAll(); + void CloseAll(); + + // Context::Activity methods + virtual void Cancel() override; + virtual bool MatchesCacheId(CacheId aCacheId) const override; + + private: + struct Entry { + explicit Entry(const nsID& aId, nsCOMPtr<nsIInputStream>&& aStream) + : mId(aId), mStream(std::move(aStream)) {} + + nsID mId; + nsCOMPtr<nsIInputStream> mStream; + }; + SafeRefPtr<Manager> mManager; + SafeRefPtr<Context> mContext; + CacheId mCacheId; + CacheStreamControlParent* mStreamControl; + nsTArray<Entry> mList; + bool mActivated; + + public: + ~StreamList(); + + NS_DECL_OWNINGTHREAD + MOZ_DECLARE_REFCOUNTED_TYPENAME(cache::StreamList) +}; + +} // namespace mozilla::dom::cache + +#endif // mozilla_dom_cache_StreamList_h diff --git a/dom/cache/TypeUtils.cpp b/dom/cache/TypeUtils.cpp new file mode 100644 index 0000000000..c350f4947d --- /dev/null +++ b/dom/cache/TypeUtils.cpp @@ -0,0 +1,499 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/TypeUtils.h" + +#include <algorithm> +#include "mozilla/StaticPrefs_extensions.h" +#include "mozilla/Unused.h" +#include "mozilla/dom/CacheBinding.h" +#include "mozilla/dom/CacheStorageBinding.h" +#include "mozilla/dom/FetchTypes.h" +#include "mozilla/dom/InternalRequest.h" +#include "mozilla/dom/Request.h" +#include "mozilla/dom/Response.h" +#include "mozilla/dom/RootedDictionary.h" +#include "mozilla/dom/cache/CacheTypes.h" +#include "mozilla/dom/cache/ReadStream.h" +#include "mozilla/ipc/BackgroundChild.h" +#include "mozilla/ipc/IPCStreamUtils.h" +#include "mozilla/ipc/PBackgroundChild.h" +#include "mozilla/ipc/InputStreamUtils.h" +#include "nsCharSeparatedTokenizer.h" +#include "nsCOMPtr.h" +#include "nsHttp.h" +#include "nsIIPCSerializableInputStream.h" +#include "nsPromiseFlatString.h" +#include "nsQueryObject.h" +#include "nsStreamUtils.h" +#include "nsString.h" +#include "nsURLParsers.h" + +namespace mozilla::dom::cache { + +using mozilla::ipc::BackgroundChild; +using mozilla::ipc::FileDescriptor; +using mozilla::ipc::PBackgroundChild; + +namespace { + +static bool HasVaryStar(mozilla::dom::InternalHeaders* aHeaders) { + nsCString varyHeaders; + ErrorResult rv; + aHeaders->Get("vary"_ns, varyHeaders, rv); + MOZ_ALWAYS_TRUE(!rv.Failed()); + + for (const nsACString& header : + nsCCharSeparatedTokenizer(varyHeaders, NS_HTTP_HEADER_SEP).ToRange()) { + if (header.EqualsLiteral("*")) { + return true; + } + } + return false; +} + +nsTArray<HeadersEntry> ToHeadersEntryList(InternalHeaders* aHeaders) { + MOZ_DIAGNOSTIC_ASSERT(aHeaders); + + AutoTArray<InternalHeaders::Entry, 16> entryList; + aHeaders->GetEntries(entryList); + + nsTArray<HeadersEntry> result; + result.SetCapacity(entryList.Length()); + std::transform(entryList.cbegin(), entryList.cend(), MakeBackInserter(result), + [](const auto& entry) { + return HeadersEntry(entry.mName, entry.mValue); + }); + + return result; +} + +} // namespace + +SafeRefPtr<InternalRequest> TypeUtils::ToInternalRequest( + JSContext* aCx, const RequestOrUSVString& aIn, BodyAction aBodyAction, + ErrorResult& aRv) { + if (aIn.IsRequest()) { + Request& request = aIn.GetAsRequest(); + + // Check and set bodyUsed flag immediately because its on Request + // instead of InternalRequest. + CheckAndSetBodyUsed(aCx, request, aBodyAction, aRv); + if (aRv.Failed()) { + return nullptr; + } + + return request.GetInternalRequest(); + } + + return ToInternalRequest(aIn.GetAsUSVString(), aRv); +} + +SafeRefPtr<InternalRequest> TypeUtils::ToInternalRequest( + JSContext* aCx, const OwningRequestOrUSVString& aIn, BodyAction aBodyAction, + ErrorResult& aRv) { + if (aIn.IsRequest()) { + Request& request = aIn.GetAsRequest(); + + // Check and set bodyUsed flag immediately because its on Request + // instead of InternalRequest. + CheckAndSetBodyUsed(aCx, request, aBodyAction, aRv); + if (aRv.Failed()) { + return nullptr; + } + + return request.GetInternalRequest(); + } + + return ToInternalRequest(aIn.GetAsUSVString(), aRv); +} + +void TypeUtils::ToCacheRequest(CacheRequest& aOut, const InternalRequest& aIn, + BodyAction aBodyAction, + SchemeAction aSchemeAction, ErrorResult& aRv) { + aIn.GetMethod(aOut.method()); + nsCString url(aIn.GetURLWithoutFragment()); + bool schemeValid; + ProcessURL(url, &schemeValid, &aOut.urlWithoutQuery(), &aOut.urlQuery(), aRv); + if (aRv.Failed()) { + return; + } + if (!schemeValid) { + if (aSchemeAction == TypeErrorOnInvalidScheme) { + aRv.ThrowTypeError<MSG_INVALID_URL_SCHEME>("Request", url); + return; + } + } + aOut.urlFragment() = aIn.GetFragment(); + + aIn.GetReferrer(aOut.referrer()); + aOut.referrerPolicy() = aIn.ReferrerPolicy_(); + RefPtr<InternalHeaders> headers = aIn.Headers(); + MOZ_DIAGNOSTIC_ASSERT(headers); + aOut.headers() = ToHeadersEntryList(headers); + aOut.headersGuard() = headers->Guard(); + aOut.mode() = aIn.Mode(); + aOut.credentials() = aIn.GetCredentialsMode(); + aOut.contentPolicyType() = aIn.ContentPolicyType(); + aOut.requestCache() = aIn.GetCacheMode(); + aOut.requestRedirect() = aIn.GetRedirectMode(); + + aOut.integrity() = aIn.GetIntegrity(); + aOut.loadingEmbedderPolicy() = aIn.GetEmbedderPolicy(); + const mozilla::UniquePtr<mozilla::ipc::PrincipalInfo>& principalInfo = + aIn.GetPrincipalInfo(); + if (principalInfo) { + aOut.principalInfo() = Some(*(principalInfo.get())); + } + + if (aBodyAction == IgnoreBody) { + aOut.body() = Nothing(); + return; + } + + // BodyUsed flag is checked and set previously in ToInternalRequest() + + nsCOMPtr<nsIInputStream> stream; + aIn.GetBody(getter_AddRefs(stream)); + SerializeCacheStream(stream, &aOut.body(), aRv); + if (NS_WARN_IF(aRv.Failed())) { + return; + } +} + +void TypeUtils::ToCacheResponseWithoutBody(CacheResponse& aOut, + InternalResponse& aIn, + ErrorResult& aRv) { + aOut.type() = aIn.Type(); + + aIn.GetUnfilteredURLList(aOut.urlList()); + AutoTArray<nsCString, 4> urlList; + aIn.GetURLList(urlList); + + for (uint32_t i = 0; i < aOut.urlList().Length(); i++) { + MOZ_DIAGNOSTIC_ASSERT(!aOut.urlList()[i].IsEmpty()); + // Pass all Response URL schemes through... The spec only requires we take + // action on invalid schemes for Request objects. + ProcessURL(aOut.urlList()[i], nullptr, nullptr, nullptr, aRv); + } + + aOut.status() = aIn.GetUnfilteredStatus(); + aOut.statusText() = aIn.GetUnfilteredStatusText(); + RefPtr<InternalHeaders> headers = aIn.UnfilteredHeaders(); + MOZ_DIAGNOSTIC_ASSERT(headers); + if (HasVaryStar(headers)) { + aRv.ThrowTypeError("Invalid Response object with a 'Vary: *' header."); + return; + } + aOut.headers() = ToHeadersEntryList(headers); + aOut.headersGuard() = headers->Guard(); + aOut.securityInfo() = aIn.GetChannelInfo().SecurityInfo(); + if (aIn.GetPrincipalInfo()) { + aOut.principalInfo() = Some(*aIn.GetPrincipalInfo()); + } else { + aOut.principalInfo() = Nothing(); + } + + aOut.paddingInfo() = aIn.GetPaddingInfo(); + aOut.paddingSize() = aIn.GetPaddingSize(); +} + +void TypeUtils::ToCacheResponse(JSContext* aCx, CacheResponse& aOut, + Response& aIn, ErrorResult& aRv) { + if (aIn.BodyUsed()) { + aRv.ThrowTypeError<MSG_FETCH_BODY_CONSUMED_ERROR>(); + return; + } + + SafeRefPtr<InternalResponse> ir = aIn.GetInternalResponse(); + ToCacheResponseWithoutBody(aOut, *ir, aRv); + if (NS_WARN_IF(aRv.Failed())) { + return; + } + + nsCOMPtr<nsIInputStream> stream; + ir->GetUnfilteredBody(getter_AddRefs(stream)); + if (stream) { + aIn.SetBodyUsed(aCx, aRv); + if (NS_WARN_IF(aRv.Failed())) { + return; + } + } + + SerializeCacheStream(stream, &aOut.body(), aRv); + if (NS_WARN_IF(aRv.Failed())) { + return; + } +} + +// static +void TypeUtils::ToCacheQueryParams(CacheQueryParams& aOut, + const CacheQueryOptions& aIn) { + aOut.ignoreSearch() = aIn.mIgnoreSearch; + aOut.ignoreMethod() = aIn.mIgnoreMethod; + aOut.ignoreVary() = aIn.mIgnoreVary; +} + +// static +void TypeUtils::ToCacheQueryParams(CacheQueryParams& aOut, + const MultiCacheQueryOptions& aIn) { + ToCacheQueryParams(aOut, static_cast<const CacheQueryOptions&>(aIn)); + aOut.cacheNameSet() = aIn.mCacheName.WasPassed(); + if (aOut.cacheNameSet()) { + aOut.cacheName() = aIn.mCacheName.Value(); + } else { + aOut.cacheName() = u""_ns; + } +} + +already_AddRefed<Response> TypeUtils::ToResponse(const CacheResponse& aIn) { + if (aIn.type() == ResponseType::Error) { + // We don't bother tracking the internal error code for cached responses... + RefPtr<Response> r = + new Response(GetGlobalObject(), + InternalResponse::NetworkError(NS_ERROR_FAILURE), nullptr); + return r.forget(); + } + + SafeRefPtr<InternalResponse> ir = + MakeSafeRefPtr<InternalResponse>(aIn.status(), aIn.statusText()); + ir->SetURLList(aIn.urlList()); + + RefPtr<InternalHeaders> internalHeaders = + ToInternalHeaders(aIn.headers(), aIn.headersGuard()); + ErrorResult result; + + // Be careful to fill the headers before setting the guard in order to + // correctly re-create the original headers. + ir->Headers()->Fill(*internalHeaders, result); + MOZ_DIAGNOSTIC_ASSERT(!result.Failed()); + ir->Headers()->SetGuard(aIn.headersGuard(), result); + MOZ_DIAGNOSTIC_ASSERT(!result.Failed()); + + ir->InitChannelInfo(aIn.securityInfo()); + if (aIn.principalInfo().isSome()) { + UniquePtr<mozilla::ipc::PrincipalInfo> info( + new mozilla::ipc::PrincipalInfo(aIn.principalInfo().ref())); + ir->SetPrincipalInfo(std::move(info)); + } + + nsCOMPtr<nsIInputStream> stream = ReadStream::Create(aIn.body()); + ir->SetBody(stream, InternalResponse::UNKNOWN_BODY_SIZE); + + switch (aIn.type()) { + case ResponseType::Basic: + ir = ir->BasicResponse(); + break; + case ResponseType::Cors: + ir = ir->CORSResponse(); + break; + case ResponseType::Default: + break; + case ResponseType::Opaque: + ir = ir->OpaqueResponse(); + break; + case ResponseType::Opaqueredirect: + ir = ir->OpaqueRedirectResponse(); + break; + default: + MOZ_CRASH("Unexpected ResponseType!"); + } + MOZ_DIAGNOSTIC_ASSERT(ir); + + ir->SetPaddingSize(aIn.paddingSize()); + + RefPtr<Response> ref = + new Response(GetGlobalObject(), std::move(ir), nullptr); + return ref.forget(); +} +SafeRefPtr<InternalRequest> TypeUtils::ToInternalRequest( + const CacheRequest& aIn) { + nsAutoCString url(aIn.urlWithoutQuery()); + url.Append(aIn.urlQuery()); + auto internalRequest = + MakeSafeRefPtr<InternalRequest>(url, aIn.urlFragment()); + internalRequest->SetMethod(aIn.method()); + internalRequest->SetReferrer(aIn.referrer()); + internalRequest->SetReferrerPolicy(aIn.referrerPolicy()); + internalRequest->SetMode(aIn.mode()); + internalRequest->SetCredentialsMode(aIn.credentials()); + internalRequest->SetContentPolicyType(aIn.contentPolicyType()); + internalRequest->SetCacheMode(aIn.requestCache()); + internalRequest->SetRedirectMode(aIn.requestRedirect()); + internalRequest->SetIntegrity(aIn.integrity()); + + RefPtr<InternalHeaders> internalHeaders = + ToInternalHeaders(aIn.headers(), aIn.headersGuard()); + ErrorResult result; + + // Be careful to fill the headers before setting the guard in order to + // correctly re-create the original headers. + internalRequest->Headers()->Fill(*internalHeaders, result); + MOZ_DIAGNOSTIC_ASSERT(!result.Failed()); + + internalRequest->Headers()->SetGuard(aIn.headersGuard(), result); + MOZ_DIAGNOSTIC_ASSERT(!result.Failed()); + + nsCOMPtr<nsIInputStream> stream = ReadStream::Create(aIn.body()); + + internalRequest->SetBody(stream, -1); + + return internalRequest; +} + +SafeRefPtr<Request> TypeUtils::ToRequest(const CacheRequest& aIn) { + return MakeSafeRefPtr<Request>(GetGlobalObject(), ToInternalRequest(aIn), + nullptr); +} + +// static +already_AddRefed<InternalHeaders> TypeUtils::ToInternalHeaders( + const nsTArray<HeadersEntry>& aHeadersEntryList, HeadersGuardEnum aGuard) { + nsTArray<InternalHeaders::Entry> entryList; + entryList.SetCapacity(aHeadersEntryList.Length()); + std::transform(aHeadersEntryList.cbegin(), aHeadersEntryList.cend(), + MakeBackInserter(entryList), [](const auto& headersEntry) { + return InternalHeaders::Entry(headersEntry.name(), + headersEntry.value()); + }); + + RefPtr<InternalHeaders> ref = + new InternalHeaders(std::move(entryList), aGuard); + return ref.forget(); +} + +// Utility function to remove the fragment from a URL, check its scheme, and +// optionally provide a URL without the query. We're not using nsIURL or URL to +// do this because they require going to the main thread. static +void TypeUtils::ProcessURL(nsACString& aUrl, bool* aSchemeValidOut, + nsACString* aUrlWithoutQueryOut, + nsACString* aUrlQueryOut, ErrorResult& aRv) { + const nsCString& flatURL = PromiseFlatCString(aUrl); + const char* url = flatURL.get(); + + // off the main thread URL parsing using nsStdURLParser. + nsCOMPtr<nsIURLParser> urlParser = new nsStdURLParser(); + + uint32_t pathPos; + int32_t pathLen; + uint32_t schemePos; + int32_t schemeLen; + aRv = urlParser->ParseURL(url, flatURL.Length(), &schemePos, &schemeLen, + nullptr, nullptr, // ignore authority + &pathPos, &pathLen); + if (NS_WARN_IF(aRv.Failed())) { + return; + } + + if (aSchemeValidOut) { + nsAutoCString scheme(Substring(flatURL, schemePos, schemeLen)); + *aSchemeValidOut = + scheme.LowerCaseEqualsLiteral("http") || + scheme.LowerCaseEqualsLiteral("https") || + (StaticPrefs::extensions_backgroundServiceWorker_enabled_AtStartup() && + scheme.LowerCaseEqualsLiteral("moz-extension")); + } + + uint32_t queryPos; + int32_t queryLen; + + aRv = urlParser->ParsePath(url + pathPos, flatURL.Length() - pathPos, nullptr, + nullptr, // ignore filepath + &queryPos, &queryLen, nullptr, nullptr); + if (NS_WARN_IF(aRv.Failed())) { + return; + } + + if (!aUrlWithoutQueryOut) { + return; + } + + MOZ_DIAGNOSTIC_ASSERT(aUrlQueryOut); + + if (queryLen < 0) { + *aUrlWithoutQueryOut = aUrl; + aUrlQueryOut->Truncate(); + return; + } + + // ParsePath gives us query position relative to the start of the path + queryPos += pathPos; + + *aUrlWithoutQueryOut = Substring(aUrl, 0, queryPos - 1); + *aUrlQueryOut = Substring(aUrl, queryPos - 1, queryLen + 1); +} + +void TypeUtils::CheckAndSetBodyUsed(JSContext* aCx, Request& aRequest, + BodyAction aBodyAction, ErrorResult& aRv) { + if (aBodyAction == IgnoreBody) { + return; + } + + if (aRequest.BodyUsed()) { + aRv.ThrowTypeError<MSG_FETCH_BODY_CONSUMED_ERROR>(); + return; + } + + nsCOMPtr<nsIInputStream> stream; + aRequest.GetBody(getter_AddRefs(stream)); + if (stream) { + aRequest.SetBodyUsed(aCx, aRv); + if (NS_WARN_IF(aRv.Failed())) { + return; + } + } +} + +SafeRefPtr<InternalRequest> TypeUtils::ToInternalRequest(const nsAString& aIn, + ErrorResult& aRv) { + RequestOrUSVString requestOrString; + requestOrString.SetAsUSVString().ShareOrDependUpon(aIn); + + // Re-create a GlobalObject stack object so we can use webidl Constructors. + AutoJSAPI jsapi; + if (NS_WARN_IF(!jsapi.Init(GetGlobalObject()))) { + aRv.Throw(NS_ERROR_UNEXPECTED); + return nullptr; + } + JSContext* cx = jsapi.cx(); + GlobalObject global(cx, GetGlobalObject()->GetGlobalJSObject()); + MOZ_DIAGNOSTIC_ASSERT(!global.Failed()); + + RootedDictionary<RequestInit> requestInit(cx); + SafeRefPtr<Request> request = + Request::Constructor(global, requestOrString, requestInit, aRv); + if (NS_WARN_IF(aRv.Failed())) { + return nullptr; + } + + return request->GetInternalRequest(); +} + +void TypeUtils::SerializeCacheStream(nsIInputStream* aStream, + Maybe<CacheReadStream>* aStreamOut, + ErrorResult& aRv) { + *aStreamOut = Nothing(); + if (!aStream) { + return; + } + + RefPtr<ReadStream> controlled = do_QueryObject(aStream); + if (controlled) { + controlled->Serialize(aStreamOut, aRv); + return; + } + + aStreamOut->emplace(CacheReadStream()); + CacheReadStream& cacheStream = aStreamOut->ref(); + + cacheStream.control() = nullptr; + + MOZ_ALWAYS_TRUE(mozilla::ipc::SerializeIPCStream(do_AddRef(aStream), + cacheStream.stream(), + /* aAllowLazy */ false)); +} + +} // namespace mozilla::dom::cache diff --git a/dom/cache/TypeUtils.h b/dom/cache/TypeUtils.h new file mode 100644 index 0000000000..5342ea3d7e --- /dev/null +++ b/dom/cache/TypeUtils.h @@ -0,0 +1,138 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_TypesUtils_h +#define mozilla_dom_cache_TypesUtils_h + +#include "mozilla/AlreadyAddRefed.h" // for already_AddRefed +#include "mozilla/UniquePtr.h" // for UniquePtr +#include "mozilla/dom/HeadersBinding.h" // for HeadersGuardEnum, HeadersGua... +#include "mozilla/dom/SafeRefPtr.h" // for SafeRefPtr +#include "nsStringFwd.h" // for nsACString, nsAString + +class nsIGlobalObject; +class nsIAsyncInputStream; +class nsIInputStream; + +namespace mozilla { + +namespace ipc { +class PBackgroundChild; +} // namespace ipc + +namespace dom { + +struct CacheQueryOptions; +struct MultiCacheQueryOptions; +class InternalHeaders; +class InternalRequest; +class InternalResponse; +class OwningRequestOrUSVString; +class Request; +class RequestOrUSVString; +class Response; + +namespace cache { + +class CacheQueryParams; +class CacheReadStream; +class CacheRequest; +class CacheResponse; +class HeadersEntry; + +class TypeUtils { + public: + enum BodyAction { IgnoreBody, ReadBody }; + + enum SchemeAction { IgnoreInvalidScheme, TypeErrorOnInvalidScheme }; + + ~TypeUtils() = default; + virtual nsIGlobalObject* GetGlobalObject() const = 0; +#ifdef DEBUG + virtual void AssertOwningThread() const = 0; +#else + inline void AssertOwningThread() const {} +#endif + + // This is mainly declared to support serializing body streams. Some + // TypeUtils implementations do not expect to be used for this kind of + // serialization. These classes will MOZ_CRASH() if you try to call + // GetIPCManager(). + virtual mozilla::ipc::PBackgroundChild* GetIPCManager() = 0; + + SafeRefPtr<InternalRequest> ToInternalRequest(JSContext* aCx, + const RequestOrUSVString& aIn, + BodyAction aBodyAction, + ErrorResult& aRv); + + SafeRefPtr<InternalRequest> ToInternalRequest( + JSContext* aCx, const OwningRequestOrUSVString& aIn, + BodyAction aBodyAction, ErrorResult& aRv); + + void ToCacheRequest(CacheRequest& aOut, const InternalRequest& aIn, + BodyAction aBodyAction, SchemeAction aSchemeAction, + ErrorResult& aRv); + + void ToCacheResponseWithoutBody(CacheResponse& aOut, InternalResponse& aIn, + ErrorResult& aRv); + + void ToCacheResponse(JSContext* aCx, CacheResponse& aOut, Response& aIn, + ErrorResult& aRv); + + void ToCacheQueryParams(CacheQueryParams& aOut, const CacheQueryOptions& aIn); + + void ToCacheQueryParams(CacheQueryParams& aOut, + const MultiCacheQueryOptions& aIn); + + already_AddRefed<Response> ToResponse(const CacheResponse& aIn); + + SafeRefPtr<InternalRequest> ToInternalRequest(const CacheRequest& aIn); + + SafeRefPtr<Request> ToRequest(const CacheRequest& aIn); + + // static methods + static already_AddRefed<InternalHeaders> ToInternalHeaders( + const nsTArray<HeadersEntry>& aHeadersEntryList, + HeadersGuardEnum aGuard = HeadersGuardEnum::None); + + // Utility method for parsing a URL and doing associated operations. A mix + // of things are done in this one method to avoid duplicated parsing: + // + // 1) The aUrl argument is modified to strip the fragment + // 2) If aSchemaValidOut is set, then a boolean value is set indicating + // if the aUrl's scheme is valid or not for storing in the cache. + // 3) If aUrlWithoutQueryOut is set, then a url string is provided without + // the search section. + // 4) If aUrlQueryOut is set then its populated with the search section + // of the URL. Note, this parameter must be set if aUrlWithoutQueryOut + // is set. They must either both be nullptr or set to valid string + // pointers. + // + // Any errors are thrown on ErrorResult. + static void ProcessURL(nsACString& aUrl, bool* aSchemeValidOut, + nsACString* aUrlWithoutQueryOut, + nsACString* aUrlQueryOut, ErrorResult& aRv); + + private: + void CheckAndSetBodyUsed(JSContext* aCx, Request& aRequest, + BodyAction aBodyAction, ErrorResult& aRv); + + SafeRefPtr<InternalRequest> ToInternalRequest(const nsAString& aIn, + ErrorResult& aRv); + + void SerializeCacheStream(nsIInputStream* aStream, + Maybe<CacheReadStream>* aStreamOut, + ErrorResult& aRv); + + void SerializeSendStream(nsIInputStream* aStream, + CacheReadStream& aReadStreamOut, ErrorResult& aRv); +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_cache_TypesUtils_h diff --git a/dom/cache/Types.h b/dom/cache/Types.h new file mode 100644 index 0000000000..b033f0cea4 --- /dev/null +++ b/dom/cache/Types.h @@ -0,0 +1,59 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_Types_h +#define mozilla_dom_cache_Types_h + +#include <functional> +#include <stdint.h> +#include "mozilla/dom/quota/CommonMetadata.h" +#include "nsCOMPtr.h" +#include "nsIFile.h" +#include "nsIInputStream.h" +#include "nsString.h" + +namespace mozilla::dom::cache { + +enum Namespace { + DEFAULT_NAMESPACE, + CHROME_ONLY_NAMESPACE, + NUMBER_OF_NAMESPACES +}; +static const Namespace INVALID_NAMESPACE = NUMBER_OF_NAMESPACES; + +using CacheId = int64_t; +static const CacheId INVALID_CACHE_ID = -1; + +struct CacheDirectoryMetadata : quota::ClientMetadata { + nsCOMPtr<nsIFile> mDir; + int64_t mDirectoryLockId = -1; + + explicit CacheDirectoryMetadata(quota::PrincipalMetadata aPrincipalMetadata) + : quota::ClientMetadata( + quota::OriginMetadata{std::move(aPrincipalMetadata), + quota::PERSISTENCE_TYPE_DEFAULT}, + quota::Client::Type::DOMCACHE) {} + + explicit CacheDirectoryMetadata(quota::OriginMetadata aOriginMetadata) + : quota::ClientMetadata(std::move(aOriginMetadata), + quota::Client::Type::DOMCACHE) { + MOZ_DIAGNOSTIC_ASSERT(aOriginMetadata.mPersistenceType == + quota::PERSISTENCE_TYPE_DEFAULT); + } +}; + +struct DeletionInfo { + nsTArray<nsID> mDeletedBodyIdList; + int64_t mDeletedPaddingSize = 0; +}; + +using InputStreamResolver = std::function<void(nsCOMPtr<nsIInputStream>&&)>; + +enum class OpenMode : uint8_t { Eager, Lazy, NumTypes }; + +} // namespace mozilla::dom::cache + +#endif // mozilla_dom_cache_Types_h diff --git a/dom/cache/moz.build b/dom/cache/moz.build new file mode 100644 index 0000000000..df9499f9c9 --- /dev/null +++ b/dom/cache/moz.build @@ -0,0 +1,99 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +with Files("**"): + BUG_COMPONENT = ("Core", "Storage: Cache API") + +EXPORTS.mozilla.dom.cache += [ + "Action.h", + "ActorChild.h", + "ActorUtils.h", + "AutoUtils.h", + "Cache.h", + "CacheChild.h", + "CacheCommon.h", + "CacheOpChild.h", + "CacheOpParent.h", + "CacheParent.h", + "CacheStorage.h", + "CacheStorageChild.h", + "CacheStorageParent.h", + "CacheStreamControlChild.h", + "CacheStreamControlParent.h", + "CacheWorkerRef.h", + "Connection.h", + "Context.h", + "DBAction.h", + "DBSchema.h", + "FileUtils.h", + "IPCUtils.h", + "Manager.h", + "ManagerId.h", + "PrincipalVerifier.h", + "QuotaClient.h", + "ReadStream.h", + "SavedTypes.h", + "StreamControl.h", + "StreamList.h", + "Types.h", + "TypeUtils.h", +] + +UNIFIED_SOURCES += [ + "Action.cpp", + "ActorChild.cpp", + "AutoUtils.cpp", + "Cache.cpp", + "CacheChild.cpp", + "CacheCommon.cpp", + "CacheOpChild.cpp", + "CacheOpParent.cpp", + "CacheParent.cpp", + "CacheStorage.cpp", + "CacheStorageChild.cpp", + "CacheStorageParent.cpp", + "CacheStreamControlChild.cpp", + "CacheStreamControlParent.cpp", + "CacheWorkerRef.cpp", + "Connection.cpp", + "Context.cpp", + "DBAction.cpp", + "DBSchema.cpp", + "FileUtils.cpp", + "Manager.cpp", + "ManagerId.cpp", + "PrincipalVerifier.cpp", + "QuotaClient.cpp", + "ReadStream.cpp", + "StreamControl.cpp", + "StreamList.cpp", + "TypeUtils.cpp", +] + +IPDL_SOURCES += [ + "CacheTypes.ipdlh", + "PCache.ipdl", + "PCacheOp.ipdl", + "PCacheStorage.ipdl", + "PCacheStreamControl.ipdl", +] + +include("/ipc/chromium/chromium-config.mozbuild") + +FINAL_LIBRARY = "xul" + +MOCHITEST_MANIFESTS += [ + "test/mochitest/mochitest.ini", +] + +BROWSER_CHROME_MANIFESTS += [ + "test/browser/browser.ini", +] + +XPCSHELL_TESTS_MANIFESTS += [ + "test/xpcshell/xpcshell.ini", +] diff --git a/dom/cache/test/browser/browser.ini b/dom/cache/test/browser/browser.ini new file mode 100644 index 0000000000..90a7a53a3a --- /dev/null +++ b/dom/cache/test/browser/browser.ini @@ -0,0 +1 @@ +[browser_cache_pb_window.js] diff --git a/dom/cache/test/browser/browser_cache_pb_window.js b/dom/cache/test/browser/browser_cache_pb_window.js new file mode 100644 index 0000000000..40a6da7005 --- /dev/null +++ b/dom/cache/test/browser/browser_cache_pb_window.js @@ -0,0 +1,169 @@ +/* eslint-disable no-shadow */ +var name = "pb-window-cache"; + +function testMatch(browser) { + return SpecialPowers.spawn(browser, [name], function (name) { + return new Promise((resolve, reject) => { + content.caches + .match("http://foo.com") + .then(function (response) { + ok(false, "caches.match() should not return success"); + reject(); + }) + .catch(function (err) { + is( + "SecurityError", + err.name, + "caches.match() should throw SecurityError" + ); + resolve(); + }); + }); + }); +} + +function testHas(browser) { + return SpecialPowers.spawn(browser, [name], function (name) { + return new Promise(function (resolve, reject) { + content.caches + .has(name) + .then(function (result) { + ok(false, "caches.has() should not return success"); + reject(); + }) + .catch(function (err) { + is( + "SecurityError", + err.name, + "caches.has() should throw SecurityError" + ); + resolve(); + }); + }); + }); +} + +function testOpen(browser) { + return SpecialPowers.spawn(browser, [name], function (name) { + return new Promise(function (resolve, reject) { + content.caches + .open(name) + .then(function (c) { + ok(false, "caches.open() should not return success"); + reject(); + }) + .catch(function (err) { + is( + "SecurityError", + err.name, + "caches.open() should throw SecurityError" + ); + resolve(); + }); + }); + }); +} + +function testDelete(browser) { + return SpecialPowers.spawn(browser, [name], function (name) { + return new Promise(function (resolve, reject) { + content.caches + .delete(name) + .then(function (result) { + ok(false, "caches.delete() should not return success"); + reject(); + }) + .catch(function (err) { + is( + "SecurityError", + err.name, + "caches.delete() should throw SecurityError" + ); + resolve(); + }); + }); + }); +} + +function testKeys(browser) { + return SpecialPowers.spawn(browser, [name], function (name) { + return new Promise(function (resolve, reject) { + content.caches + .keys() + .then(function (names) { + ok(false, "caches.keys() should not return success"); + reject(); + }) + .catch(function (err) { + is( + "SecurityError", + err.name, + "caches.keys() should throw SecurityError" + ); + resolve(); + }); + }); + }); +} + +function testOpen_worker(browser) { + return SpecialPowers.spawn(browser, [], function () { + let workerFunctionString = function () { + caches.open("pb-worker-cache").then( + function (cacheObject) { + postMessage(cacheObject.toString()); + }, + function (reason) { + postMessage(reason.name); + } + ); + }.toString(); + let workerBlobURL = content.URL.createObjectURL( + new Blob(["(", workerFunctionString, ")()"], { + type: "application/javascript", + }) + ); + let worker = new content.Worker(workerBlobURL); + content.URL.revokeObjectURL(workerBlobURL); + return new Promise(function (resolve, reject) { + worker.addEventListener("message", function (e) { + let isGood = e.data === "SecurityError"; + ok(isGood, "caches.open() should throw SecurityError from worker"); + isGood ? resolve() : reject(); + }); + }); + }); +} + +function test() { + let privateWin, privateTab; + waitForExplicitFinish(); + SpecialPowers.pushPrefEnv({ + set: [ + ["dom.caches.enabled", true], + ["dom.caches.testing.enabled", true], + ], + }) + .then(() => { + return BrowserTestUtils.openNewBrowserWindow({ private: true }); + }) + .then(pw => { + privateWin = pw; + privateTab = BrowserTestUtils.addTab(pw.gBrowser, "http://example.com/"); + return BrowserTestUtils.browserLoaded(privateTab.linkedBrowser); + }) + .then(tab => { + return Promise.all([ + testMatch(privateTab.linkedBrowser), + testHas(privateTab.linkedBrowser), + testOpen(privateTab.linkedBrowser), + testDelete(privateTab.linkedBrowser), + testKeys(privateTab.linkedBrowser), + testOpen_worker(privateTab.linkedBrowser), + ]); + }) + .then(() => { + return BrowserTestUtils.closeWindow(privateWin); + }) + .then(finish); +} diff --git a/dom/cache/test/marionette/manifest.ini b/dom/cache/test/marionette/manifest.ini new file mode 100644 index 0000000000..c0bf3194b2 --- /dev/null +++ b/dom/cache/test/marionette/manifest.ini @@ -0,0 +1 @@ +[test_caches_delete_cleanup_after_shutdown.py] diff --git a/dom/cache/test/marionette/test_caches_delete_cleanup_after_shutdown.py b/dom/cache/test/marionette/test_caches_delete_cleanup_after_shutdown.py new file mode 100644 index 0000000000..4427d1c9c7 --- /dev/null +++ b/dom/cache/test/marionette/test_caches_delete_cleanup_after_shutdown.py @@ -0,0 +1,181 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from marionette_driver import Wait +from marionette_harness import MarionetteTestCase + +""" + Currently we expect our size after cleanup to be 98k for the current + database schema and page sizes and constants used in this test. We + set the threshold at 128k so the test doesn't start failing if these + control objects/structures increase somewhat in size, but should still + fail if we fail to delete all of the 5,000 1k files we create on disk + as part of the test. +""" +EXPECTED_CACHEDIR_SIZE_AFTER_CLEANUP = 128 * 1024 # 128KB +CACHE_ID = "data" + + +class CachesDeleteCleanupAtShutdownTestCase(MarionetteTestCase): + + """ + Bug1784700: This test ensures that cache body files get cleaned up + properly after cache has been deleted. Note that body files gets + cleaned up asynchronously which means that these files might still + be around even after Caches.Delete promise gets resolved. Currently, + we would only clean up body files on origin initialization and that's + a firefox is necessary here. + """ + + def setUp(self): + super(CachesDeleteCleanupAtShutdownTestCase, self).setUp() + self.marionette.restart(in_app=False, clean=True) + + def tearDown(self): + self.marionette.restart(in_app=False, clean=True) + super(CachesDeleteCleanupAtShutdownTestCase, self).tearDown() + + def getUsage(self): + return self.marionette.execute_script( + """ + return window.wrappedJSObject.getStorageEstimate(); + """, + new_sandbox=False, + ) + + def doCacheWork(self, n): + # max timeout for this script to execute is 5 minutes + maxTimeout = 5 * 60 * 1000 + + return self.marionette.execute_script( + """ + const [cacheId, n] = arguments; + return window.wrappedJSObject.doCacheWork(cacheId, n); + """, + script_args=( + CACHE_ID, + n, + ), + new_sandbox=False, + script_timeout=maxTimeout, + ) + + def openCache(self): + return self.marionette.execute_async_script( + """ + const [cacheId, resolve] = arguments; + window.wrappedJSObject.openCache(cacheId).then(resolve("success")); + """, + new_sandbox=False, + script_args=(CACHE_ID,), + ) + + def ensureCleanDirectory(self): + with self.marionette.using_context("chrome"): + return self.marionette.execute_script( + """ + let originDir = arguments[0]; + const pathDelimiter = "/"; + + function getRelativeFile(relativePath) { + let file = Services.dirsvc + .get("ProfD", Ci.nsIFile) + .clone(); + + relativePath.split(pathDelimiter).forEach(function(component) { + if (component == "..") { + file = file.parent; + } else { + file.append(component); + } + }); + + return file; + } + + function getCacheDir() { + + const storageDirName = "storage"; + const defaultPersistenceDirName = "default"; + + return getRelativeFile( + `${storageDirName}/${defaultPersistenceDirName}/${originDir}/cache` + ); + } + + const cacheDir = getCacheDir(); + let morgueDir = cacheDir.clone(); + + // morgue directory should be empty + // or atleast directories under morgue should be empty + morgueDir.append("morgue"); + for (let dir of morgueDir.directoryEntries) { + for (let file of dir.directoryEntries) { + return false; + } + } + return true; + """, + script_args=( + self.marionette.absolute_url("")[:-1] + .replace(":", "+") + .replace("/", "+"), + ), + new_sandbox=False, + ) + + def create_and_cleanup_cache(self, ensureCleanCallback, in_app): + # create 640 cache entries + self.doCacheWork(640) + + self.marionette.restart(in_app=in_app) + print("restart successful") + + self.marionette.navigate(self.marionette.absolute_url("dom/cacheUsage.html")) + return ensureCleanCallback() + + def test_ensure_cache_cleanup_after_clean_restart(self): + self.marionette.navigate(self.marionette.absolute_url("dom/cacheUsage.html")) + beforeUsage = self.getUsage() + + def ensureCleanCallback(): + + Wait(self.marionette, timeout=60).until( + lambda x: (self.getUsage() - beforeUsage) + < EXPECTED_CACHEDIR_SIZE_AFTER_CLEANUP, + message="Cache directory is not cleaned up properly", + ) + + return ( + abs(beforeUsage - self.getUsage()) + <= EXPECTED_CACHEDIR_SIZE_AFTER_CLEANUP + and self.ensureCleanDirectory() + ) + + if not self.create_and_cleanup_cache(ensureCleanCallback, True): + print(f"beforeUsage = {beforeUsage}, and afterUsage = {self.getUsage()}") + assert False + + def test_ensure_cache_cleanup_after_unclean_restart(self): + self.marionette.navigate(self.marionette.absolute_url("dom/cacheUsage.html")) + beforeUsage = self.getUsage() + + def ensureCleanCallback(): + self.openCache() + + Wait(self.marionette, timeout=60).until( + lambda x: (self.getUsage() - beforeUsage) + < EXPECTED_CACHEDIR_SIZE_AFTER_CLEANUP, + message="Cache directory is not cleaned up properly", + ) + + return ( + abs(beforeUsage - self.getUsage()) + <= EXPECTED_CACHEDIR_SIZE_AFTER_CLEANUP + and self.ensureCleanDirectory() + ) + + if not self.create_and_cleanup_cache(ensureCleanCallback, False): + print(f"beforeUsage = {beforeUsage}, and afterUsage = {self.getUsage()}") + assert False diff --git a/dom/cache/test/mochitest/driver.js b/dom/cache/test/mochitest/driver.js new file mode 100644 index 0000000000..4ea3e2ad03 --- /dev/null +++ b/dom/cache/test/mochitest/driver.js @@ -0,0 +1,157 @@ +// Any copyright is dedicated to the Public Domain. +// http://creativecommons.org/publicdomain/zero/1.0/ +// +// This helper script exposes a runTests function that takes the name of a +// test script as its input argument and runs the test in three different +// contexts: +// 1. Regular Worker context +// 2. Service Worker context +// 3. Window context +// The function returns a promise which will get resolved once all tests +// finish. The testFile argument is the name of the test file to be run +// in the different contexts, and the optional order argument can be set +// to either "parallel" or "sequential" depending on how the caller wants +// the tests to be run. If this argument is not provided, the default is +// "both", which runs the tests in both modes. +// The caller of this function is responsible to call SimpleTest.finish +// when the returned promise is resolved. + +function runTests(testFile, order) { + async function setupPrefs() { + // Bug 1746646: Make mochitests work with TCP enabled (cookieBehavior = 5) + // Acquire storage access permission here so that the Cache API is avaialable + SpecialPowers.wrap(document).notifyUserGestureActivation(); + await SpecialPowers.addPermission( + "storageAccessAPI", + true, + window.location.href + ); + await SpecialPowers.wrap(document).requestStorageAccess(); + return SpecialPowers.pushPrefEnv({ + set: [ + ["dom.caches.enabled", true], + ["dom.caches.testing.enabled", true], + ["dom.serviceWorkers.enabled", true], + ["dom.serviceWorkers.testing.enabled", true], + ["dom.serviceWorkers.exemptFromPerDomainMax", true], + [ + "privacy.partition.always_partition_third_party_non_cookie_storage", + false, + ], + ], + }); + } + + // adapted from dom/indexedDB/test/helpers.js + function clearStorage() { + var clearUnpartitionedStorage = new Promise(function (resolve, reject) { + var qms = SpecialPowers.Services.qms; + var principal = SpecialPowers.wrap(document).nodePrincipal; + var request = qms.clearStoragesForPrincipal(principal); + var cb = SpecialPowers.wrapCallback(resolve); + request.callback = cb; + }); + var clearPartitionedStorage = new Promise(function (resolve, reject) { + var qms = SpecialPowers.Services.qms; + var principal = SpecialPowers.wrap(document).partitionedPrincipal; + var request = qms.clearStoragesForPrincipal(principal); + var cb = SpecialPowers.wrapCallback(resolve); + request.callback = cb; + }); + return Promise.all([clearUnpartitionedStorage, clearPartitionedStorage]); + } + + function loadScript(script) { + return new Promise(function (resolve, reject) { + var s = document.createElement("script"); + s.src = script; + s.onerror = reject; + s.onload = resolve; + document.body.appendChild(s); + }); + } + + function importDrivers() { + /* import-globals-from worker_driver.js */ + /* import-globals-from serviceworker_driver.js */ + return Promise.all([ + loadScript("worker_driver.js"), + loadScript("serviceworker_driver.js"), + ]); + } + + function runWorkerTest() { + return workerTestExec(testFile); + } + + function runServiceWorkerTest() { + return serviceWorkerTestExec(testFile); + } + + function runFrameTest() { + return new Promise(function (resolve, reject) { + var iframe = document.createElement("iframe"); + iframe.src = "frame.html"; + iframe.onload = function () { + var doc = iframe.contentDocument; + var s = doc.createElement("script"); + s.src = testFile; + window.addEventListener("message", function onMessage(event) { + if (event.data.context != "Window") { + return; + } + if (event.data.type == "finish") { + window.removeEventListener("message", onMessage); + resolve(); + } else if (event.data.type == "status") { + ok(event.data.status, event.data.context + ": " + event.data.msg); + } + }); + doc.body.appendChild(s); + }; + document.body.appendChild(iframe); + }); + } + + SimpleTest.waitForExplicitFinish(); + + if (typeof order == "undefined") { + order = "sequential"; // sequential by default, see bug 1143222. + // TODO: Make this "both" again. + } + + ok( + order == "parallel" || order == "sequential" || order == "both", + "order argument should be valid" + ); + + if (order == "both") { + info("Running tests in both modes; first: sequential"); + return runTests(testFile, "sequential").then(function () { + info("Running tests in parallel mode"); + return runTests(testFile, "parallel"); + }); + } + if (order == "sequential") { + return setupPrefs() + .then(importDrivers) + .then(runWorkerTest) + .then(clearStorage) + .then(runServiceWorkerTest) + .then(clearStorage) + .then(runFrameTest) + .then(clearStorage) + .catch(function (e) { + ok(false, "A promise was rejected during test execution: " + e); + }); + } + return setupPrefs() + .then(importDrivers) + .then(() => + Promise.all([runWorkerTest(), runServiceWorkerTest(), runFrameTest()]) + ) + .then(clearStorage) + .catch(function (e) { + ok(false, "A promise was rejected during test execution: " + e); + }); +} diff --git a/dom/cache/test/mochitest/empty.html b/dom/cache/test/mochitest/empty.html new file mode 100644 index 0000000000..5b56631db7 --- /dev/null +++ b/dom/cache/test/mochitest/empty.html @@ -0,0 +1,2 @@ +<!DOCTYPE html> +<!-- This is only used to give us access to the caches global after setting the pref. --> diff --git a/dom/cache/test/mochitest/frame.html b/dom/cache/test/mochitest/frame.html new file mode 100644 index 0000000000..0e6dd35a24 --- /dev/null +++ b/dom/cache/test/mochitest/frame.html @@ -0,0 +1,17 @@ +<!DOCTYPE html> +<script> + var context = "Window"; + function ok(a, msg) { + parent.postMessage({type: "status", status: !!a, + msg: a + ": " + msg, context}, "*"); + } + + function is(a, b, msg) { + parent.postMessage({type: "status", status: a === b, + msg: a + " === " + b + ": " + msg, context}, "*"); + } + + function testDone() { + parent.postMessage({type: "finish", context}, "*"); + } +</script> diff --git a/dom/cache/test/mochitest/idle_worker.js b/dom/cache/test/mochitest/idle_worker.js new file mode 100644 index 0000000000..baba3ed4dc --- /dev/null +++ b/dom/cache/test/mochitest/idle_worker.js @@ -0,0 +1,12 @@ +// Touch CacheStorage, but then idle and do nothing. +const name = "idle_worker_cache"; +var cache; +self.caches + .open(name) + .then(c => { + cache = c; + return self.caches.delete(name); + }) + .then(_ => { + postMessage("LOADED"); + }); diff --git a/dom/cache/test/mochitest/large_url_list.js b/dom/cache/test/mochitest/large_url_list.js new file mode 100644 index 0000000000..db24c790fc --- /dev/null +++ b/dom/cache/test/mochitest/large_url_list.js @@ -0,0 +1,1002 @@ +var largeUrlList = [ + "https://example.com/cia5rherm0000hkjx7r8of8b1/cia5rhern0001hkjxes0mptxq/cia5rhern0002hkjx0ze9t1oy/cia5rhern0003hkjxa07rei71", + "https://example.com/cia5rhern0004hkjx15dh2a23/cia5rhern0005hkjxukeg547n/cia5rhern0006hkjxn78qlfk0/cia5rhern0007hkjxx7pkqem5", + "https://example.com/cia5rhern0008hkjx6xh95dcp/cia5rhern0009hkjxu3212vkx/cia5rhern000ahkjxlhu69sv7/cia5rhern000bhkjxwffaxb8c", + "https://example.com/cia5rhern000chkjxylxzrc0l/cia5rhern000dhkjxoe8p7jap/cia5rhern000ehkjx49ssgz0i/cia5rhern000fhkjxlqcm6jq9", + "https://example.com/cia5rhern000ghkjxjfag4uqk/cia5rhern000hhkjxxjmvakfj/cia5rhern000ihkjxp5j7d9ic/cia5rhern000jhkjxdog5rxfb", + "https://example.com/cia5rhern000khkjxrdrwsflr/cia5rhern000lhkjxjqj2ydrd/cia5rhern000mhkjxujiqkc1c/cia5rhern000nhkjxw0nk2nvg", + "https://example.com/cia5rhern000ohkjxpglj3lmb/cia5rhern000phkjxjekhrrqd/cia5rhern000qhkjxalsyb73l/cia5rhern000rhkjxyjbnn6er", + "https://example.com/cia5rhero000shkjxmhknjjrv/cia5rhero000thkjx8xtfdrgh/cia5rhero000uhkjxpe3dwxk6/cia5rhero000vhkjxint5ohhn", + "https://example.com/cia5rhero000whkjxvaiypkdj/cia5rhero000xhkjxaqm599g5/cia5rhero000yhkjxivrhnj8f/cia5rhero000zhkjxvmd4t8h6", + "https://example.com/cia5rhero0010hkjxsx3ke38t/cia5rhero0011hkjx8xnw9tqf/cia5rhero0012hkjxl0lf3iup/cia5rhero0013hkjxbvhj1jj9", + "https://example.com/cia5rhero0014hkjx4fmpoeu0/cia5rhero0015hkjxwpptyghv/cia5rhero0016hkjxva5k7lbw/cia5rhero0017hkjx138e6bmj", + "https://example.com/cia5rhero0018hkjxx7j78vei/cia5rhero0019hkjx6xrih33b/cia5rhero001ahkjxd4zgngrg/cia5rhero001bhkjx1tloxb1q", + "https://example.com/cia5rhero001chkjxrt8zdtde/cia5rhero001dhkjxm9mbqci7/cia5rhero001ehkjxowk5e4q6/cia5rhero001fhkjxh6qmsl1v", + "https://example.com/cia5rhero001ghkjxfadnveq2/cia5rhero001hhkjx4edkaq54/cia5rhero001ihkjx9fqoqxga/cia5rhero001jhkjx58vqa1zr", + "https://example.com/cia5rhero001khkjxdrmypltg/cia5rhero001lhkjxhqdzis67/cia5rhero001mhkjxcggzuqdx/cia5rhero001nhkjxctfvi81n", + "https://example.com/cia5rhero001ohkjxe8pk4cgt/cia5rhero001phkjx6sm4qi4x/cia5rhero001qhkjxm9lqhc6e/cia5rhero001rhkjxy9k6yacj", + "https://example.com/cia5rhero001shkjx0zfh0f80/cia5rhero001thkjx06fdx9h9/cia5rherp001uhkjxqdmh6jtq/cia5rherp001vhkjxywng3c85", + "https://example.com/cia5rherp001whkjxm2qihavv/cia5rherp001xhkjxw9v508hp/cia5rherp001yhkjx3jtn1gc8/cia5rherp001zhkjxsmvesh8h", + "https://example.com/cia5rherp0020hkjxz8yaovlg/cia5rherp0021hkjx66mvfalu/cia5rherp0022hkjxqpilypyo/cia5rherp0023hkjxsp9txbui", + "https://example.com/cia5rherp0024hkjxcc9ni9ai/cia5rherp0025hkjxslerqaui/cia5rherp0026hkjxe0jl3k05/cia5rherp0027hkjxr0hf5tjn", + "https://example.com/cia5rherp0028hkjxsg9l2j7w/cia5rherp0029hkjx8zs585su/cia5rherp002ahkjxuwrjr76m/cia5rherp002bhkjxh5nsz7nf", + "https://example.com/cia5rherp002chkjxzph0v3u0/cia5rherp002dhkjxdayhh6lg/cia5rherp002ehkjxrkvcxz41/cia5rherp002fhkjxlqa9ul9r", + "https://example.com/cia5rherp002ghkjxf6egbyrh/cia5rherp002hhkjxaw3isyif/cia5rherp002ihkjxboruj7fc/cia5rherp002jhkjx6fgq322u", + "https://example.com/cia5rherp002khkjx5adgzcsd/cia5rherp002lhkjxo84dwluf/cia5rherp002mhkjxijwq9esb/cia5rherp002nhkjx59ky0n1a", + "https://example.com/cia5rherp002ohkjx8r9ldy53/cia5rherp002phkjxn5vwv7yi/cia5rherp002qhkjxdsg26179/cia5rherp002rhkjxxvufen34", + "https://example.com/cia5rherp002shkjxj17ade31/cia5rherp002thkjx37xtqdld/cia5rherp002uhkjxbl9a3b96/cia5rherp002vhkjxrq82quxi", + "https://example.com/cia5rherp002whkjxj0kiekos/cia5rherp002xhkjxltksjia2/cia5rherp002yhkjx9f6j16y4/cia5rherp002zhkjxub535mh5", + "https://example.com/cia5rherp0030hkjxsd01uqhn/cia5rherp0031hkjxxy7v0enj/cia5rherp0032hkjxm7afimyn/cia5rherp0033hkjxu9rm6wnw", + "https://example.com/cia5rherp0034hkjxoe7lub90/cia5rherp0035hkjxdqfkfwe5/cia5rherp0036hkjx7bydg1o0/cia5rherp0037hkjxwiayjj4h", + "https://example.com/cia5rherp0038hkjxrxyltlgt/cia5rherp0039hkjxq8m8gzd9/cia5rherp003ahkjxfzlwk84z/cia5rherp003bhkjxhzsd9psq", + "https://example.com/cia5rherp003chkjx8k4d0ev6/cia5rherp003dhkjxe5rhd9bk/cia5rherp003ehkjxhdb9pe1z/cia5rherp003fhkjxnwfpch8g", + "https://example.com/cia5rherp003ghkjxv07j52h2/cia5rherp003hhkjx1i0x37cm/cia5rherp003ihkjxey8otr6v/cia5rherp003jhkjxk1np7zo5", + "https://example.com/cia5rherp003khkjxscsb30qa/cia5rherp003lhkjxcuap7ls3/cia5rherq003mhkjxsdoqgpxu/cia5rherq003nhkjxz21dqdpc", + "https://example.com/cia5rherq003ohkjx66ms0nn0/cia5rherq003phkjxb8hblxdd/cia5rherq003qhkjxmr2bkt0b/cia5rherq003rhkjxkla58f3d", + "https://example.com/cia5rherr003shkjxutmljflc/cia5rherr003thkjx9sglm092/cia5rherr003uhkjx6ttae5q1/cia5rherr003vhkjx2i22wbci", + "https://example.com/cia5rherr003whkjx0gmqk0qu/cia5rherr003xhkjxnnopsaqa/cia5rherr003yhkjxhicji4pa/cia5rherr003zhkjxg7dm7usj", + "https://example.com/cia5rherr0040hkjxqymjv2aj/cia5rherr0041hkjxgjublrsc/cia5rherr0042hkjxu34zz1y2/cia5rherr0043hkjx0v1t6s9s", + "https://example.com/cia5rherr0044hkjxourvl7pc/cia5rherr0045hkjxem1bv16o/cia5rherr0046hkjxlza5reyz/cia5rherr0047hkjxc0hqmpn8", + "https://example.com/cia5rherr0048hkjxab7n8nos/cia5rherr0049hkjxjwltgxvq/cia5rherr004ahkjxpskjnyvt/cia5rherr004bhkjx9t1zafr8", + "https://example.com/cia5rherr004chkjxos007jlr/cia5rherr004dhkjxside72xk/cia5rherr004ehkjxmsb8r01v/cia5rherr004fhkjx1c1v6ypg", + "https://example.com/cia5rherr004ghkjxl9hin1sd/cia5rherr004hhkjx1ktxr1zz/cia5rherr004ihkjxjnq6wq6b/cia5rherr004jhkjx6z9ffji0", + "https://example.com/cia5rherr004khkjxxu7r4mzw/cia5rherr004lhkjxj9vru5i2/cia5rherr004mhkjxowwrqrzg/cia5rherr004nhkjxejliyhz4", + "https://example.com/cia5rherr004ohkjx23jh1sos/cia5rherr004phkjxrq8hqfpy/cia5rherr004qhkjx4232owb6/cia5rherr004rhkjx6dbg83qw", + "https://example.com/cia5rherr004shkjx6t43ggcp/cia5rherr004thkjxmf2k2w6n/cia5rherr004uhkjxwhhmhvx5/cia5rherr004vhkjxt4qahjbz", + "https://example.com/cia5rherr004whkjx0bkl7a31/cia5rherr004xhkjxk34mm030/cia5rherr004yhkjxoundtp7u/cia5rherr004zhkjxr1htazrx", + "https://example.com/cia5rherr0050hkjxfm9e7rcy/cia5rherr0051hkjxvp7y4api/cia5rherr0052hkjxdairex18/cia5rherr0053hkjxoqt8of5n", + "https://example.com/cia5rherr0054hkjxbg8to9br/cia5rherr0055hkjxymwr54ie/cia5rherr0056hkjxez64qj8r/cia5rherr0057hkjxann7kwhj", + "https://example.com/cia5rherr0058hkjxov69tz76/cia5rherr0059hkjxxh6rrvs8/cia5rherr005ahkjxwsmsljoc/cia5rherr005bhkjxi5o005me", + "https://example.com/cia5rherr005chkjxu6w2wby8/cia5rherr005dhkjxvcmga3pp/cia5rherr005ehkjxcr90eaeq/cia5rherr005fhkjxon6fcg5h", + "https://example.com/cia5rherr005ghkjxqo4vucke/cia5rherr005hhkjxxvdv1j8f/cia5rherr005ihkjxo2vl23qb/cia5rherr005jhkjx3tbu637k", + "https://example.com/cia5rherr005khkjx8fnhdy4n/cia5rherr005lhkjxvgvc27dd/cia5rherr005mhkjxydbxzvrw/cia5rherr005nhkjx0ev5rnx0", + "https://example.com/cia5rherr005ohkjxymk6ffpy/cia5rherr005phkjx79eg202o/cia5rherr005qhkjxloql9sm9/cia5rherr005rhkjxf0fcwa88", + "https://example.com/cia5rherr005shkjxkclvai2n/cia5rherr005thkjx6lhii1h5/cia5rherr005uhkjx21imwf9f/cia5rherr005vhkjxct3akix6", + "https://example.com/cia5rherr005whkjxam3tkrf9/cia5rherr005xhkjx3ooh8d77/cia5rherr005yhkjx3ih1052q/cia5rherr005zhkjxjn08ve87", + "https://example.com/cia5rherr0060hkjxbovht1zk/cia5rherr0061hkjx3mq7yzke/cia5rherr0062hkjxc3jyhls3/cia5rherr0063hkjxw5y2kgtl", + "https://example.com/cia5rherr0064hkjxlzchg2lf/cia5rherr0065hkjxjf4qz7h7/cia5rherr0066hkjxhzxfzly2/cia5rherr0067hkjxxpsm77n2", + "https://example.com/cia5rherr0068hkjxqakagjms/cia5rherr0069hkjxahy5cahd/cia5rherr006ahkjxlx3eo0d8/cia5rherr006bhkjxqhufagp0", + "https://example.com/cia5rherr006chkjxs6l4reuh/cia5rherr006dhkjx22dhomat/cia5rherr006ehkjxv8bhjw6s/cia5rherr006fhkjxiembar5h", + "https://example.com/cia5rherr006ghkjx7uk2ahcv/cia5rherr006hhkjxrptkdzpc/cia5rherr006ihkjxoelimxey/cia5rherr006jhkjxh00sc9q0", + "https://example.com/cia5rherr006khkjx7bjaq4p8/cia5rherr006lhkjx0t17bwqu/cia5rherr006mhkjx63m6nx0x/cia5rherr006nhkjx4aylco20", + "https://example.com/cia5rherr006ohkjxqncx2bfn/cia5rherr006phkjxcs9lhj4k/cia5rherr006qhkjxltiaiox0/cia5rherr006rhkjxpxhk6ceh", + "https://example.com/cia5rherr006shkjxkjpkluh5/cia5rherr006thkjxksjhi0rp/cia5rherr006uhkjxuluxeq56/cia5rherr006vhkjx2i5mkp5o", + "https://example.com/cia5rherr006whkjx94ph66s7/cia5rherr006xhkjx963ey6z0/cia5rherr006yhkjxklr3zey7/cia5rherr006zhkjxhi9ckzcj", + "https://example.com/cia5rherr0070hkjx8qu91dki/cia5rherr0071hkjxnsj72h7a/cia5rherr0072hkjxpcch22pw/cia5rherr0073hkjxr88cl6td", + "https://example.com/cia5rherr0074hkjx33v1pc9p/cia5rherr0075hkjxbijzax59/cia5rherr0076hkjxdatwas40/cia5rherr0077hkjxp92fa2bh", + "https://example.com/cia5rherr0078hkjxids7sgwp/cia5rherr0079hkjxptr6nbvk/cia5rherr007ahkjx51bynaee/cia5rherr007bhkjx2f0oimkg", + "https://example.com/cia5rherr007chkjxj243t6fq/cia5rherr007dhkjxx0z54s2z/cia5rherr007ehkjxvtk1saqr/cia5rherr007fhkjxc99ot3di", + "https://example.com/cia5rherr007ghkjxcdyrgbe5/cia5rherr007hhkjx4g2zda93/cia5rherr007ihkjx3dhwfh5w/cia5rherr007jhkjxnucjju1k", + "https://example.com/cia5rherr007khkjxtucv4xam/cia5rherr007lhkjxzb52dzam/cia5rherr007mhkjxe5ytefsg/cia5rherr007nhkjxhn1htqim", + "https://example.com/cia5rherr007ohkjxyhvqunje/cia5rherr007phkjxg1rmr8ia/cia5rherr007qhkjxvne6tsg2/cia5rherr007rhkjx92plnv33", + "https://example.com/cia5rherr007shkjxedxbud9y/cia5rherr007thkjxsfcv846z/cia5rherr007uhkjxrtoyvp1k/cia5rherr007vhkjxwkkvb63p", + "https://example.com/cia5rherr007whkjxhom33u72/cia5rherr007xhkjx00i52itn/cia5rherr007yhkjxlbwyrfbh/cia5rherr007zhkjx7hqxgotj", + "https://example.com/cia5rherr0080hkjxm6fr03t9/cia5rherr0081hkjxsskkldqg/cia5rherr0082hkjx6agt52v9/cia5rherr0083hkjxmw1qpljq", + "https://example.com/cia5rherr0084hkjx66lbczid/cia5rherr0085hkjx1yhf4qqz/cia5rherr0086hkjx156ah7x1/cia5rhers0087hkjxf7zmqrz3", + "https://example.com/cia5rhers0088hkjxlvoy1vtq/cia5rhers0089hkjxwopzmrl8/cia5rhers008ahkjxa76skipt/cia5rhers008bhkjxchpeggii", + "https://example.com/cia5rhers008chkjx77jxzrj5/cia5rhers008dhkjxwdtj4fxt/cia5rhers008ehkjxyk1rbqs4/cia5rhers008fhkjxi6c9h7on", + "https://example.com/cia5rhers008ghkjxcjbq4qio/cia5rhers008hhkjx46uf4z78/cia5rhers008ihkjxiwjs7rs8/cia5rhers008jhkjx6gow2oc8", + "https://example.com/cia5rhers008khkjxam5doybe/cia5rhers008lhkjx33h8n79o/cia5rhers008mhkjxch0uhhqa/cia5rhers008nhkjxvkh7mio2", + "https://example.com/cia5rhers008ohkjxfixhe4vq/cia5rhers008phkjxrw2g2zi3/cia5rhers008qhkjxoynia1m2/cia5rhers008rhkjxoh1yvlac", + "https://example.com/cia5rhers008shkjxewzwesdt/cia5rhers008thkjx2yx5o3rs/cia5rhers008uhkjxm75q9exl/cia5rhers008vhkjx4jogh9q0", + "https://example.com/cia5rhers008whkjxdo6vglei/cia5rhers008xhkjx0bx8rrph/cia5rhers008yhkjx7sotv4z4/cia5rhers008zhkjxid6dq8hl", + "https://example.com/cia5rhers0090hkjxpmid4zyj/cia5rhers0091hkjx7swur7ci/cia5rhers0092hkjxd4mc5j7l/cia5rhers0093hkjxri89p7jy", + "https://example.com/cia5rhers0094hkjxirrewas9/cia5rhers0095hkjx1wkqq1g1/cia5rhers0096hkjxvpc3hp0t/cia5rhers0097hkjx1ugphwhs", + "https://example.com/cia5rhers0098hkjxxptvggpn/cia5rhers0099hkjx0hgy6tpl/cia5rhers009ahkjxrgk4cwx1/cia5rhers009bhkjxhff7ocag", + "https://example.com/cia5rhers009chkjxt2zanh56/cia5rhers009dhkjxeh17c4wg/cia5rhers009ehkjxkhdljhm4/cia5rhers009fhkjxa78k166d", + "https://example.com/cia5rhers009ghkjxblshruiu/cia5rhers009hhkjx6bpkys70/cia5rhers009ihkjx097xi5jr/cia5rhers009jhkjxl1v2ym6h", + "https://example.com/cia5rhers009khkjxvciqe48d/cia5rhers009lhkjxxgfzrzqn/cia5rhers009mhkjx99k29erp/cia5rhers009nhkjxnmdineg9", + "https://example.com/cia5rhers009ohkjx0rr58cbk/cia5rhers009phkjxo0wwxbtm/cia5rhers009qhkjx8j0n0ta4/cia5rhers009rhkjx8jok0mo9", + "https://example.com/cia5rhers009shkjxo1trhdsu/cia5rhers009thkjxtukv1mwy/cia5rhers009uhkjx0e82f0we/cia5rhers009vhkjx8j0ysbxu", + "https://example.com/cia5rhers009whkjxdl5recmi/cia5rhers009xhkjx801pcdzp/cia5rhers009yhkjx2wnyv52l/cia5rhers009zhkjxdmjnipn0", + "https://example.com/cia5rhers00a0hkjxj46kgklk/cia5rhers00a1hkjxmaxskno1/cia5rhers00a2hkjxrv6jtia4/cia5rhers00a3hkjx524js0bd", + "https://example.com/cia5rhers00a4hkjx6y55uml1/cia5rhers00a5hkjxahhtmal1/cia5rhers00a6hkjx0pt7rbot/cia5rhers00a7hkjxtboirxmu", + "https://example.com/cia5rhers00a8hkjxqmrjjens/cia5rhers00a9hkjxxbmuxaqn/cia5rhers00aahkjx2fdz4j2q/cia5rhers00abhkjxflwb8nml", + "https://example.com/cia5rhers00achkjxc7n5xjqb/cia5rhers00adhkjx98mur9hy/cia5rhers00aehkjxrmv4miio/cia5rhers00afhkjxm1wmuk24", + "https://example.com/cia5rhers00aghkjxhh9nyajy/cia5rhers00ahhkjxypuj93ni/cia5rhers00aihkjxqhvpkhvu/cia5rhers00ajhkjxcebhpi2g", + "https://example.com/cia5rhers00akhkjxpq2z92bg/cia5rhers00alhkjx6ztkugxa/cia5rhers00amhkjxys8qr2ae/cia5rhers00anhkjxugpiwy4o", + "https://example.com/cia5rhers00aohkjx5lptk7ll/cia5rhert00aphkjx3m0tdlo6/cia5rhert00aqhkjxtun34qda/cia5rhert00arhkjxgn2cukgo", + "https://example.com/cia5rhert00ashkjxgndi0t1w/cia5rhert00athkjxngdtw2ac/cia5rhert00auhkjxtlqdm9tk/cia5rhert00avhkjxpmv39lb6", + "https://example.com/cia5rhert00awhkjxsm1i5606/cia5rhert00axhkjxiini4u7n/cia5rhert00ayhkjxawcdih8y/cia5rhert00azhkjx2uwcskyo", + "https://example.com/cia5rhert00b0hkjxjosiu610/cia5rhert00b1hkjx0inj6sis/cia5rhert00b2hkjx687j5ca5/cia5rhert00b3hkjxaaepyb37", + "https://example.com/cia5rhert00b4hkjxccpt8awt/cia5rhert00b5hkjxb4kuj419/cia5rhert00b6hkjxq8bi4zga/cia5rhert00b7hkjxrgomql9g", + "https://example.com/cia5rhert00b8hkjxjxy3fhb0/cia5rhert00b9hkjxftkoktwh/cia5rhert00bahkjxjx5o3cnn/cia5rhert00bbhkjxa6frtobg", + "https://example.com/cia5rhert00bchkjxe5tkaifo/cia5rhert00bdhkjx5jppppvo/cia5rhert00behkjxdmsqdq9p/cia5rhert00bfhkjxnw4tk86n", + "https://example.com/cia5rhert00bghkjx49wz4l5r/cia5rhert00bhhkjx2mfza2xe/cia5rhert00bihkjxbsd9ovfr/cia5rhert00bjhkjx3vye5ep3", + "https://example.com/cia5rhert00bkhkjxwtdsj589/cia5rhert00blhkjxmjchgsmf/cia5rhert00bmhkjxm5fxzqpq/cia5rhert00bnhkjxyf65cmel", + "https://example.com/cia5rhert00bohkjx7c22ja5m/cia5rhert00bphkjxhe6e895b/cia5rhert00bqhkjxj6kge909/cia5rhert00brhkjx65rkq4ma", + "https://example.com/cia5rhert00bshkjx1cn2s8w0/cia5rhert00bthkjxlbbqak7z/cia5rhert00buhkjxfy9yreib/cia5rhert00bvhkjxaivuz0fw", + "https://example.com/cia5rhert00bwhkjxinogvtl2/cia5rhert00bxhkjxw62nijla/cia5rhert00byhkjxl5nlc2eo/cia5rhert00bzhkjxhlmdfgi7", + "https://example.com/cia5rhert00c0hkjx4n6b40nr/cia5rhert00c1hkjxsnmkujr5/cia5rhert00c2hkjxv6w0h717/cia5rhert00c3hkjxrcvb5qs6", + "https://example.com/cia5rhert00c4hkjxgd95z86n/cia5rhert00c5hkjxv1tvmf9c/cia5rhert00c6hkjx4cv3ix8y/cia5rhert00c7hkjxli2to7w7", + "https://example.com/cia5rhert00c8hkjxgaiuvkul/cia5rhert00c9hkjxkcpdyj68/cia5rhert00cahkjxjxrfjfl0/cia5rhert00cbhkjxk64v5mhq", + "https://example.com/cia5rhert00cchkjxzkjnwkn1/cia5rhert00cdhkjx1cdlk3ms/cia5rhert00cehkjxaw4lnp43/cia5rhert00cfhkjxz4hk5gsw", + "https://example.com/cia5rhert00cghkjxpil3za50/cia5rhert00chhkjxp7t5jz46/cia5rhert00cihkjx5cwk8th0/cia5rhert00cjhkjxaxp4y2o0", + "https://example.com/cia5rhert00ckhkjxcylxiiem/cia5rhert00clhkjxzkx50uda/cia5rhert00cmhkjxs5uv5t4e/cia5rhert00cnhkjxxz9rplpv", + "https://example.com/cia5rhert00cohkjxa1r51z2k/cia5rhert00cphkjx1zw5bwvd/cia5rhert00cqhkjxkmcxldzz/cia5rhert00crhkjx8xm9oeny", + "https://example.com/cia5rhert00cshkjxuoqaoftt/cia5rhert00cthkjx1x01dmec/cia5rhert00cuhkjx9sixehxp/cia5rhert00cvhkjxomyssiza", + "https://example.com/cia5rhert00cwhkjx7et34wux/cia5rhert00cxhkjxncva74m3/cia5rhert00cyhkjxfkizvrik/cia5rhert00czhkjxgbzmphlh", + "https://example.com/cia5rhert00d0hkjx5amei299/cia5rhert00d1hkjxumygde64/cia5rhert00d2hkjxpnuisg9m/cia5rhert00d3hkjxakoawt1r", + "https://example.com/cia5rhert00d4hkjx1h1sd1xk/cia5rhert00d5hkjxiszqt7gt/cia5rhert00d6hkjxrgly5qmw/cia5rhert00d7hkjx0khb7is3", + "https://example.com/cia5rhert00d8hkjx8q3rh7mm/cia5rhert00d9hkjxsc3r3d6p/cia5rhert00dahkjxi4ka0kza/cia5rhert00dbhkjxzvnrcsv0", + "https://example.com/cia5rhert00dchkjxqlqg4rat/cia5rhert00ddhkjxtjjv0jz2/cia5rhert00dehkjxf7uayrst/cia5rhert00dfhkjxzjqnclyl", + "https://example.com/cia5rhert00dghkjxdq5ojgf7/cia5rhert00dhhkjxs3fjff88/cia5rheru00dihkjxo2axpgvn/cia5rheru00djhkjx8lrs9xha", + "https://example.com/cia5rheru00dkhkjx45m33mct/cia5rheru00dlhkjxzrjk6adw/cia5rheru00dmhkjxhhgorfhb/cia5rheru00dnhkjxwadtejpl", + "https://example.com/cia5rheru00dohkjxj8rdgozp/cia5rheru00dphkjxsqktwr0d/cia5rheru00dqhkjxlodva5ul/cia5rheru00drhkjxy79ve13k", + "https://example.com/cia5rheru00dshkjxwawhyuhd/cia5rheru00dthkjxovwtz7zd/cia5rheru00duhkjxnhebnm70/cia5rheru00dvhkjxt28pubcm", + "https://example.com/cia5rheru00dwhkjxasdto0h0/cia5rheru00dxhkjxvoxohbir/cia5rheru00dyhkjxr50yvvwt/cia5rheru00dzhkjxiffhdptt", + "https://example.com/cia5rheru00e0hkjx9htn1puz/cia5rheru00e1hkjxqvm483lo/cia5rheru00e2hkjxbr3v8ysf/cia5rheru00e3hkjx1xxys4u4", + "https://example.com/cia5rheru00e4hkjx0pdbwiqd/cia5rheru00e5hkjxcjyvelv6/cia5rheru00e6hkjxfar4jska/cia5rheru00e7hkjxfkw7rkrq", + "https://example.com/cia5rheru00e8hkjx1rtfronr/cia5rheru00e9hkjxpgfyuqxd/cia5rheru00eahkjx9hw9j0h3/cia5rheru00ebhkjx83acges9", + "https://example.com/cia5rheru00echkjxbrhqzezc/cia5rheru00edhkjxu3iycpdz/cia5rheru00eehkjxs91i2o4s/cia5rheru00efhkjxgsawfxoa", + "https://example.com/cia5rheru00eghkjxdhzfetw8/cia5rheru00ehhkjx1wnw1va3/cia5rheru00eihkjxo0vft0jh/cia5rheru00ejhkjxa8vgdyzn", + "https://example.com/cia5rheru00ekhkjxfo36jiae/cia5rheru00elhkjxim0qkqcu/cia5rheru00emhkjxyv3cm8s8/cia5rheru00enhkjx4zsm4g1k", + "https://example.com/cia5rherv00eohkjxl16e03j1/cia5rherv00ephkjxtfqgb09e/cia5rherv00eqhkjxxnuib8vx/cia5rherv00erhkjx3c2eh7rw", + "https://example.com/cia5rherv00eshkjxk2ugn862/cia5rherv00ethkjx4a4i5bu4/cia5rherv00euhkjx4x1ll1hu/cia5rherv00evhkjx4a90801w", + "https://example.com/cia5rherv00ewhkjx16ofuvju/cia5rherv00exhkjxbjicg8ee/cia5rherv00eyhkjxi37ly2xd/cia5rherv00ezhkjxme2gnc8n", + "https://example.com/cia5rherv00f0hkjxupxvqy6f/cia5rherv00f1hkjxif6y2ebt/cia5rherv00f2hkjx2hxktwi1/cia5rherv00f3hkjxboq2udk4", + "https://example.com/cia5rherv00f4hkjxe5t1s9vm/cia5rherv00f5hkjx2p1bkk8q/cia5rherv00f6hkjx3ugeare1/cia5rherv00f7hkjxw5xpdje2", + "https://example.com/cia5rherv00f8hkjxr347e4en/cia5rherv00f9hkjxjmjvxubi/cia5rherv00fahkjxhgdld04j/cia5rherv00fbhkjxury8qhpe", + "https://example.com/cia5rherv00fchkjx36kklfip/cia5rherv00fdhkjx51q4kxdz/cia5rherv00fehkjxm1xs1n6p/cia5rherv00ffhkjxzrms1ho0", + "https://example.com/cia5rherv00fghkjxq54a40bh/cia5rherv00fhhkjxfjd33rlp/cia5rherv00fihkjx8ydqzwn4/cia5rherv00fjhkjx800mxgbz", + "https://example.com/cia5rherv00fkhkjxr5ktedsn/cia5rherv00flhkjxkh6ip88y/cia5rherv00fmhkjxx824akqe/cia5rherv00fnhkjxn8b9z0y4", + "https://example.com/cia5rherv00fohkjximtuc4oo/cia5rherv00fphkjxo8xnj3a1/cia5rherv00fqhkjx3frdvz0l/cia5rherv00frhkjxnt4ip06p", + "https://example.com/cia5rherv00fshkjxebbqlsj4/cia5rherv00fthkjxhejhs2en/cia5rherv00fuhkjxww4v962t/cia5rherv00fvhkjx5n0tpk2r", + "https://example.com/cia5rherv00fwhkjxomhb4741/cia5rherv00fxhkjxkgs095w0/cia5rherv00fyhkjxtve4rfnk/cia5rherv00fzhkjxkl9m13ke", + "https://example.com/cia5rherv00g0hkjxwfqm1g52/cia5rherv00g1hkjx0sajw56m/cia5rherv00g2hkjxcs30a0m9/cia5rherv00g3hkjxuq2edyvx", + "https://example.com/cia5rherv00g4hkjxpmffchhu/cia5rherv00g5hkjxkhh7fc92/cia5rherv00g6hkjxsak4bwp4/cia5rherv00g7hkjxemsqgdmx", + "https://example.com/cia5rherv00g8hkjxxd4iaucx/cia5rherv00g9hkjx82yt525c/cia5rherv00gahkjx9higv8wz/cia5rherv00gbhkjx0bxq8ejn", + "https://example.com/cia5rherv00gchkjxolbgqpvx/cia5rherv00gdhkjxh1q1mlra/cia5rherv00gehkjx6rmpv7a6/cia5rherv00gfhkjx7v471pja", + "https://example.com/cia5rherv00gghkjx4swi554w/cia5rherv00ghhkjxrbcdqi5x/cia5rherv00gihkjxrzkje2z3/cia5rherv00gjhkjx23xefzn2", + "https://example.com/cia5rherv00gkhkjx7om372u4/cia5rherv00glhkjx7t4qbrj9/cia5rherv00gmhkjxm4021iiz/cia5rherv00gnhkjxgxmosagw", + "https://example.com/cia5rherv00gohkjx92hhv2g9/cia5rherv00gphkjx7bf3eo7t/cia5rherv00gqhkjx6wj13cjb/cia5rherv00grhkjx9xdd6xqg", + "https://example.com/cia5rherv00gshkjxapw92pmq/cia5rherv00gthkjx7uqh0m79/cia5rherv00guhkjx9uxnghi5/cia5rherv00gvhkjx9dk24e4x", + "https://example.com/cia5rherv00gwhkjxm8spj2z7/cia5rherv00gxhkjx1bzjkoj5/cia5rherv00gyhkjxj4n460vb/cia5rherv00gzhkjx6824pani", + "https://example.com/cia5rherv00h0hkjx8wfm03s5/cia5rherv00h1hkjxboapbcoy/cia5rherv00h2hkjxqgzz6g4s/cia5rherv00h3hkjxcbpuhpk2", + "https://example.com/cia5rherv00h4hkjx0eho0z4b/cia5rherv00h5hkjx1z59ioaa/cia5rherv00h6hkjxzqmxtasv/cia5rherv00h7hkjxflt86e5f", + "https://example.com/cia5rherv00h8hkjxxlr7008t/cia5rherv00h9hkjxz8kctoc9/cia5rherv00hahkjxq60p0kx0/cia5rherv00hbhkjxd0zud58a", + "https://example.com/cia5rherv00hchkjx4o8rphb9/cia5rherv00hdhkjxs3n7yko7/cia5rherv00hehkjxnxwntfg5/cia5rherv00hfhkjxb39xgejv", + "https://example.com/cia5rherv00hghkjxb1yco8y1/cia5rherv00hhhkjx2v6pz42w/cia5rherv00hihkjx400aow1n/cia5rherv00hjhkjxtwik0d1s", + "https://example.com/cia5rherv00hkhkjxwnaoc3oc/cia5rherv00hlhkjxsmntvjhm/cia5rherv00hmhkjx4ag0j8cq/cia5rherv00hnhkjxbiigrngm", + "https://example.com/cia5rherv00hohkjx2s4opprd/cia5rherv00hphkjx8pkc39p2/cia5rherv00hqhkjxif9qfvct/cia5rherv00hrhkjxe4vtvd6z", + "https://example.com/cia5rherv00hshkjxampz8fwm/cia5rherv00hthkjxf48ybrar/cia5rherv00huhkjx3asrrys5/cia5rherv00hvhkjxc570882r", + "https://example.com/cia5rherv00hwhkjxfooadgv0/cia5rherv00hxhkjxnmtttd03/cia5rherv00hyhkjx0deqal5b/cia5rherv00hzhkjxleqe94v1", + "https://example.com/cia5rherv00i0hkjxs3ps41wq/cia5rherv00i1hkjxw46qft2r/cia5rherv00i2hkjxfmfjpssy/cia5rherv00i3hkjxayjpzx43", + "https://example.com/cia5rherv00i4hkjxpxotlyhp/cia5rherv00i5hkjxhz77s1t6/cia5rherv00i6hkjxwgw9wpq7/cia5rherv00i7hkjxy724la5w", + "https://example.com/cia5rherv00i8hkjx6wko56vc/cia5rherv00i9hkjxiqs85dqr/cia5rherv00iahkjx32pvc8lh/cia5rherv00ibhkjxouu7jcs7", + "https://example.com/cia5rherv00ichkjxub141io8/cia5rherv00idhkjx4xp7yqzz/cia5rherv00iehkjx4l9xkah3/cia5rherv00ifhkjx3qco4ypm", + "https://example.com/cia5rherv00ighkjx9sp14gne/cia5rherv00ihhkjx713r0569/cia5rherv00iihkjxorq3d5yp/cia5rherv00ijhkjxtdo94p17", + "https://example.com/cia5rherv00ikhkjx4j5jbzre/cia5rherv00ilhkjxcgsn6xlu/cia5rherv00imhkjxx8la8kf5/cia5rherv00inhkjx3hz5opiw", + "https://example.com/cia5rherv00iohkjx3dgmyl4e/cia5rherv00iphkjx8glt32u3/cia5rherv00iqhkjx6cb6wlxs/cia5rherv00irhkjxza76xuqt", + "https://example.com/cia5rherv00ishkjxo51i49wr/cia5rherv00ithkjxxa546aho/cia5rherv00iuhkjx50q0dsyb/cia5rherv00ivhkjxcq50j6b5", + "https://example.com/cia5rherv00iwhkjxaklfq2jh/cia5rherv00ixhkjxq47wm5d8/cia5rherv00iyhkjx93fsh773/cia5rherv00izhkjx05qw5ls6", + "https://example.com/cia5rherv00j0hkjxcpk80dqg/cia5rherv00j1hkjxe5fze7ph/cia5rherv00j2hkjx0b4x6jj2/cia5rherv00j3hkjxslnfjk8f", + "https://example.com/cia5rherv00j4hkjxn02a7r4w/cia5rherv00j5hkjxqbq2abzk/cia5rherv00j6hkjx046gf9gh/cia5rherv00j7hkjxw8y6aybl", + "https://example.com/cia5rherv00j8hkjxua9fh4f2/cia5rherv00j9hkjx0n7j7tf0/cia5rherv00jahkjx1h8p0yzt/cia5rherv00jbhkjx6isout34", + "https://example.com/cia5rherv00jchkjxscxpp7mk/cia5rherv00jdhkjxx7314ajy/cia5rherv00jehkjx2wej8wjh/cia5rherv00jfhkjxs4i5eiho", + "https://example.com/cia5rherv00jghkjxcrkt8o3t/cia5rherv00jhhkjx2u0sm2r0/cia5rherv00jihkjxtpjif9k6/cia5rherv00jjhkjxaagcsohi", + "https://example.com/cia5rherv00jkhkjx8gq5x6ov/cia5rherv00jlhkjxev9zq161/cia5rherv00jmhkjx2sjcp2px/cia5rherv00jnhkjxn8jpfipq", + "https://example.com/cia5rherv00johkjxbx8744i5/cia5rherv00jphkjxi2eza54a/cia5rherv00jqhkjx1q5y2n4p/cia5rherv00jrhkjxvl0soujd", + "https://example.com/cia5rherv00jshkjxs7ziy4vs/cia5rherv00jthkjx3ewtvj26/cia5rherv00juhkjxkwhid4b7/cia5rherv00jvhkjxx7nurgrx", + "https://example.com/cia5rherv00jwhkjxxlcfjhj3/cia5rherv00jxhkjxoresrx2f/cia5rherv00jyhkjxtsmt6apo/cia5rherv00jzhkjxghybzsqs", + "https://example.com/cia5rherv00k0hkjx7o0bc0o5/cia5rherv00k1hkjxmzwu9w7x/cia5rherv00k2hkjxw6z2g76g/cia5rherv00k3hkjx8htly8zt", + "https://example.com/cia5rherv00k4hkjx59yd941i/cia5rherv00k5hkjx0j5ccesw/cia5rherv00k6hkjxduxlqh65/cia5rherv00k7hkjx5et0n6t0", + "https://example.com/cia5rherv00k8hkjxj72kcc3p/cia5rherv00k9hkjx0zy8vr32/cia5rherv00kahkjx6kmi5wtf/cia5rherv00kbhkjxh2ut5f4w", + "https://example.com/cia5rherv00kchkjxq9ki0lgr/cia5rherv00kdhkjx8mafuept/cia5rherv00kehkjx3kffqdj5/cia5rherv00kfhkjxk30tt2a0", + "https://example.com/cia5rherv00kghkjxaw98qyx2/cia5rherv00khhkjxieg0bycx/cia5rherw00kihkjxdmdkp4x9/cia5rherw00kjhkjxijtzapeq", + "https://example.com/cia5rherw00kkhkjxh1oh0kl4/cia5rherw00klhkjx6ffidsax/cia5rherw00kmhkjxqtgyxcy0/cia5rherw00knhkjx2xyipie3", + "https://example.com/cia5rherw00kohkjxdq8vr7d0/cia5rherw00kphkjxes36xbks/cia5rherw00kqhkjx4ektym79/cia5rherw00krhkjxib4btsp0", + "https://example.com/cia5rherw00kshkjxpc4m0m9e/cia5rherw00kthkjxgdi6nnzj/cia5rherw00kuhkjxcwj7gk86/cia5rherw00kvhkjx7cj949ld", + "https://example.com/cia5rherw00kwhkjxmwj8brl1/cia5rherw00kxhkjxae8yeb7p/cia5rherw00kyhkjxqze289bb/cia5rherw00kzhkjx2jy2h9ji", + "https://example.com/cia5rherw00l0hkjxxadfzs3n/cia5rherw00l1hkjxhngih2c7/cia5rherw00l2hkjxn5zwgxrx/cia5rherw00l3hkjxz3mp1gb9", + "https://example.com/cia5rherw00l4hkjxhcw0erdu/cia5rherw00l5hkjxj6j3l2zh/cia5rherw00l6hkjx533r5z4r/cia5rherw00l7hkjxvjjbvgo6", + "https://example.com/cia5rherw00l8hkjxx61zlkek/cia5rherw00l9hkjxb7xj73l2/cia5rherw00lahkjxrpt6fk4m/cia5rherw00lbhkjxlron4prb", + "https://example.com/cia5rherw00lchkjxtq99vutt/cia5rherw00ldhkjxon95rdss/cia5rherw00lehkjxblctcrfw/cia5rherw00lfhkjxc3mffk2x", + "https://example.com/cia5rherw00lghkjx4v9dujig/cia5rherw00lhhkjx734cxan0/cia5rherw00lihkjxlxx3bx3y/cia5rherw00ljhkjxa5te3vm6", + "https://example.com/cia5rherw00lkhkjxy4akb3lo/cia5rherw00llhkjxxg2t6ecs/cia5rherw00lmhkjx5v0ur1zi/cia5rherw00lnhkjxuw4grqgg", + "https://example.com/cia5rherw00lohkjx1tx7bq0f/cia5rherw00lphkjx118uu54q/cia5rherw00lqhkjxo63inpim/cia5rherw00lrhkjx60iprrlh", + "https://example.com/cia5rherw00lshkjxq67z9znh/cia5rherw00lthkjxknfimvv6/cia5rherw00luhkjxln059pxx/cia5rherw00lvhkjx0b5u1wvh", + "https://example.com/cia5rherw00lwhkjxdnbo7o8x/cia5rherw00lxhkjxialw39ph/cia5rherw00lyhkjxq44794oz/cia5rherw00lzhkjx063oj379", + "https://example.com/cia5rherw00m0hkjxa51hcx2d/cia5rherw00m1hkjx5udmt5pw/cia5rherw00m2hkjxh2zenkxy/cia5rherw00m3hkjxaa4b1ukf", + "https://example.com/cia5rherw00m4hkjxu0txyp2a/cia5rherw00m5hkjxku5t6nia/cia5rherw00m6hkjxpuvkxgxe/cia5rherw00m7hkjx7mgdnt4i", + "https://example.com/cia5rherw00m8hkjx9tjak8nq/cia5rherw00m9hkjx6tjbc4c1/cia5rherw00mahkjxhrlq09fa/cia5rherw00mbhkjxncngvvyl", + "https://example.com/cia5rherw00mchkjxqjijeepd/cia5rherw00mdhkjx5n3u57g1/cia5rherw00mehkjxlhfhx86m/cia5rherw00mfhkjxhtz36qq5", + "https://example.com/cia5rherw00mghkjxiknnl4br/cia5rherw00mhhkjxl1kv5f98/cia5rherw00mihkjxzo6yez34/cia5rherw00mjhkjx2ffeb3lh", + "https://example.com/cia5rherw00mkhkjxziimpsbk/cia5rherw00mlhkjxp3bcocf5/cia5rherw00mmhkjxbwztbp6k/cia5rherw00mnhkjxyc9eixhz", + "https://example.com/cia5rherw00mohkjxm8kai7n5/cia5rherw00mphkjx65gvhaf2/cia5rherw00mqhkjxfh4vu8eq/cia5rherw00mrhkjxfv2gj9bt", + "https://example.com/cia5rherw00mshkjxmnuhrdj4/cia5rherw00mthkjxjj5iizh6/cia5rherw00muhkjxiw62tfl7/cia5rherw00mvhkjxhzemezd3", + "https://example.com/cia5rherw00mwhkjxlml9rs61/cia5rherw00mxhkjxl2mbwrnp/cia5rherw00myhkjxj96ye1zv/cia5rherw00mzhkjxodju2h5o", + "https://example.com/cia5rherw00n0hkjxuivmtahu/cia5rherw00n1hkjx8gbwu61v/cia5rherw00n2hkjx1j9gzlce/cia5rherw00n3hkjxaz67uu2l", + "https://example.com/cia5rherw00n4hkjxoz2v47cp/cia5rherw00n5hkjxc2m0xq1d/cia5rherw00n6hkjxs5pej7ym/cia5rherw00n7hkjxrj28b2tt", + "https://example.com/cia5rherw00n8hkjxikm4mbwi/cia5rherw00n9hkjxcfsu9f9d/cia5rherw00nahkjxawec3u0q/cia5rherw00nbhkjx38j8uycv", + "https://example.com/cia5rherw00nchkjxpxks6dv1/cia5rherw00ndhkjxme2uh9cm/cia5rherw00nehkjxiu6s9qet/cia5rherw00nfhkjxjeycmmn7", + "https://example.com/cia5rherw00nghkjx66hdwbtg/cia5rherw00nhhkjxki3i81i4/cia5rherw00nihkjxvve944gl/cia5rherw00njhkjx3jwe46il", + "https://example.com/cia5rherw00nkhkjx8znncv54/cia5rherw00nlhkjxsyqrjo5g/cia5rherw00nmhkjxkir8br45/cia5rherw00nnhkjxexumdwa2", + "https://example.com/cia5rherw00nohkjxzs49c8vj/cia5rherw00nphkjx5eq7mllr/cia5rherw00nqhkjxxqc7wn0n/cia5rherw00nrhkjx3xnlk87f", + "https://example.com/cia5rherw00nshkjxis89wcyt/cia5rherw00nthkjxmo962d98/cia5rherw00nuhkjx5102qsyd/cia5rherw00nvhkjx4dhdkq7d", + "https://example.com/cia5rherw00nwhkjx6o2era0y/cia5rherw00nxhkjxibwv5nl3/cia5rherw00nyhkjxnobkkmmt/cia5rherw00nzhkjx2rfj3yw8", + "https://example.com/cia5rherw00o0hkjxys1f6cpt/cia5rherw00o1hkjxv7y1wk16/cia5rherw00o2hkjx72rwd3p7/cia5rherw00o3hkjxdpw2fjf4", + "https://example.com/cia5rherw00o4hkjxj0qwjpur/cia5rherw00o5hkjxr0vma4yn/cia5rherw00o6hkjxjdio6m66/cia5rherw00o7hkjx6tah5wos", + "https://example.com/cia5rherw00o8hkjxtseqnpix/cia5rherw00o9hkjxcll29qxs/cia5rherw00oahkjxy6j4514j/cia5rherw00obhkjxyt12uhto", + "https://example.com/cia5rherw00ochkjxv94azfo3/cia5rherw00odhkjxvvb9a4cf/cia5rherw00oehkjxro8j0dau/cia5rherw00ofhkjxazh1bnkv", + "https://example.com/cia5rherw00oghkjxaiqblgaz/cia5rherw00ohhkjxq0jg2ekb/cia5rherw00oihkjxhxp787hh/cia5rherw00ojhkjxwn2bygkc", + "https://example.com/cia5rherw00okhkjxrvuzh590/cia5rherw00olhkjxukov5sdm/cia5rherw00omhkjxprnz66kz/cia5rherw00onhkjxd19rb8es", + "https://example.com/cia5rherw00oohkjxdp73iojj/cia5rherw00ophkjx0ejclbnc/cia5rherw00oqhkjxzbrto5j9/cia5rherw00orhkjx64ybndrk", + "https://example.com/cia5rherw00oshkjxto8g8o9d/cia5rherw00othkjx0siti4zs/cia5rherw00ouhkjxn4k2rxwd/cia5rherw00ovhkjxlfh7kchr", + "https://example.com/cia5rherw00owhkjx5u0y9ly7/cia5rherw00oxhkjxuc2947n2/cia5rherw00oyhkjxclqlf4cz/cia5rherw00ozhkjxw4ljr3n2", + "https://example.com/cia5rherw00p0hkjxprwdryfg/cia5rherw00p1hkjxnm9mqxjr/cia5rherw00p2hkjxo97f8u6g/cia5rherw00p3hkjxu3v7vxny", + "https://example.com/cia5rherw00p4hkjx52wsz26a/cia5rherw00p5hkjxbcznr0do/cia5rherw00p6hkjxytrdjjnt/cia5rherw00p7hkjx6l5uvkab", + "https://example.com/cia5rherw00p8hkjxm2zue659/cia5rherx00p9hkjxbkzeky4l/cia5rherx00pahkjxrqp8ljqj/cia5rherx00pbhkjxuxod17kg", + "https://example.com/cia5rherx00pchkjxa26ekxyy/cia5rherx00pdhkjx0kxle2w1/cia5rherx00pehkjxxugq6n6r/cia5rherx00pfhkjx2ucx69kc", + "https://example.com/cia5rherx00pghkjxnqt22yl7/cia5rherx00phhkjxuxv53gmq/cia5rherx00pihkjxcb70hxmq/cia5rherx00pjhkjxlvjdb6xl", + "https://example.com/cia5rherx00pkhkjxjc8qkw9h/cia5rherx00plhkjxcbs6t9k7/cia5rherx00pmhkjxh6cijkuv/cia5rherx00pnhkjxpf14evbg", + "https://example.com/cia5rherx00pohkjxfgxqyo6d/cia5rherx00pphkjxtpilb91o/cia5rherx00pqhkjxny7abx9y/cia5rherx00prhkjx0l5g83bc", + "https://example.com/cia5rherx00pshkjxxuxfrdbr/cia5rherx00pthkjxxnj6u6sk/cia5rherx00puhkjxpubm3g6s/cia5rherx00pvhkjxyj7u9c6t", + "https://example.com/cia5rherx00pwhkjx6ppgibkl/cia5rherx00pxhkjxvv4p4kry/cia5rherx00pyhkjxpkbho8g0/cia5rherx00pzhkjxivjo0784", + "https://example.com/cia5rherx00q0hkjxffxtz7i5/cia5rherx00q1hkjxygfowug9/cia5rherx00q2hkjxvsau87zx/cia5rherx00q3hkjx6z1kw9b2", + "https://example.com/cia5rherx00q4hkjx4auglr08/cia5rherx00q5hkjxno848f23/cia5rherx00q6hkjxy6y8cv6y/cia5rherx00q7hkjxzzoogxhg", + "https://example.com/cia5rherx00q8hkjx70m64of5/cia5rherx00q9hkjxg5c1aukr/cia5rherx00qahkjxqn1h5a85/cia5rherx00qbhkjxg2cf36ig", + "https://example.com/cia5rherx00qchkjxlwf1o9ji/cia5rherx00qdhkjx1gdhjcsr/cia5rherx00qehkjx172b5dpn/cia5rherx00qfhkjxe3uruk25", + "https://example.com/cia5rherx00qghkjx2ptty8ex/cia5rherx00qhhkjx5latps1q/cia5rherx00qihkjx9bdo19z2/cia5rherx00qjhkjxfj5a849t", + "https://example.com/cia5rherx00qkhkjxataa91qp/cia5rherx00qlhkjxkos37rp2/cia5rherx00qmhkjxnr52z1ck/cia5rherx00qnhkjxg2wv4j3b", + "https://example.com/cia5rherx00qohkjx5zybfy6z/cia5rherx00qphkjx3jotkzjk/cia5rherx00qqhkjxzqnuoxc1/cia5rherx00qrhkjxjqx7n6dd", + "https://example.com/cia5rherx00qshkjxfusl4u8i/cia5rherx00qthkjx7nax9k3j/cia5rherx00quhkjxce6sda7o/cia5rherx00qvhkjxvjw9krhf", + "https://example.com/cia5rherx00qwhkjx3myek18h/cia5rherx00qxhkjxye1vc3g5/cia5rherx00qyhkjx0qnkwhv8/cia5rherx00qzhkjx7xbpfb2g", + "https://example.com/cia5rherx00r0hkjxr05rkysp/cia5rherx00r1hkjx5skxve27/cia5rherx00r2hkjxm42ww2wl/cia5rherx00r3hkjxvgaok3d7", + "https://example.com/cia5rherx00r4hkjxhqn73qfk/cia5rherx00r5hkjx0vhqzi3y/cia5rherx00r6hkjxzo83uw13/cia5rherx00r7hkjxshtbkjap", + "https://example.com/cia5rherx00r8hkjxkolk05tr/cia5rherx00r9hkjx87txftcu/cia5rherx00rahkjx7zt1mxfl/cia5rherx00rbhkjxj3225obu", + "https://example.com/cia5rherx00rchkjxec3j4cbw/cia5rherx00rdhkjx2o60bre2/cia5rherx00rehkjx1gza5jo1/cia5rherx00rfhkjx6i15t347", + "https://example.com/cia5rherx00rghkjxqsx1tilb/cia5rherx00rhhkjxld7q3ees/cia5rherx00rihkjxywwmwh7a/cia5rherx00rjhkjxa7lzkj0b", + "https://example.com/cia5rherx00rkhkjxo2uekt9y/cia5rherx00rlhkjxz1y92fdx/cia5rherx00rmhkjxrls010bq/cia5rherx00rnhkjx5phg7y9q", + "https://example.com/cia5rherx00rohkjxt69rlmpb/cia5rherx00rphkjxxdo9vbof/cia5rherx00rqhkjxgnvs2gxf/cia5rherx00rrhkjx5vv7kk7v", + "https://example.com/cia5rherx00rshkjxfn0v3dx4/cia5rherx00rthkjx5wkktp8p/cia5rherx00ruhkjx6palbn57/cia5rherx00rvhkjxugk01wvb", + "https://example.com/cia5rherx00rwhkjxzgza4olt/cia5rherx00rxhkjxwdssdcbj/cia5rherx00ryhkjxxnyxv3t6/cia5rherx00rzhkjxhr6w38i4", + "https://example.com/cia5rherx00s0hkjxerf3k9ib/cia5rherx00s1hkjxnqqw079u/cia5rherx00s2hkjxifw1h8n3/cia5rherx00s3hkjxd05rx85o", + "https://example.com/cia5rherx00s4hkjx2x89mh5w/cia5rherx00s5hkjx87d0h7li/cia5rherx00s6hkjxqjueoeqw/cia5rherx00s7hkjxyq9w3n82", + "https://example.com/cia5rherx00s8hkjxzigd1zp5/cia5rherx00s9hkjxdtx2amst/cia5rherx00sahkjx2onc2f21/cia5rherx00sbhkjx4hgu22zb", + "https://example.com/cia5rherx00schkjxrz7trjeu/cia5rherx00sdhkjxwmrp365i/cia5rherx00sehkjx7eq317yf/cia5rherx00sfhkjxo93dnhcw", + "https://example.com/cia5rherx00sghkjx2zsmv5zb/cia5rherx00shhkjxuu1u80qs/cia5rherx00sihkjx8avektr4/cia5rherx00sjhkjxpg3tjre5", + "https://example.com/cia5rherx00skhkjxm9hrr8dp/cia5rherx00slhkjxmklu8fxx/cia5rherx00smhkjxpz58b4co/cia5rherx00snhkjx6kflkbwz", + "https://example.com/cia5rherx00sohkjx6zveco0b/cia5rherx00sphkjx9tzv10q9/cia5rherx00sqhkjx5kw9y9vt/cia5rherx00srhkjx5q2dpc7o", + "https://example.com/cia5rherx00sshkjxfmf3zzhg/cia5rherx00sthkjx085rnzf5/cia5rherx00suhkjxo7eaxytl/cia5rherx00svhkjx3bfbsur9", + "https://example.com/cia5rherx00swhkjxzihyd64f/cia5rherx00sxhkjxor1bcxl3/cia5rhery00syhkjxlensj1wa/cia5rhery00szhkjxwk1jdpzj", + "https://example.com/cia5rhery00t0hkjxz5hf0kfl/cia5rhery00t1hkjx36ar1r16/cia5rhery00t2hkjxcv7t3hqq/cia5rhery00t3hkjxkzgqe0a6", + "https://example.com/cia5rhery00t4hkjxpmbibq16/cia5rhery00t5hkjxrqr5n4lt/cia5rhery00t6hkjx4npmmnvj/cia5rhery00t7hkjxewqanavg", + "https://example.com/cia5rhery00t8hkjxci9wud4s/cia5rhery00t9hkjxui809qxy/cia5rhery00tahkjx870oqect/cia5rhery00tbhkjx8g24crc0", + "https://example.com/cia5rhery00tchkjxzjllkr5i/cia5rhery00tdhkjxqnnjio0r/cia5rhery00tehkjxnice4c5a/cia5rhery00tfhkjx0b0tfkbd", + "https://example.com/cia5rhery00tghkjx1hfn5jnm/cia5rhery00thhkjx0m68lrh0/cia5rhery00tihkjxe24uktvm/cia5rhery00tjhkjxwudbxvgf", + "https://example.com/cia5rhery00tkhkjxyupjrqmt/cia5rhery00tlhkjxns9kt84a/cia5rhery00tmhkjxnjnkvsza/cia5rhery00tnhkjx2u1kf42m", + "https://example.com/cia5rhery00tohkjxv8euxxvv/cia5rhery00tphkjxcewtixg8/cia5rhery00tqhkjxm0fvhod1/cia5rhery00trhkjxzfels6hy", + "https://example.com/cia5rhery00tshkjxcnmhpytv/cia5rhery00tthkjxgmwy284j/cia5rhery00tuhkjxvl9f0bet/cia5rhery00tvhkjxvd9h00tu", + "https://example.com/cia5rhery00twhkjxworddumj/cia5rhery00txhkjx5hn3bob3/cia5rhery00tyhkjxwxhy5o31/cia5rhery00tzhkjxy7swe892", + "https://example.com/cia5rhery00u0hkjx9v2rskyu/cia5rhery00u1hkjxt65535lp/cia5rhery00u2hkjxiephk9x0/cia5rhery00u3hkjxylul9icr", + "https://example.com/cia5rhery00u4hkjxo1tucbyj/cia5rhery00u5hkjxchfewpdu/cia5rhery00u6hkjxzh1f7dsv/cia5rhery00u7hkjxow4myzvc", + "https://example.com/cia5rhery00u8hkjxjkljwzmx/cia5rhery00u9hkjxb2hq0zff/cia5rhery00uahkjx3zil5iye/cia5rhery00ubhkjxpx6l4i62", + "https://example.com/cia5rhery00uchkjxzybwg1aj/cia5rhery00udhkjxbc85v998/cia5rhery00uehkjx9x8k1ebt/cia5rhery00ufhkjxziinfjvs", + "https://example.com/cia5rhery00ughkjxqjq7rbqe/cia5rhery00uhhkjxn422gi7s/cia5rhery00uihkjx6wdh0kru/cia5rhery00ujhkjxgx2z6i30", + "https://example.com/cia5rhery00ukhkjxxwekiqd8/cia5rhery00ulhkjxti0u03ji/cia5rhery00umhkjxneh94911/cia5rhery00unhkjx5uothdt2", + "https://example.com/cia5rhery00uohkjxh8wvz750/cia5rhery00uphkjxya408v8j/cia5rhery00uqhkjxowcw4c0j/cia5rhery00urhkjxp0hxhyjr", + "https://example.com/cia5rhery00ushkjxj5ezkt47/cia5rhery00uthkjxfcbhp09u/cia5rhery00uuhkjxw14otqjw/cia5rhery00uvhkjxhkdyfxrs", + "https://example.com/cia5rhery00uwhkjxehm078n0/cia5rhery00uxhkjxbadu0bio/cia5rhery00uyhkjxak1ocp8h/cia5rhery00uzhkjxing0qkah", + "https://example.com/cia5rhery00v0hkjxl8s5to3x/cia5rhery00v1hkjx9t9obxjk/cia5rhery00v2hkjx37ndtfyo/cia5rhery00v3hkjxgsrqx8wo", + "https://example.com/cia5rhery00v4hkjxcv3mqk3k/cia5rhery00v5hkjxm8gbw43x/cia5rhery00v6hkjxu55dmspc/cia5rhery00v7hkjx34me677j", + "https://example.com/cia5rhery00v8hkjxwecik8go/cia5rhery00v9hkjxap89471j/cia5rhery00vahkjxllo77l7s/cia5rhery00vbhkjxrqbtypmt", + "https://example.com/cia5rhery00vchkjx4uu12kzr/cia5rhery00vdhkjx5sxrg1cw/cia5rhery00vehkjxpimi78w5/cia5rhery00vfhkjxvu1h0bnc", + "https://example.com/cia5rhery00vghkjxryq1kku2/cia5rhery00vhhkjx8yq7g6dg/cia5rhery00vihkjxrhoe95rs/cia5rhery00vjhkjxg036tj71", + "https://example.com/cia5rhery00vkhkjxfk603ubw/cia5rhery00vlhkjxv6cvncpa/cia5rhery00vmhkjxtlptcbfj/cia5rhery00vnhkjx019qtozs", + "https://example.com/cia5rhery00vohkjx55roqfyq/cia5rhery00vphkjxq51jjd0w/cia5rhery00vqhkjxzl5r049u/cia5rhery00vrhkjx8w085tma", + "https://example.com/cia5rhery00vshkjx43gw6sxr/cia5rhery00vthkjx20l8em51/cia5rhery00vuhkjxdwoh2vk9/cia5rhery00vvhkjxw1g6vrut", + "https://example.com/cia5rhery00vwhkjxvn2ebxzm/cia5rhery00vxhkjxz2hlqhzd/cia5rhery00vyhkjxqvadz9wb/cia5rhery00vzhkjx9rh2a3uh", + "https://example.com/cia5rhery00w0hkjxa0rqkpov/cia5rhery00w1hkjxp833u09z/cia5rhery00w2hkjxn2awahj4/cia5rhery00w3hkjxwqcb9cgd", + "https://example.com/cia5rhery00w4hkjx2qn7xtr0/cia5rhery00w5hkjx4mw9p5o5/cia5rhery00w6hkjx1h6f1nne/cia5rhery00w7hkjxqgof32en", + "https://example.com/cia5rhery00w8hkjx40u225qd/cia5rhery00w9hkjx6jc1e5lj/cia5rhery00wahkjxqqn44l7k/cia5rhery00wbhkjxdwnm2lan", + "https://example.com/cia5rhery00wchkjxmf3n36f9/cia5rhery00wdhkjxjitsfpkb/cia5rhery00wehkjxrwzzunbx/cia5rhery00wfhkjxlu5ar2zw", + "https://example.com/cia5rhery00wghkjxkoktvbqj/cia5rhery00whhkjxjs6l9c1l/cia5rhery00wihkjx4c6l8wcz/cia5rhery00wjhkjxwrrx1kta", + "https://example.com/cia5rhery00wkhkjx4o64pmsg/cia5rhery00wlhkjx5jwjko3n/cia5rhery00wmhkjxuj79fu0e/cia5rhery00wnhkjxdxjnmwu4", + "https://example.com/cia5rhery00wohkjxnj9yz95o/cia5rhery00wphkjxqs39n7we/cia5rhery00wqhkjxw7enbbt8/cia5rhery00wrhkjxhirvzsj8", + "https://example.com/cia5rhery00wshkjxklfnj99r/cia5rhery00wthkjxyvxvxzxm/cia5rhery00wuhkjxselr08ti/cia5rhery00wvhkjx0n4m1gwb", + "https://example.com/cia5rhery00wwhkjxidbq1641/cia5rhery00wxhkjxer0uj0bx/cia5rhery00wyhkjx3rj98a5m/cia5rhery00wzhkjxar7ucjyp", + "https://example.com/cia5rhery00x0hkjxikq8tega/cia5rhery00x1hkjxgq9t00p7/cia5rhery00x2hkjxzczhf4ta/cia5rhery00x3hkjxs8rl0xle", + "https://example.com/cia5rhery00x4hkjxtpgdpam7/cia5rhery00x5hkjxnudpwm02/cia5rhery00x6hkjx96jfugp6/cia5rhery00x7hkjxugyl5bsm", + "https://example.com/cia5rhery00x8hkjx26k3912r/cia5rhery00x9hkjx8j1o37fy/cia5rhery00xahkjxcnx1kjtl/cia5rhery00xbhkjxws0y4q9u", + "https://example.com/cia5rhery00xchkjxnceot2tu/cia5rhery00xdhkjxmfcanvsn/cia5rhery00xehkjxti3dt4zk/cia5rhery00xfhkjx4r9pxmk8", + "https://example.com/cia5rhery00xghkjxcl0s61iu/cia5rhery00xhhkjxy85ou2fq/cia5rhery00xihkjx8p53n3u3/cia5rhery00xjhkjx6dzo2asw", + "https://example.com/cia5rhery00xkhkjxcfvflel3/cia5rhery00xlhkjxjs82vnte/cia5rhery00xmhkjxrisis221/cia5rhery00xnhkjx92ojt9kd", + "https://example.com/cia5rhery00xohkjxxu18v57w/cia5rhery00xphkjx5gfp65ut/cia5rhery00xqhkjx0zug4xqu/cia5rhery00xrhkjxxqj8k3ce", + "https://example.com/cia5rhery00xshkjxbpshqaoi/cia5rhery00xthkjxthb498xj/cia5rhery00xuhkjxu6o0heam/cia5rhery00xvhkjxcxp7f3yh", + "https://example.com/cia5rhery00xwhkjxre2n0fww/cia5rhery00xxhkjxtu8s69t6/cia5rhery00xyhkjx7q1fm4xo/cia5rhery00xzhkjx6o1nu6ga", + "https://example.com/cia5rhery00y0hkjxlsfyk6o1/cia5rhery00y1hkjxpaqyucwy/cia5rhery00y2hkjxapp8yfj0/cia5rhery00y3hkjx0fnzbtnb", + "https://example.com/cia5rhery00y4hkjxlm2p2v6d/cia5rhery00y5hkjx0vzxj59x/cia5rhery00y6hkjxkamlg3ck/cia5rhery00y7hkjxkayvfx3a", + "https://example.com/cia5rhery00y8hkjxdwdcp5cs/cia5rherz00y9hkjxjb0teg8f/cia5rherz00yahkjxs1uzi2l6/cia5rherz00ybhkjxp449moik", + "https://example.com/cia5rherz00ychkjxwvi52xlt/cia5rherz00ydhkjxr7g9xhla/cia5rherz00yehkjxmrqqq92m/cia5rherz00yfhkjxljrtx13a", + "https://example.com/cia5rherz00yghkjx6sxe01ps/cia5rherz00yhhkjxh27l6kvi/cia5rherz00yihkjxbrazjcpk/cia5rherz00yjhkjxmi6ft3qb", + "https://example.com/cia5rherz00ykhkjxifkk8p2x/cia5rherz00ylhkjxpqz9t3cb/cia5rherz00ymhkjxc00r45v0/cia5rherz00ynhkjxgnbgvycv", + "https://example.com/cia5rherz00yohkjxmr9yq0jk/cia5rherz00yphkjxaamwbi9t/cia5rherz00yqhkjxv53l03jj/cia5rherz00yrhkjxsvkylos9", + "https://example.com/cia5rherz00yshkjx3vhhy1ut/cia5rherz00ythkjxnx4cssnw/cia5rherz00yuhkjxyojj0lzk/cia5rherz00yvhkjxoyozoftr", + "https://example.com/cia5rherz00ywhkjxl9hqei9p/cia5rherz00yxhkjxyefcfdtf/cia5rherz00yyhkjxpmcgonjp/cia5rherz00yzhkjxtp7r9f3r", + "https://example.com/cia5rherz00z0hkjxrhistyei/cia5rherz00z1hkjxinya1udt/cia5rherz00z2hkjxt2uibesw/cia5rherz00z3hkjxrv504cf7", + "https://example.com/cia5rherz00z4hkjxo0a9j311/cia5rherz00z5hkjxazvef1je/cia5rherz00z6hkjxb75qrdko/cia5rherz00z7hkjxnv1dgzal", + "https://example.com/cia5rherz00z8hkjx8ny20q55/cia5rherz00z9hkjxwlj70f0m/cia5rherz00zahkjx42hyz0m2/cia5rherz00zbhkjxwotnxn7y", + "https://example.com/cia5rherz00zchkjxa1tke93x/cia5rherz00zdhkjxzd8wghy0/cia5rherz00zehkjxba383v6a/cia5rherz00zfhkjxjkcs4bwl", + "https://example.com/cia5rherz00zghkjxk8sklzkb/cia5rherz00zhhkjxfwy3q53n/cia5rherz00zihkjxvheuc2pr/cia5rherz00zjhkjxbzljm6zl", + "https://example.com/cia5rherz00zkhkjxv84kqu85/cia5rherz00zlhkjxso9bvlw7/cia5rherz00zmhkjxib6w80kz/cia5rherz00znhkjx41d5nf7s", + "https://example.com/cia5rherz00zohkjxdd3iy5vu/cia5rherz00zphkjxwad53vl4/cia5rherz00zqhkjx4ad3e109/cia5rherz00zrhkjx4r6bwsia", + "https://example.com/cia5rherz00zshkjx1rzmmdvs/cia5rherz00zthkjx6165yn1j/cia5rherz00zuhkjxnefkdvqx/cia5rherz00zvhkjxgm3q4960", + "https://example.com/cia5rherz00zwhkjx78quz63t/cia5rherz00zxhkjxdj7uhzb1/cia5rherz00zyhkjxgfrciagd/cia5rherz00zzhkjx2fhew3jm", + "https://example.com/cia5rherz0100hkjxpoguaspc/cia5rherz0101hkjxj8wf0hvv/cia5rherz0102hkjxft1bu9bg/cia5rherz0103hkjxh7rc7icq", + "https://example.com/cia5rherz0104hkjxl336njqd/cia5rherz0105hkjxq5od0pbq/cia5rherz0106hkjxtgk2vqld/cia5rherz0107hkjxnyquy58x", + "https://example.com/cia5rherz0108hkjx0e8vlkuv/cia5rherz0109hkjx1w4ao3zl/cia5rherz010ahkjx0uvsczyx/cia5rherz010bhkjxctueaktb", + "https://example.com/cia5rherz010chkjxgocyaoln/cia5rherz010dhkjxvym9xjpu/cia5rherz010ehkjxkmv9qd1h/cia5rherz010fhkjxux4kwy9m", + "https://example.com/cia5rherz010ghkjxgjg51jll/cia5rherz010hhkjxheyloqok/cia5rherz010ihkjxocdihpf0/cia5rherz010jhkjxkjob7g4l", + "https://example.com/cia5rherz010khkjxyqnrc520/cia5rherz010lhkjxyfmva03e/cia5rherz010mhkjx7pf5a1q1/cia5rherz010nhkjxvcbajwq5", + "https://example.com/cia5rherz010ohkjxlj1peujy/cia5rherz010phkjxbe0edddw/cia5rherz010qhkjxese0v0d0/cia5rherz010rhkjx3z5sqmvd", + "https://example.com/cia5rherz010shkjx031cq64v/cia5rherz010thkjxo31twbuq/cia5rherz010uhkjxx0w89wkt/cia5rherz010vhkjxunc21rd5", + "https://example.com/cia5rherz010whkjxokxwgntu/cia5rherz010xhkjx8zhcummr/cia5rherz010yhkjxrr19wgdd/cia5rherz010zhkjx3krrrmfy", + "https://example.com/cia5rherz0110hkjxsnag5gy8/cia5rherz0111hkjxti3yt0uo/cia5rherz0112hkjxuvxezwly/cia5rherz0113hkjx42tjs0w1", + "https://example.com/cia5rherz0114hkjxhap5ggkp/cia5rherz0115hkjxhpvs28ez/cia5rherz0116hkjxchtzr6ub/cia5rherz0117hkjxldrskwe8", + "https://example.com/cia5rherz0118hkjx3moumoc6/cia5rherz0119hkjx8jmsv1po/cia5rherz011ahkjx5tbk1781/cia5rherz011bhkjxda8axg94", + "https://example.com/cia5rherz011chkjxpch14tnq/cia5rherz011dhkjx779uxhge/cia5rherz011ehkjxd1gossfl/cia5rherz011fhkjxe6fsxfle", + "https://example.com/cia5rherz011ghkjxkv0e5x6o/cia5rherz011hhkjxrmm01lop/cia5rherz011ihkjxadtpfth3/cia5rherz011jhkjxmsbqnjtx", + "https://example.com/cia5rherz011khkjxs9v8q989/cia5rherz011lhkjxbb47ojmz/cia5rherz011mhkjxfqrbtm2s/cia5rherz011nhkjxf0ukz3z7", + "https://example.com/cia5rherz011ohkjx1ljxj85v/cia5rherz011phkjxzyc52kr1/cia5rherz011qhkjxx03aq7rt/cia5rherz011rhkjxib8yi6wz", + "https://example.com/cia5rherz011shkjxelvnazea/cia5rherz011thkjx6ge3ekjc/cia5rherz011uhkjxj6spkxml/cia5rherz011vhkjx2g4n6l67", + "https://example.com/cia5rherz011whkjxoa9tcq7o/cia5rherz011xhkjx0aa7y41v/cia5rherz011yhkjx56c9pqwk/cia5rherz011zhkjx5iy36w89", + "https://example.com/cia5rherz0120hkjx04d4k3fi/cia5rherz0121hkjxvl6nw4m5/cia5rherz0122hkjxxc1jbk55/cia5rherz0123hkjx0pjnf99r", + "https://example.com/cia5rherz0124hkjx91z4pze5/cia5rherz0125hkjxg7qc4u38/cia5rherz0126hkjx1yegfvw4/cia5rherz0127hkjxl15ygp9h", + "https://example.com/cia5rherz0128hkjx4tkyj2om/cia5rherz0129hkjx7h7oqbrp/cia5rherz012ahkjxkajenzcs/cia5rherz012bhkjxi4bowdxs", + "https://example.com/cia5rherz012chkjx2szlut25/cia5rherz012dhkjxupa098p0/cia5rherz012ehkjxlg93n5ca/cia5rherz012fhkjx3e3lqc5s", + "https://example.com/cia5rherz012ghkjxiypurty6/cia5rherz012hhkjxnpuba7yn/cia5rherz012ihkjxqajb87r0/cia5rherz012jhkjx3w8tfq58", + "https://example.com/cia5rherz012khkjx6kpvfmqc/cia5rherz012lhkjxle4ozx4b/cia5rherz012mhkjxcx3zh6l8/cia5rherz012nhkjxwhol1p7z", + "https://example.com/cia5rherz012ohkjx8hcbuea8/cia5rherz012phkjxjn78pjpu/cia5rherz012qhkjxcso0w7ob/cia5rherz012rhkjxuwsrzjnb", + "https://example.com/cia5rherz012shkjxmfrs9kl4/cia5rherz012thkjxqd79q3jk/cia5rherz012uhkjx4y92c2l9/cia5rherz012vhkjxah5zn3ql", + "https://example.com/cia5rherz012whkjxnoco7250/cia5rherz012xhkjx50xbnyn6/cia5rherz012yhkjxhiz0qo7f/cia5rherz012zhkjxpvm1udb0", + "https://example.com/cia5rherz0130hkjxo3o4fndr/cia5rherz0131hkjxkz4fvrzq/cia5rherz0132hkjxz7tax104/cia5rherz0133hkjx1g0g06dn", + "https://example.com/cia5rherz0134hkjx5y2031n3/cia5rherz0135hkjx7uqpqo6r/cia5rherz0136hkjxy33y4m0i/cia5rherz0137hkjxi2zwuxom", + "https://example.com/cia5rherz0138hkjx2r63kcp2/cia5rherz0139hkjxqhukego2/cia5rherz013ahkjxrct5kwo5/cia5rherz013bhkjxlwdsrkdb", + "https://example.com/cia5rherz013chkjxqb7drtld/cia5rherz013dhkjxhk7nzkp6/cia5rherz013ehkjxt59enx4r/cia5rherz013fhkjxmequgudl", + "https://example.com/cia5rherz013ghkjxsvtndprv/cia5rherz013hhkjx5qzj9yky/cia5rherz013ihkjx7wi51091/cia5rherz013jhkjx07qd1vho", + "https://example.com/cia5rherz013khkjxxhuteqrg/cia5rherz013lhkjxnytihmq0/cia5rherz013mhkjxxcuyopb7/cia5rherz013nhkjxz8wjm6kv", + "https://example.com/cia5rherz013ohkjxti624ceh/cia5rherz013phkjx9es0m8z1/cia5rherz013qhkjx2thq0yiq/cia5rherz013rhkjxcz1h935h", + "https://example.com/cia5rherz013shkjxy1fcs2p9/cia5rherz013thkjxqj1f3hzd/cia5rherz013uhkjx9n3img9m/cia5rherz013vhkjxbbsd9s7u", + "https://example.com/cia5rherz013whkjxzrks74yw/cia5rherz013xhkjx19u9gnum/cia5rherz013yhkjxf189dqov/cia5rherz013zhkjxn840ifqp", + "https://example.com/cia5rherz0140hkjxzflkd8o3/cia5rherz0141hkjxth1k5pcv/cia5rherz0142hkjxk4tx2d6t/cia5rherz0143hkjxua6in4hi", + "https://example.com/cia5rherz0144hkjxh5223dp4/cia5rherz0145hkjxggdx0inf/cia5rherz0146hkjxukma20rn/cia5rherz0147hkjxbz6yr3vj", + "https://example.com/cia5rherz0148hkjxj6yz49cp/cia5rherz0149hkjxnshaboc7/cia5rherz014ahkjx9k7w03oz/cia5rherz014bhkjxim3qdl32", + "https://example.com/cia5rherz014chkjxkwm4bedt/cia5rherz014dhkjxd87owzz9/cia5rherz014ehkjx7gvsq5h8/cia5rherz014fhkjxvg5i2lzo", + "https://example.com/cia5rhes0014ghkjx57nwx3bd/cia5rhes0014hhkjx7yg5lnmm/cia5rhes0014ihkjxpcj5y5pc/cia5rhes0014jhkjxx7pqjwyi", + "https://example.com/cia5rhes0014khkjx8je94eu9/cia5rhes0014lhkjxj7s0ayqj/cia5rhes0014mhkjxaj8frq8f/cia5rhes0014nhkjxsvfwikzc", + "https://example.com/cia5rhes0014ohkjx7w5zhsfa/cia5rhes0014phkjxb8znpn93/cia5rhes0014qhkjx26dojt2q/cia5rhes0014rhkjx4z51j3v1", + "https://example.com/cia5rhes0014shkjxsrfqfh66/cia5rhes0014thkjxchkutc4l/cia5rhes0014uhkjx61hu197u/cia5rhes0014vhkjx88tbe055", + "https://example.com/cia5rhes0014whkjxnsy6o8oh/cia5rhes0014xhkjxhf7qa11c/cia5rhes0014yhkjx8dg54bhs/cia5rhes0014zhkjxwddjjbfx", + "https://example.com/cia5rhes00150hkjxvbx9bs0t/cia5rhes00151hkjx1ndja821/cia5rhes00152hkjxgre5jaft/cia5rhes00153hkjx403j16ab", + "https://example.com/cia5rhes00154hkjx850qetqc/cia5rhes00155hkjx25fuxyq1/cia5rhes00156hkjxt0otyqf9/cia5rhes00157hkjxkrckit2g", + "https://example.com/cia5rhes00158hkjxeka610dd/cia5rhes00159hkjxirohiw4g/cia5rhes0015ahkjx4a2e7hwj/cia5rhes0015bhkjx54ew959x", + "https://example.com/cia5rhes0015chkjxdbymvixv/cia5rhes0015dhkjxehyc0l2p/cia5rhes0015ehkjxkkzsw7sr/cia5rhes0015fhkjx694x9jdr", + "https://example.com/cia5rhes0015ghkjx8524c513/cia5rhes0015hhkjx9gbh0axg/cia5rhes0015ihkjxhux4m9va/cia5rhes0015jhkjxizfpn19a", + "https://example.com/cia5rhes0015khkjxmy1viucg/cia5rhes0015lhkjx4k9cpi1x/cia5rhes0015mhkjxlwccit5i/cia5rhes0015nhkjxyyerl1x5", + "https://example.com/cia5rhes0015ohkjxse6u4cq1/cia5rhes0015phkjxmbosv5k1/cia5rhes0015qhkjxagd18q9e/cia5rhes0015rhkjx18k37mza", + "https://example.com/cia5rhes0015shkjxkf88qpr2/cia5rhes0015thkjxxjngloiv/cia5rhes0015uhkjxw3p51ph3/cia5rhes0015vhkjxuv117q6n", + "https://example.com/cia5rhes0015whkjxepiuli5w/cia5rhes0015xhkjx6912ozju/cia5rhes0015yhkjxs50s4iw9/cia5rhes0015zhkjx4fqv3fj5", + "https://example.com/cia5rhes00160hkjxaezek04y/cia5rhes00161hkjxo7jexmqa/cia5rhes00162hkjx8qt4t84r/cia5rhes00163hkjx0x35v1ea", + "https://example.com/cia5rhes00164hkjxum5cztru/cia5rhes00165hkjxykw801f6/cia5rhes00166hkjx87cgbtl9/cia5rhes00167hkjxr5laneo4", + "https://example.com/cia5rhes00168hkjx4675xx8q/cia5rhes00169hkjxa69bs98w/cia5rhes0016ahkjxgxbg1ktg/cia5rhes0016bhkjxcssrfeb6", + "https://example.com/cia5rhes0016chkjxskrmxbeu/cia5rhes0016dhkjxchuf9w7d/cia5rhes0016ehkjx96tmup0w/cia5rhes0016fhkjx3b2ir9k8", + "https://example.com/cia5rhes0016ghkjxshn9r5cd/cia5rhes0016hhkjxt0okuboo/cia5rhes0016ihkjx3xc5n1z7/cia5rhes0016jhkjxmm1rhinv", + "https://example.com/cia5rhes0016khkjxs4md552m/cia5rhes0016lhkjxdqs7jlks/cia5rhes0016mhkjxsbqpbr27/cia5rhes0016nhkjxsw0eoqxh", + "https://example.com/cia5rhes0016ohkjxj1uyawl4/cia5rhes0016phkjx4s3keqvp/cia5rhes0016qhkjxlr2ujty3/cia5rhes0016rhkjxshluxgzs", + "https://example.com/cia5rhes0016shkjxmjm47a2n/cia5rhes0016thkjxvawl3vod/cia5rhes0016uhkjxlkwcmxrn/cia5rhes0016vhkjx0yrlq0k0", + "https://example.com/cia5rhes0016whkjx47wacy0d/cia5rhes0016xhkjxx20x2adr/cia5rhes0016yhkjxpyafhbax/cia5rhes0016zhkjxkm7homqb", + "https://example.com/cia5rhes00170hkjx58dppj32/cia5rhes00171hkjxgl9ekfiu/cia5rhes00172hkjx35hn4ajh/cia5rhes00173hkjx4k5x795i", + "https://example.com/cia5rhes00174hkjx431k7e7c/cia5rhes00175hkjxdfvrfwqy/cia5rhes00176hkjx2wqmhu6x/cia5rhes00177hkjxd8ykmqj9", + "https://example.com/cia5rhes00178hkjxpbwz0dv1/cia5rhes00179hkjxfxnx8xde/cia5rhes0017ahkjxnumozh8d/cia5rhes0017bhkjxjvgv7bu3", + "https://example.com/cia5rhes0017chkjx6cbtaca4/cia5rhes0017dhkjxl6oa2n77/cia5rhes0017ehkjxv4e7c73p/cia5rhes0017fhkjxarapkb0w", + "https://example.com/cia5rhes0017ghkjxvzviznq4/cia5rhes0017hhkjxkxkq2w0r/cia5rhes0017ihkjxfdhga9qz/cia5rhes0017jhkjxzr0zbpli", + "https://example.com/cia5rhes0017khkjxd6x7dl0m/cia5rhes0017lhkjxpa472u8x/cia5rhes0017mhkjxi7scj2zd/cia5rhes0017nhkjxcrar3doc", + "https://example.com/cia5rhes1017ohkjxim1b2tgs/cia5rhes1017phkjxido7zpq3/cia5rhes1017qhkjxzhgszmuh/cia5rhes1017rhkjxh9n4vlu9", + "https://example.com/cia5rhes1017shkjxazazffwt/cia5rhes1017thkjxt1mu7dkg/cia5rhes1017uhkjx79p8vex3/cia5rhes1017vhkjxzk3rwfaj", + "https://example.com/cia5rhes1017whkjxg9ldz44h/cia5rhes1017xhkjxubjc7d35/cia5rhes1017yhkjxyn9t58r7/cia5rhes1017zhkjx822o28jf", + "https://example.com/cia5rhes10180hkjxy0zeitqi/cia5rhes10181hkjxuiumypud/cia5rhes10182hkjxqhf3xprn/cia5rhes10183hkjx9orcdf2t", + "https://example.com/cia5rhes10184hkjx60vpjosn/cia5rhes10185hkjxiuxdbrjp/cia5rhes10186hkjxjazso4v3/cia5rhes10187hkjx1yda3p8i", + "https://example.com/cia5rhes10188hkjx67qn30yn/cia5rhes10189hkjxd8e62yud/cia5rhes1018ahkjxr1ogihzw/cia5rhes1018bhkjxqa83yxs2", + "https://example.com/cia5rhes1018chkjxcijm6ol2/cia5rhes1018dhkjxn27lkryl/cia5rhes1018ehkjxin74swtd/cia5rhes1018fhkjxc3n9hjub", + "https://example.com/cia5rhes1018ghkjxs06i4n1v/cia5rhes1018hhkjxtbrrprdd/cia5rhes1018ihkjxh375u2d5/cia5rhes1018jhkjxwe4m1w3k", + "https://example.com/cia5rhes1018khkjxc2fo3tyn/cia5rhes1018lhkjx11wqgr3o/cia5rhes1018mhkjx55cz73km/cia5rhes1018nhkjx027g05rh", + "https://example.com/cia5rhes1018ohkjxuxt9w0qg/cia5rhes1018phkjxuppi0zpt/cia5rhes1018qhkjx3hedzfgq/cia5rhes1018rhkjxdbef85sg", + "https://example.com/cia5rhes1018shkjxh23bn4hl/cia5rhes1018thkjx2jo33xt5/cia5rhes1018uhkjxgrf6z3q5/cia5rhes1018vhkjxs51u2bsq", + "https://example.com/cia5rhes1018whkjxh98q42o4/cia5rhes1018xhkjxtktqtwob/cia5rhes1018yhkjxxf9qq7me/cia5rhes1018zhkjx540am2xr", + "https://example.com/cia5rhes10190hkjxcom1s1af/cia5rhes10191hkjxr15i8zfz/cia5rhes10192hkjxbsyx6pqa/cia5rhes10193hkjx5lfk3tnz", + "https://example.com/cia5rhes10194hkjx63khbmh5/cia5rhes10195hkjx23tzm25c/cia5rhes10196hkjx3tu55kps/cia5rhes10197hkjxt9kgwuye", + "https://example.com/cia5rhes10198hkjxvsic8zyi/cia5rhes10199hkjxiqcxj6ha/cia5rhes1019ahkjxul53ymxv/cia5rhes1019bhkjx8j5i4gjo", + "https://example.com/cia5rhes1019chkjxhkzab45h/cia5rhes1019dhkjxp9m537kv/cia5rhes1019ehkjxflgayfwl/cia5rhes1019fhkjxpxcfuwm7", + "https://example.com/cia5rhes1019ghkjx0ec3mbfs/cia5rhes1019hhkjxpzum6b24/cia5rhes1019ihkjx8l7ygjw5/cia5rhes1019jhkjxc4kywxia", + "https://example.com/cia5rhes1019khkjxcgdm2x8i/cia5rhes1019lhkjxsd4z7axk/cia5rhes1019mhkjxrivl4h0v/cia5rhes1019nhkjxwq5r7rjw", + "https://example.com/cia5rhes1019ohkjxlbgrt2qs/cia5rhes1019phkjxrqx7xr97/cia5rhes1019qhkjxqxuaxnbc/cia5rhes1019rhkjx479nva7e", + "https://example.com/cia5rhes1019shkjxo903skww/cia5rhes1019thkjx835fib01/cia5rhes1019uhkjxqnb5hb1c/cia5rhes1019vhkjx985hdr8a", + "https://example.com/cia5rhes1019whkjxul29xzs7/cia5rhes1019xhkjx8v769rft/cia5rhes1019yhkjx8moz4avh/cia5rhes1019zhkjxltk1bmj1", + "https://example.com/cia5rhes101a0hkjxgj1bgqcu/cia5rhes101a1hkjxam87hyv6/cia5rhes101a2hkjx6n7xfzcf/cia5rhes101a3hkjxwhuzx4bu", + "https://example.com/cia5rhes101a4hkjxkz4gt4bb/cia5rhes101a5hkjx8jto6sbw/cia5rhes101a6hkjxdkz6q053/cia5rhes101a7hkjxafsa477k", + "https://example.com/cia5rhes101a8hkjxmawq81f9/cia5rhes101a9hkjx7a1m25vl/cia5rhes101aahkjxn9vib2k1/cia5rhes101abhkjxnd9lr35m", + "https://example.com/cia5rhes101achkjxaz60ife8/cia5rhes101adhkjxb9jyduwc/cia5rhes101aehkjximmqxxdc/cia5rhes101afhkjxrivbhs77", + "https://example.com/cia5rhes101aghkjxrq2da1pd/cia5rhes101ahhkjxjcuopb44/cia5rhes101aihkjxmeqnm90k/cia5rhes101ajhkjxli2mp598", + "https://example.com/cia5rhes101akhkjx3uzisjok/cia5rhes101alhkjx2z2rnozw/cia5rhes101amhkjxektigddg/cia5rhes101anhkjxvsxrqsn5", + "https://example.com/cia5rhes101aohkjxfb78h44w/cia5rhes101aphkjx2g9hr8n2/cia5rhes101aqhkjx61plt1q1/cia5rhes101arhkjxajstaro6", + "https://example.com/cia5rhes101ashkjxm5zox05g/cia5rhes101athkjxqnofipd6/cia5rhes101auhkjxr39j3scv/cia5rhes101avhkjxcnv43592", + "https://example.com/cia5rhes101awhkjxi5amucip/cia5rhes101axhkjxy05rb4by/cia5rhes101ayhkjxoqbug0w2/cia5rhes101azhkjxobqv30io", + "https://example.com/cia5rhes101b0hkjxgtxrq6a1/cia5rhes101b1hkjx3kckskk9/cia5rhes101b2hkjxgp3x3n2k/cia5rhes101b3hkjxjoa91opd", + "https://example.com/cia5rhes101b4hkjxl0uryndo/cia5rhes101b5hkjxn8o6oumu/cia5rhes101b6hkjxrjbze70s/cia5rhes101b7hkjxv5mrjv5y", + "https://example.com/cia5rhes101b8hkjx7yeg3s3m/cia5rhes101b9hkjxnchy3mil/cia5rhes101bahkjxawomopeo/cia5rhes101bbhkjx9oml99jy", + "https://example.com/cia5rhes101bchkjxzaccplvr/cia5rhes101bdhkjxmv4u1l8n/cia5rhes101behkjxja90rgy0/cia5rhes101bfhkjxolfzxocw", + "https://example.com/cia5rhes101bghkjxy1vfbaaw/cia5rhes101bhhkjxg6xznpan/cia5rhes101bihkjxlg9fzku8/cia5rhes101bjhkjxnh2hjnui", + "https://example.com/cia5rhes101bkhkjxsclo2zp3/cia5rhes101blhkjxuvrcudv5/cia5rhes101bmhkjx605j2zjj/cia5rhes101bnhkjx2xml0fvu", + "https://example.com/cia5rhes101bohkjx5gf3ijos/cia5rhes101bphkjxpe0su46e/cia5rhes101bqhkjxs22f7ad2/cia5rhes101brhkjx9agg7eo1", + "https://example.com/cia5rhes101bshkjx3sn7g8yy/cia5rhes101bthkjxe04n3g8b/cia5rhes101buhkjxgy5w6ts0/cia5rhes101bvhkjx7q91193i", + "https://example.com/cia5rhes101bwhkjxfgzjxdtg/cia5rhes101bxhkjx9fof34tp/cia5rhes101byhkjxoyqeyb8o/cia5rhes101bzhkjxs4h8rhgv", + "https://example.com/cia5rhes101c0hkjxxj1zh5us/cia5rhes101c1hkjxoc2z40bk/cia5rhes101c2hkjxp2mh4dck/cia5rhes101c3hkjx11dpdt65", + "https://example.com/cia5rhes101c4hkjxrfzebrql/cia5rhes101c5hkjxhgaz06ty/cia5rhes101c6hkjxrs79e85g/cia5rhes101c7hkjxggrhbqyx", + "https://example.com/cia5rhes101c8hkjxvgxdgnbw/cia5rhes101c9hkjxyjttv60a/cia5rhes101cahkjxsq6fq2jl/cia5rhes101cbhkjx7q41av4q", + "https://example.com/cia5rhes101cchkjxovipv8ev/cia5rhes101cdhkjxqmj2adv7/cia5rhes101cehkjxkac54kr0/cia5rhes101cfhkjxuqcmlixm", + "https://example.com/cia5rhes101cghkjx8p3hy50r/cia5rhes101chhkjxitheakp9/cia5rhes101cihkjxrm1z2bcp/cia5rhes101cjhkjx6e60mdcr", + "https://example.com/cia5rhes101ckhkjxavxp0z0w/cia5rhes101clhkjxo78s8ce3/cia5rhes101cmhkjx3q5plsy4/cia5rhes101cnhkjxbr2dyljs", + "https://example.com/cia5rhes101cohkjxx6uzzh6z/cia5rhes101cphkjx39t0gmdt/cia5rhes101cqhkjx5cpi5gv2/cia5rhes101crhkjx8tiw2khg", + "https://example.com/cia5rhes101cshkjxx26p2oew/cia5rhes101cthkjxh5x6cctw/cia5rhes101cuhkjxmqbok5qb/cia5rhes101cvhkjx98q4u6vg", + "https://example.com/cia5rhes101cwhkjxca46qqdc/cia5rhes101cxhkjxkya9jblw/cia5rhes101cyhkjxsx55uj72/cia5rhes101czhkjx4px01ypv", + "https://example.com/cia5rhes201d0hkjxrfq1bxuy/cia5rhes201d1hkjxum4pm3s6/cia5rhes201d2hkjx9djj6tvc/cia5rhes201d3hkjxkobt5p5a", + "https://example.com/cia5rhes201d4hkjx6vbuy1h3/cia5rhes201d5hkjxtyrtq6sn/cia5rhes201d6hkjxyn0dbgeq/cia5rhes201d7hkjx9g1d2pu0", + "https://example.com/cia5rhes201d8hkjxsci6f24w/cia5rhes201d9hkjxd8q6ugbk/cia5rhes201dahkjx8c8yunrs/cia5rhes201dbhkjxb657b9hh", + "https://example.com/cia5rhes201dchkjxhpytp0es/cia5rhes201ddhkjxzz6or9dl/cia5rhes201dehkjxvt1iaj4e/cia5rhes201dfhkjxcovukh36", + "https://example.com/cia5rhes201dghkjxs4wcuyr5/cia5rhes201dhhkjxa3ltvy94/cia5rhes201dihkjx1q3i72ys/cia5rhes201djhkjxjgthq1xi", + "https://example.com/cia5rhes201dkhkjxsvsw8r7g/cia5rhes201dlhkjxho9dzz7z/cia5rhes201dmhkjxtd6y9lt9/cia5rhes201dnhkjx2mryfja5", + "https://example.com/cia5rhes201dohkjx1qpsam6z/cia5rhes201dphkjxyqckmdus/cia5rhes201dqhkjx05x0cua4/cia5rhes201drhkjxlv48ezca", + "https://example.com/cia5rhes201dshkjxv3tvrlnv/cia5rhes201dthkjxsb1vp68a/cia5rhes201duhkjxr3dpwbsl/cia5rhes201dvhkjxooy13asr", + "https://example.com/cia5rhes201dwhkjxy2yxmf1a/cia5rhes201dxhkjxg7ddbk62/cia5rhes201dyhkjxyfw66d9i/cia5rhes201dzhkjxhiriqvpp", + "https://example.com/cia5rhes201e0hkjxgnojdvfu/cia5rhes201e1hkjx35d46pkf/cia5rhes201e2hkjx8al6xyxc/cia5rhes201e3hkjxpm8o33n5", + "https://example.com/cia5rhes201e4hkjxgmt6q22i/cia5rhes201e5hkjxcxujptph/cia5rhes201e6hkjxvjqvqv5y/cia5rhes201e7hkjx7frk9v00", + "https://example.com/cia5rhes201e8hkjxwksc2h6k/cia5rhes201e9hkjxmrv2nebe/cia5rhes201eahkjxju4ycxem/cia5rhes201ebhkjxu63x5ai0", + "https://example.com/cia5rhes201echkjxq4et8qb3/cia5rhes201edhkjxmawlqvb6/cia5rhes201eehkjx5mvbc5jf/cia5rhes201efhkjxf81g9ft0", + "https://example.com/cia5rhes201eghkjxwc2n8rrz/cia5rhes201ehhkjx96jrb9qp/cia5rhes201eihkjxolmvqk0b/cia5rhes201ejhkjx8t4yxqdy", + "https://example.com/cia5rhes201ekhkjxjj375p8m/cia5rhes201elhkjxd7n988u0/cia5rhes201emhkjx4sgv75jt/cia5rhes201enhkjx89v2rpwd", + "https://example.com/cia5rhes201eohkjx441c02sl/cia5rhes201ephkjxicff9k4p/cia5rhes201eqhkjx5c7sjm4x/cia5rhes201erhkjxvl0a13y1", + "https://example.com/cia5rhes201eshkjxf8inxrty/cia5rhes201ethkjxqjixrhe3/cia5rhes201euhkjx6cq543as/cia5rhes201evhkjxiq4rvbm6", + "https://example.com/cia5rhes201ewhkjxpzr481o0/cia5rhes201exhkjxfqo3ya1u/cia5rhes201eyhkjxzaieceuz/cia5rhes201ezhkjxrp9aiyto", + "https://example.com/cia5rhes201f0hkjxdxg04ktt/cia5rhes201f1hkjxc5xqh8w6/cia5rhes201f2hkjxxqt7mk69/cia5rhes201f3hkjxhz4mt35k", + "https://example.com/cia5rhes201f4hkjxwif8ix73/cia5rhes201f5hkjxcnk41o2f/cia5rhes201f6hkjxqvmwkmte/cia5rhes201f7hkjxjhf0rwkd", + "https://example.com/cia5rhes201f8hkjxgu0mbayd/cia5rhes201f9hkjxoirm2pi6/cia5rhes201fahkjx3eggxv1v/cia5rhes201fbhkjx3dr0v5lr", + "https://example.com/cia5rhes201fchkjx9bl00653/cia5rhes201fdhkjxd986f7dy/cia5rhes201fehkjxcjhtezko/cia5rhes201ffhkjx2g6pp08r", + "https://example.com/cia5rhes201fghkjxieabfnjf/cia5rhes201fhhkjxhncmkptc/cia5rhes201fihkjxd2idn405/cia5rhes201fjhkjxxh12k6dz", + "https://example.com/cia5rhes201fkhkjxjhb8dl6c/cia5rhes201flhkjxjesmmxj5/cia5rhes201fmhkjxgdq4watu/cia5rhes201fnhkjxcx2v7046", + "https://example.com/cia5rhes201fohkjxooyrgbd6/cia5rhes201fphkjxnswgkqhg/cia5rhes201fqhkjxr1olqtyi/cia5rhes201frhkjxpylkppc7", + "https://example.com/cia5rhes201fshkjxss3f3m7a/cia5rhes201fthkjxtb752b31/cia5rhes201fuhkjxl8v5tked/cia5rhes201fvhkjxs83n3lna", + "https://example.com/cia5rhes201fwhkjx7mn2ufyp/cia5rhes201fxhkjxykgvds9s/cia5rhes201fyhkjxzj880aau/cia5rhes201fzhkjx9hmzn5w1", + "https://example.com/cia5rhes201g0hkjx40m23pfq/cia5rhes201g1hkjxu5axzq44/cia5rhes201g2hkjxtenwlezp/cia5rhes201g3hkjxeaanxtc3", + "https://example.com/cia5rhes201g4hkjxfko2j17a/cia5rhes201g5hkjxdngk92iq/cia5rhes201g6hkjxyiixm3h3/cia5rhes201g7hkjx1e0o9rbr", + "https://example.com/cia5rhes201g8hkjxzvuuf9mr/cia5rhes201g9hkjx9i4067eb/cia5rhes201gahkjxe0877b7o/cia5rhes201gbhkjxhjeqydx3", + "https://example.com/cia5rhes201gchkjx28buxxph/cia5rhes201gdhkjx11mlvzu6/cia5rhes201gehkjx56z31f3p/cia5rhes201gfhkjxon8mxyaq", + "https://example.com/cia5rhes201gghkjxzhavbsbu/cia5rhes201ghhkjxpalfbbgq/cia5rhes201gihkjxmg14pb0i/cia5rhes201gjhkjxz1k6lfox", + "https://example.com/cia5rhes201gkhkjxr91y8n1x/cia5rhes201glhkjxcd8gf56b/cia5rhes201gmhkjxgmgi5aag/cia5rhes201gnhkjxzuskm0u3", + "https://example.com/cia5rhes201gohkjxh6stmdzj/cia5rhes201gphkjxjhxmrc1z/cia5rhes201gqhkjxbsb6x26m/cia5rhes201grhkjx2qjq5azu", + "https://example.com/cia5rhes201gshkjxwgykuiuh/cia5rhes201gthkjxshezzoh7/cia5rhes201guhkjxhxk5wn0c/cia5rhes201gvhkjxfgd3dy5o", + "https://example.com/cia5rhes201gwhkjxrjdm59mt/cia5rhes201gxhkjx5p1au9tm/cia5rhes201gyhkjx2fhr9h8l/cia5rhes201gzhkjx1d6ey84l", + "https://example.com/cia5rhes201h0hkjxw539qclb/cia5rhes201h1hkjxtasbgd4k/cia5rhes201h2hkjxnggs4jvi/cia5rhes201h3hkjx10i4oa01", + "https://example.com/cia5rhes201h4hkjxc4yc7ah2/cia5rhes201h5hkjxpp8h6vjy/cia5rhes201h6hkjx3is219tv/cia5rhes201h7hkjxi5vczfdr", + "https://example.com/cia5rhes201h8hkjx0pnfnjv8/cia5rhes201h9hkjxvab7mw12/cia5rhes201hahkjxpdkx31mo/cia5rhes201hbhkjx5qrrpii9", + "https://example.com/cia5rhes201hchkjxxpstoh0r/cia5rhes201hdhkjxd3fqr26w/cia5rhes201hehkjxa89a8p00/cia5rhes201hfhkjxjb7dx816", + "https://example.com/cia5rhes201hghkjxoundwtvv/cia5rhes201hhhkjxq0l8n544/cia5rhes201hihkjxfnxig9tq/cia5rhes201hjhkjxmthbhba3", + "https://example.com/cia5rhes201hkhkjxh8del6ix/cia5rhes201hlhkjxbbkqryiz/cia5rhes201hmhkjxsrbt8rwc/cia5rhes201hnhkjxvjinr83g", + "https://example.com/cia5rhes201hohkjxnm39jamh/cia5rhes201hphkjxbpdbg85s/cia5rhes201hqhkjxt4xsrvvw/cia5rhes201hrhkjx28uncmqm", + "https://example.com/cia5rhes201hshkjx9y3havo3/cia5rhes201hthkjxsl3xf65k/cia5rhes201huhkjxevlc6mpu/cia5rhes201hvhkjxios9hjnc", + "https://example.com/cia5rhes201hwhkjx1gqoupdx/cia5rhes201hxhkjxoezqmdn4/cia5rhes201hyhkjxsd34q556/cia5rhes201hzhkjxvkqinyu3", + "https://example.com/cia5rhes201i0hkjxtuqp1qgj/cia5rhes201i1hkjxjq0bui86/cia5rhes201i2hkjxlu4behua/cia5rhes201i3hkjxbarxd26f", + "https://example.com/cia5rhes201i4hkjx1dgyo81l/cia5rhes201i5hkjx3xb9oqcc/cia5rhes201i6hkjxgiyz2tkh/cia5rhes201i7hkjx6w3cspdt", + "https://example.com/cia5rhes201i8hkjxgazao8qk/cia5rhes201i9hkjx9g0kulps/cia5rhes201iahkjxo3xkc8pd/cia5rhes201ibhkjx1dqefi47", + "https://example.com/cia5rhes201ichkjxg5gkkixu/cia5rhes201idhkjxy4ocvh6v/cia5rhes201iehkjx4cyin399/cia5rhes201ifhkjxx2gjdbml", + "https://example.com/cia5rhes201ighkjxhg0kk0p1/cia5rhes201ihhkjxt9erqxzy/cia5rhes201iihkjxorqialmn/cia5rhes201ijhkjxuj6s809s", + "https://example.com/cia5rhes201ikhkjx0vne1gub/cia5rhes201ilhkjxvumlvx2e/cia5rhes201imhkjxkwp8knsu/cia5rhes201inhkjxi7n4t5yd", + "https://example.com/cia5rhes201iohkjxzho5l61h/cia5rhes201iphkjxxe8fo8zr/cia5rhes201iqhkjxqnsimx8u/cia5rhes201irhkjxecgdhcvp", + "https://example.com/cia5rhes201ishkjx2fi6dek5/cia5rhes201ithkjxf3i7k6mm/cia5rhes201iuhkjx0uioh430/cia5rhes201ivhkjxqw1gumyl", + "https://example.com/cia5rhes201iwhkjxlqpv0zzb/cia5rhes201ixhkjxfxx80lsv/cia5rhes201iyhkjx6f1rt1ik/cia5rhes201izhkjx0pmgbqf9", + "https://example.com/cia5rhes201j0hkjxi3jo8eqm/cia5rhes201j1hkjx8iahnhoa/cia5rhes201j2hkjxcp1bjuci/cia5rhes201j3hkjxushcyv9h", + "https://example.com/cia5rhes201j4hkjxmfy3u8bq/cia5rhes201j5hkjxl6j0ozf5/cia5rhes201j6hkjx0jrm1hr3/cia5rhes201j7hkjxkgzgfuc2", + "https://example.com/cia5rhes201j8hkjx6sbhqirt/cia5rhes201j9hkjx3jm8ttkr/cia5rhes201jahkjx4ww2jkjb/cia5rhes201jbhkjx2vwmj8mw", + "https://example.com/cia5rhes201jchkjx6s6c38ry/cia5rhes201jdhkjxo5iduoju/cia5rhes201jehkjxl337z10k/cia5rhes201jfhkjxennzj2ed", + "https://example.com/cia5rhes201jghkjx65xshc5s/cia5rhes201jhhkjxtrvnzhf5/cia5rhes201jihkjxers53yxq/cia5rhes201jjhkjxw8nisucr", + "https://example.com/cia5rhes301jkhkjx7rpx2kp1/cia5rhes301jlhkjxa3840rux/cia5rhes301jmhkjx1943602l/cia5rhes301jnhkjxft42idno", + "https://example.com/cia5rhes301johkjxoa2e62n8/cia5rhes301jphkjx8jfoflvc/cia5rhes301jqhkjxjt9rh0u6/cia5rhes301jrhkjxofoa9vq2", + "https://example.com/cia5rhes301jshkjxnm4wl4ab/cia5rhes301jthkjx89ehf3ty/cia5rhes301juhkjxwubr4oap/cia5rhes301jvhkjxk2e8yz43", + "https://example.com/cia5rhes301jwhkjx4bo3r583/cia5rhes301jxhkjxfmbottug/cia5rhes301jyhkjx86nc7v6s/cia5rhes301jzhkjx9h9x7167", + "https://example.com/cia5rhes301k0hkjx0oc98odu/cia5rhes301k1hkjxdjynl4c1/cia5rhes301k2hkjxc471ye9i/cia5rhes301k3hkjxcawvse26", + "https://example.com/cia5rhes301k4hkjxsqss9ydm/cia5rhes301k5hkjx9e2cz05j/cia5rhes301k6hkjxjb18jpjj/cia5rhes301k7hkjxai3k1edl", + "https://example.com/cia5rhes301k8hkjxeqdbbtwd/cia5rhes301k9hkjxeliovzfz/cia5rhes301kahkjxt8kvwaw8/cia5rhes301kbhkjx334ytlc2", + "https://example.com/cia5rhes301kchkjxl1lize37/cia5rhes301kdhkjxczqjsftr/cia5rhes301kehkjxercwjrhh/cia5rhes301kfhkjxdeb3fvgv", + "https://example.com/cia5rhes301kghkjx0yk4gm2e/cia5rhes301khhkjxt4gdd4ly/cia5rhes301kihkjxegsqzb2u/cia5rhes301kjhkjxp1cug6e2", + "https://example.com/cia5rhes301kkhkjxyhe6rxl6/cia5rhes301klhkjxqfebfzea/cia5rhes301kmhkjxrz0qs4pq/cia5rhes301knhkjxyskiwz5y", + "https://example.com/cia5rhes301kohkjx3tpgvarg/cia5rhes301kphkjx19vihidz/cia5rhes301kqhkjxos60mu4k/cia5rhes301krhkjxbvenxr93", + "https://example.com/cia5rhes301kshkjx9ysyvjir/cia5rhes301kthkjxrk2z4v9t/cia5rhes301kuhkjxitxi78qg/cia5rhes301kvhkjx6m0cf7dl", + "https://example.com/cia5rhes301kwhkjxt4f6hr4z/cia5rhes301kxhkjxilxbilms/cia5rhes301kyhkjxotkf8aaj/cia5rhes301kzhkjx7czn8fdy", + "https://example.com/cia5rhes301l0hkjxf9jhzc7i/cia5rhes301l1hkjx1by7b0y3/cia5rhes301l2hkjxoo8obxiq/cia5rhes301l3hkjxvoc40tkj", + "https://example.com/cia5rhes301l4hkjxgjpxmlpv/cia5rhes301l5hkjx94yuj664/cia5rhes301l6hkjxr8e8y97y/cia5rhes301l7hkjxwznfxlhr", + "https://example.com/cia5rhes301l8hkjxif8hgss9/cia5rhes301l9hkjxls026lu2/cia5rhes301lahkjx8g221cqp/cia5rhes301lbhkjx5nnfkl1o", + "https://example.com/cia5rhes301lchkjx55outsg7/cia5rhes301ldhkjxa32ta3im/cia5rhes301lehkjxqzx1v4ag/cia5rhes301lfhkjxzs4h9iq8", + "https://example.com/cia5rhes301lghkjx9zymf1is/cia5rhes301lhhkjxxi8tt4p0/cia5rhes301lihkjxwsqjsjxe/cia5rhes301ljhkjxa2tfzt6w", + "https://example.com/cia5rhes301lkhkjxs9t7x1x9/cia5rhes301llhkjxo81fsok6/cia5rhes301lmhkjxgi0i3j3b/cia5rhes301lnhkjx5i1k3l6t", + "https://example.com/cia5rhes301lohkjxj1t98ds7/cia5rhes301lphkjxdee7ecco/cia5rhes301lqhkjxgfslix18/cia5rhes301lrhkjx4w0teefo", + "https://example.com/cia5rhes301lshkjxghppkl49/cia5rhes301lthkjx7b8lqwg7/cia5rhes301luhkjx9a10fkrm/cia5rhes301lvhkjxbbotm5de", + "https://example.com/cia5rhes301lwhkjxmjdqwoun/cia5rhes301lxhkjxqscqyygp/cia5rhes301lyhkjx9v1twpxt/cia5rhes301lzhkjxedovrwz9", + "https://example.com/cia5rhes301m0hkjxa79l057t/cia5rhes301m1hkjxi4lf4pam/cia5rhes301m2hkjxbc54aj2i/cia5rhes301m3hkjxh0uiocv9", + "https://example.com/cia5rhes301m4hkjxehur1yoh/cia5rhes301m5hkjxa360j1dg/cia5rhes301m6hkjxxu566hbq/cia5rhes301m7hkjxx3ynzmno", + "https://example.com/cia5rhes301m8hkjxboi8g565/cia5rhes301m9hkjxcw8d20dp/cia5rhes301mahkjxgep6vvnb/cia5rhes301mbhkjxaig0kixq", + "https://example.com/cia5rhes301mchkjxrjfsemox/cia5rhes301mdhkjxwifr2cdy/cia5rhes301mehkjx0mfczty9/cia5rhes301mfhkjxw0d7du38", + "https://example.com/cia5rhes301mghkjxma95b0fw/cia5rhes301mhhkjxe08g59uf/cia5rhes301mihkjx6uflwnsd/cia5rhes301mjhkjxrwodr62q", + "https://example.com/cia5rhes301mkhkjx6rb2vspf/cia5rhes301mlhkjx95l13vr9/cia5rhes301mmhkjx8nf0whp6/cia5rhes301mnhkjxpa5k4qfz", + "https://example.com/cia5rhes301mohkjx3lbw4jvc/cia5rhes301mphkjx80vx4999/cia5rhes301mqhkjxyyl6bvqt/cia5rhes301mrhkjxq4xrjjqk", + "https://example.com/cia5rhes301mshkjx8bf0phng/cia5rhes301mthkjxorxsclwf/cia5rhes301muhkjxi92z3o3d/cia5rhes301mvhkjxc3pvc5j6", + "https://example.com/cia5rhes301mwhkjxm1nnxnhb/cia5rhes301mxhkjxtkkdy3i1/cia5rhes301myhkjxcrbfcl0b/cia5rhes301mzhkjxq6p026u3", + "https://example.com/cia5rhes301n0hkjx1c2gv11c/cia5rhes301n1hkjxxfi36cpe/cia5rhes301n2hkjx38o7jvti/cia5rhes301n3hkjx9x9p0qh6", + "https://example.com/cia5rhes301n4hkjx04xmiymb/cia5rhes301n5hkjx1bimz4eh/cia5rhes301n6hkjxnjqswkw6/cia5rhes301n7hkjxx11qd98z", + "https://example.com/cia5rhes301n8hkjxdgkuvg3u/cia5rhes301n9hkjx9408l8sy/cia5rhes301nahkjxbdy48hsf/cia5rhes301nbhkjx9gbl5y30", + "https://example.com/cia5rhes301nchkjx4rj2l6gk/cia5rhes301ndhkjxw603iycn/cia5rhes301nehkjxq9p4xm5r/cia5rhes301nfhkjx17hqhk81", + "https://example.com/cia5rhes301nghkjxm9macc2i/cia5rhes301nhhkjxubpvluxn/cia5rhes301nihkjxtznh1gve/cia5rhes301njhkjxwft7i8zr", + "https://example.com/cia5rhes301nkhkjxf33xqqss/cia5rhes301nlhkjxt2nsxi7y/cia5rhes301nmhkjxx1k3jzgs/cia5rhes301nnhkjx377meb7x", + "https://example.com/cia5rhes301nohkjx1iur2w22/cia5rhes301nphkjxrj1q40j2/cia5rhes301nqhkjxpv78uwi7/cia5rhes301nrhkjx74y43ako", + "https://example.com/cia5rhes301nshkjxgqi4066n/cia5rhes301nthkjxzeax16t1/cia5rhes301nuhkjxkus1cy9e/cia5rhes301nvhkjxrk8s23la", + "https://example.com/cia5rhes301nwhkjxtz5i4jno/cia5rhes301nxhkjxnve6r7to/cia5rhes301nyhkjxoy3cq981/cia5rhes301nzhkjxsteraq5a", + "https://example.com/cia5rhes301o0hkjx0bjvkfri/cia5rhes301o1hkjxl3tpcl1b/cia5rhes301o2hkjxih6vk4ck/cia5rhes301o3hkjxh0vrl561", + "https://example.com/cia5rhes301o4hkjxmqogfrad/cia5rhes301o5hkjx9m8hdpcc/cia5rhes301o6hkjxaluh3wcr/cia5rhes301o7hkjx4m8wommz", + "https://example.com/cia5rhes301o8hkjxml1xa1az/cia5rhes301o9hkjxx678ystu/cia5rhes301oahkjxndq6nh65/cia5rhes301obhkjxadfjm3wa", + "https://example.com/cia5rhes301ochkjxwz6f2spm/cia5rhes301odhkjxgvmfaaq8/cia5rhes301oehkjxt17j08ud/cia5rhes301ofhkjxneg64ahh", + "https://example.com/cia5rhes301oghkjx2odplosg/cia5rhes301ohhkjx6lsxvvhc/cia5rhes301oihkjx1zjlr3lf/cia5rhes301ojhkjx4too7ovk", + "https://example.com/cia5rhes301okhkjx5ie6svqi/cia5rhes301olhkjx1dvra0d8/cia5rhes301omhkjx01eottp8/cia5rhes301onhkjx0k4cthfm", + "https://example.com/cia5rhes301oohkjx2uggrotk/cia5rhes301ophkjxo0nc672k/cia5rhes301oqhkjxyxv3yip2/cia5rhes301orhkjx1lzdi04w", + "https://example.com/cia5rhes301oshkjx239gzsvl/cia5rhes301othkjxegmfaqs4/cia5rhes301ouhkjx3k7u7klw/cia5rhes301ovhkjxx0w3i22n", + "https://example.com/cia5rhes301owhkjx43szuyvt/cia5rhes301oxhkjxwn8rt15b/cia5rhes301oyhkjxn9plrtrh/cia5rhes301ozhkjx939j8ua7", + "https://example.com/cia5rhes301p0hkjx933v5a7c/cia5rhes301p1hkjxnptb4syc/cia5rhes301p2hkjxlbdlt4c7/cia5rhes301p3hkjxdnx9ndcb", + "https://example.com/cia5rhes301p4hkjxgwkvdwyk/cia5rhes301p5hkjxu5l7j6a8/cia5rhes301p6hkjx69gflmy6/cia5rhes301p7hkjxl0ebaafj", + "https://example.com/cia5rhes301p8hkjxf8355jja/cia5rhes301p9hkjxynm9lc74/cia5rhes301pahkjx9gj8htwg/cia5rhes301pbhkjx9dnwyvr7", + "https://example.com/cia5rhes401pchkjxvc8103ko/cia5rhes401pdhkjxdvj8k8ys/cia5rhes401pehkjx1yvwz1t3/cia5rhes401pfhkjx9tdmf2pk", + "https://example.com/cia5rhes401pghkjxm2moyuwg/cia5rhes401phhkjx6sd8pxql/cia5rhes401pihkjx0f56qfr0/cia5rhes401pjhkjxe09zm4ee", + "https://example.com/cia5rhes401pkhkjxmw6xikqs/cia5rhes401plhkjxtrw32c5n/cia5rhes401pmhkjx2gs5r2uw/cia5rhes401pnhkjx3lnqjuvy", + "https://example.com/cia5rhes401pohkjx18h593mm/cia5rhes401pphkjx74f4atkm/cia5rhes401pqhkjxl804wbka/cia5rhes401prhkjxvjq32png", + "https://example.com/cia5rhes401pshkjxq7ig2fmw/cia5rhes401pthkjx94834nui/cia5rhes401puhkjxg5h7u1tk/cia5rhes401pvhkjx83fsa82j", + "https://example.com/cia5rhes401pwhkjxslfaan9d/cia5rhes401pxhkjx5qqbf367/cia5rhes401pyhkjx9uafkt0z/cia5rhes401pzhkjxyk4qxvdq", + "https://example.com/cia5rhes401q0hkjxxovjahis/cia5rhes401q1hkjx7811zjvy/cia5rhes401q2hkjx87k6qna2/cia5rhes401q3hkjxoj0w4dpu", + "https://example.com/cia5rhes401q4hkjxln1jw5x1/cia5rhes401q5hkjxrh7gm7b7/cia5rhes401q6hkjx7r2y10bk/cia5rhes401q7hkjxhqkthpq6", + "https://example.com/cia5rhes401q8hkjx6u394gyd/cia5rhes401q9hkjxrtrhrat9/cia5rhes401qahkjxdt7xqdcp/cia5rhes401qbhkjxm5ymdwfi", + "https://example.com/cia5rhes401qchkjx025wiukn/cia5rhes401qdhkjxpovs1w4l/cia5rhes401qehkjxjdc5rv3v/cia5rhes401qfhkjxe3c0v82a", + "https://example.com/cia5rhes401qghkjxzhl0kyt9/cia5rhes401qhhkjxx91x3w69/cia5rhes401qihkjxsldvc9au/cia5rhes401qjhkjxnag09g7f", + "https://example.com/cia5rhes401qkhkjxp81l6si9/cia5rhes401qlhkjxdzg4648q/cia5rhes401qmhkjx5rysqo3m/cia5rhes401qnhkjxquhuyu1t", + "https://example.com/cia5rhes401qohkjxsko3ojrg/cia5rhes401qphkjxvda749pk/cia5rhes401qqhkjxudg42xak/cia5rhes401qrhkjx7edixyt2", + "https://example.com/cia5rhes401qshkjx9jc7o9ik/cia5rhes401qthkjxvhwfd027/cia5rhes401quhkjx22ja7ygg/cia5rhes401qvhkjxx2yc25pu", + "https://example.com/cia5rhes401qwhkjxs1yqlawy/cia5rhes401qxhkjxap6eqaza/cia5rhes401qyhkjxhbq9zkww/cia5rhes401qzhkjx5j6rm35k", + "https://example.com/cia5rhes401r0hkjxk8f23od8/cia5rhes401r1hkjxf2hw9jtn/cia5rhes401r2hkjx0dcvkzlo/cia5rhes401r3hkjxqgiol3kt", + "https://example.com/cia5rhes401r4hkjxk8rzt66b/cia5rhes401r5hkjx4zc092sq/cia5rhes401r6hkjxdkgh2lu3/cia5rhes401r7hkjxrxdp47yk", + "https://example.com/cia5rhes401r8hkjxl08yep62/cia5rhes401r9hkjx1xzzdt21/cia5rhes401rahkjxl1d6b0c6/cia5rhes401rbhkjx6zyydco5", + "https://example.com/cia5rhes401rchkjx76v07kx8/cia5rhes401rdhkjxr6p5yan5/cia5rhes401rehkjxx6g8s1x3/cia5rhes401rfhkjxjwzjn0xv", + "https://example.com/cia5rhes401rghkjxhiae442a/cia5rhes401rhhkjx28xtp7j6/cia5rhes401rihkjxqsail6xh/cia5rhes401rjhkjxr7kiu6ki", + "https://example.com/cia5rhes401rkhkjxqne03tot/cia5rhes401rlhkjxjhxcypey/cia5rhes401rmhkjxsma2ekxx/cia5rhes401rnhkjx02z0sp28", + "https://example.com/cia5rhes401rohkjxnrlforbh/cia5rhes401rphkjxuuk7smlp/cia5rhes401rqhkjxp387ih60/cia5rhes401rrhkjxxn9o7q8q", + "https://example.com/cia5rhes401rshkjxh4lkmqca/cia5rhes401rthkjx0jm2hnqp/cia5rhes401ruhkjxeo73uqck/cia5rhes401rvhkjxjbn6t4yy", + "https://example.com/cia5rhes401rwhkjxzepyozzy/cia5rhes401rxhkjx1pykuc1m/cia5rhes401ryhkjxpgqmomw4/cia5rhes401rzhkjx3zbmunev", + "https://example.com/cia5rhes401s0hkjxjbbqwl70/cia5rhes401s1hkjx9xuj3zqs/cia5rhes401s2hkjxafsii503/cia5rhes401s3hkjxuu216w98", + "https://example.com/cia5rhes401s4hkjxbbt02xp1/cia5rhes401s5hkjx1wbhsus2/cia5rhes401s6hkjx1ml5tjx2/cia5rhes401s7hkjxmxzwknq3", + "https://example.com/cia5rhes401s8hkjxjna2smh1/cia5rhes401s9hkjxxqoxe1xs/cia5rhes401sahkjxta8cres1/cia5rhes401sbhkjxlobgkg5k", + "https://example.com/cia5rhes401schkjx4a93mw54/cia5rhes401sdhkjx11zbb5rf/cia5rhes401sehkjxztk9dbrf/cia5rhes401sfhkjxgh3yzmo1", + "https://example.com/cia5rhes401sghkjxc28fo8pm/cia5rhes401shhkjx94mcy08n/cia5rhes401sihkjxk9c7sc1a/cia5rhes401sjhkjx2kpauvo7", + "https://example.com/cia5rhes401skhkjxil6rkwln/cia5rhes401slhkjx7rw9fbmh/cia5rhes401smhkjxp1azo0ra/cia5rhes401snhkjxjn6ske3g", + "https://example.com/cia5rhes401sohkjxld97hjxq/cia5rhes401sphkjxw88rub86/cia5rhes401sqhkjxdepedlux/cia5rhes401srhkjxtz9wqykr", + "https://example.com/cia5rhes401sshkjx23nj0gw3/cia5rhes401sthkjxamuty3aa/cia5rhes401suhkjxkzkkksxw/cia5rhes401svhkjxfy7t55xc", + "https://example.com/cia5rhes401swhkjx2fv1t47w/cia5rhes401sxhkjx493ijzth/cia5rhes401syhkjxlt98ctc5/cia5rhes401szhkjxcgaqy6ks", + "https://example.com/cia5rhes401t0hkjxibyaz6lz/cia5rhes401t1hkjxajqz3b7v/cia5rhes401t2hkjxutdwnzqk/cia5rhes401t3hkjxqr2zpknp", + "https://example.com/cia5rhes401t4hkjx2me8lthv/cia5rhes401t5hkjxej9j1ggl/cia5rhes401t6hkjxoxxbptsl/cia5rhes401t7hkjx31lkyc9v", + "https://example.com/cia5rhes401t8hkjxljaq5d24/cia5rhes401t9hkjxcj9ozsjc/cia5rhes401tahkjx45acwqjh/cia5rhes401tbhkjxsfepsuqn", + "https://example.com/cia5rhes401tchkjx2d54v2mc/cia5rhes401tdhkjxg995kn83/cia5rhes401tehkjx3sa4rnpk/cia5rhes401tfhkjx9p5zj2fw", + "https://example.com/cia5rhes401tghkjxetiwdot9/cia5rhes401thhkjxdzft6ee5/cia5rhes401tihkjxc44k574p/cia5rhes401tjhkjxhlaamwjt", + "https://example.com/cia5rhes401tkhkjxfhcebmkr/cia5rhes401tlhkjx6d2ahwy8/cia5rhes401tmhkjxnvqrt43n/cia5rhes401tnhkjx6y3x0tl6", + "https://example.com/cia5rhes401tohkjxm76vc3bd/cia5rhes401tphkjxe4toa8ix/cia5rhes401tqhkjx44k31o69/cia5rhes401trhkjx06h29ag1", + "https://example.com/cia5rhes401tshkjx0en3ww5b/cia5rhes401tthkjxj0sbg5rs/cia5rhes401tuhkjx4mbcemrx/cia5rhes401tvhkjxzah5kckz", + "https://example.com/cia5rhes401twhkjxbc8vo2b5/cia5rhes401txhkjx1quodrlw/cia5rhes401tyhkjx5q10omzn/cia5rhes401tzhkjxhoknv1pd", + "https://example.com/cia5rhes401u0hkjxyboul8es/cia5rhes401u1hkjxb2vn5wu5/cia5rhes401u2hkjxat1dog9k/cia5rhes401u3hkjxg9cpxurx", + "https://example.com/cia5rhes401u4hkjxcy69r1cg/cia5rhes401u5hkjxakh0jykj/cia5rhes401u6hkjxpsaz87je/cia5rhes401u7hkjx3ujs32jl", + "https://example.com/cia5rhes401u8hkjxogdqi93b/cia5rhes401u9hkjxh0e8e5it/cia5rhes401uahkjxcypc72ho/cia5rhes401ubhkjx07fholph", + "https://example.com/cia5rhes401uchkjx96q3s4y9/cia5rhes401udhkjxbw6z849k/cia5rhes401uehkjxhbtqh3g4/cia5rhes401ufhkjxbp1hjydk", + "https://example.com/cia5rhes401ughkjxq6z30rsc/cia5rhes401uhhkjxgnc7011n/cia5rhes401uihkjx00l0t29g/cia5rhes401ujhkjxpdkefo86", + "https://example.com/cia5rhes401ukhkjx5w5u4uez/cia5rhes401ulhkjxkp60rcm2/cia5rhes401umhkjx2o152chr/cia5rhes401unhkjxj1c837fv", + "https://example.com/cia5rhes401uohkjxkm3hwgxw/cia5rhes401uphkjxr9fpwgxo/cia5rhes401uqhkjxbju1cc6a/cia5rhes401urhkjxnyjsugye", + "https://example.com/cia5rhes401ushkjxnl9fzmwd/cia5rhes401uthkjx829ud4hl/cia5rhes401uuhkjxgzo6bd97/cia5rhes401uvhkjxninvqfmi", + "https://example.com/cia5rhes401uwhkjx23xkeeyb/cia5rhes401uxhkjxr7f81k32/cia5rhes401uyhkjxu8gwxp2s/cia5rhes401uzhkjx0zbojk5h", + "https://example.com/cia5rhes401v0hkjxnp3m2er4/cia5rhes401v1hkjxh6zxquzd/cia5rhes401v2hkjxcp9r8512/cia5rhes401v3hkjxfj2ziffr", + "https://example.com/cia5rhes401v4hkjx450sdsy6/cia5rhes401v5hkjxid7nsxhs/cia5rhes401v6hkjx5umhcl29/cia5rhes401v7hkjx8c4ntx9f", + "https://example.com/cia5rhes401v8hkjxm7493idl/cia5rhes401v9hkjxvp3boxa7/cia5rhes401vahkjxpdhxc0bd/cia5rhes401vbhkjxjq8g7bbv", + "https://example.com/cia5rhes401vchkjxutbrig4f/cia5rhes401vdhkjxs6v3l5bs/cia5rhes401vehkjxz0s7ot2j/cia5rhes401vfhkjx6e86cpuy", + "https://example.com/cia5rhes401vghkjxn3ce11di/cia5rhes401vhhkjx0lgmp1co/cia5rhes401vihkjxgjby3l0n/cia5rhes401vjhkjxh7bj2rti", + "https://example.com/cia5rhes401vkhkjxq9xd82bm/cia5rhes401vlhkjxs2x6daye/cia5rhes401vmhkjxnv72qdm3/cia5rhes401vnhkjxjuu4sj2i", + "https://example.com/cia5rhes401vohkjxf8wvg4tv/cia5rhes401vphkjxus1ibfvl/cia5rhes401vqhkjxaapjjznh/cia5rhes401vrhkjxhpfk9ana", + "https://example.com/cia5rhes401vshkjxb314pxv2/cia5rhes401vthkjxcfenzpqi/cia5rhes401vuhkjxvbef4uzt/cia5rhes401vvhkjxcg2mtju1", + "https://example.com/cia5rhes401vwhkjxobjolxrt/cia5rhes401vxhkjx8n6q1mbj/cia5rhes401vyhkjx1ffiobsm/cia5rhes401vzhkjx845f4yrb", + "https://example.com/cia5rhes501w0hkjxnkd5nvx4/cia5rhes501w1hkjxd10zyp5f/cia5rhes501w2hkjxed1isr4c/cia5rhes501w3hkjx52w25p6h", + "https://example.com/cia5rhes501w4hkjxz590qwcl/cia5rhes501w5hkjxirypp5am/cia5rhes501w6hkjx0la9fxfb/cia5rhes501w7hkjxqn8mmj3v", + "https://example.com/cia5rhes501w8hkjxtvynj745/cia5rhes501w9hkjxicsfn3ft/cia5rhes501wahkjxawuf4y2u/cia5rhes501wbhkjxuioyhj09", + "https://example.com/cia5rhes501wchkjxpzp5z6gl/cia5rhes501wdhkjxqyu6yfrv/cia5rhes501wehkjxksohpokd/cia5rhes501wfhkjxocde2wt3", + "https://example.com/cia5rhes501wghkjxocbfchks/cia5rhes501whhkjx0tpiw1nt/cia5rhes501wihkjxslhtkvr0/cia5rhes501wjhkjx3f2wxmki", + "https://example.com/cia5rhes501wkhkjxpmltynvl/cia5rhes501wlhkjxig3aj85x/cia5rhes501wmhkjxplefjg23/cia5rhes501wnhkjxanfao1fs", + "https://example.com/cia5rhes501wohkjx9gnuza2e/cia5rhes501wphkjxi89ym1sn/cia5rhes501wqhkjxmpb91ix0/cia5rhes501wrhkjx6vdiefye", + "https://example.com/cia5rhes501wshkjxxy1dl1w5/cia5rhes501wthkjxs40731ag/cia5rhes501wuhkjx5tu8ptk3/cia5rhes501wvhkjxb83m364e", + "https://example.com/cia5rhes501wwhkjxyxi7zia4/cia5rhes501wxhkjxfttjkfl1/cia5rhes501wyhkjx73a609nu/cia5rhes501wzhkjxhrqkcsc9", + "https://example.com/cia5rhes501x0hkjxpzoc18gx/cia5rhes501x1hkjx2evbj8dh/cia5rhes501x2hkjxcmt0dte5/cia5rhes501x3hkjxs2o08cdn", + "https://example.com/cia5rhes501x4hkjxn8ppwkl6/cia5rhes501x5hkjxve994e14/cia5rhes501x6hkjxp3nroxzg/cia5rhes501x7hkjxdzh6iphg", + "https://example.com/cia5rhes501x8hkjx3dxj6rdf/cia5rhes501x9hkjx0uek477t/cia5rhes501xahkjxyomgqdjw/cia5rhes501xbhkjx0adcgz3e", + "https://example.com/cia5rhes501xchkjxjr38fuho/cia5rhes501xdhkjxi9h8gxgv/cia5rhes501xehkjx9lnq5x48/cia5rhes501xfhkjx6x7q34qn", + "https://example.com/cia5rhes501xghkjx7kdv4j16/cia5rhes501xhhkjxzh3h1621/cia5rhes501xihkjx2tll48zr/cia5rhes501xjhkjx2mgqrjx7", + "https://example.com/cia5rhes501xkhkjxb38ktam2/cia5rhes501xlhkjxqe4kly98/cia5rhes501xmhkjxs8kc7y4g/cia5rhes501xnhkjxon8hd6t9", + "https://example.com/cia5rhes501xohkjxvh07sqak/cia5rhes501xphkjxa8cjku7k/cia5rhes501xqhkjx7czbmtzz/cia5rhes501xrhkjx2v5gm68q", + "https://example.com/cia5rhes501xshkjxhafeuujz/cia5rhes501xthkjx83z1ik2e/cia5rhes501xuhkjxfwfbdp20/cia5rhes501xvhkjxat92izys", + "https://example.com/cia5rhes501xwhkjxmxkavbl1/cia5rhes501xxhkjxjmwfaudp/cia5rhes501xyhkjxjb6y5ckv/cia5rhes501xzhkjxx1qzw43n", + "https://example.com/cia5rhes501y0hkjxjqq91tnx/cia5rhes501y1hkjx5fqvt95y/cia5rhes501y2hkjx213i79od/cia5rhes501y3hkjx2bhrwh3c", + "https://example.com/cia5rhes501y4hkjxpg0w8tm1/cia5rhes501y5hkjx4rfqmukn/cia5rhes501y6hkjxdmm6zlwo/cia5rhes501y7hkjxuuszpo6e", + "https://example.com/cia5rhes501y8hkjx1dijvw0o/cia5rhes501y9hkjxh1co3ai2/cia5rhes501yahkjxfcerdd6h/cia5rhes501ybhkjx52yrnztr", + "https://example.com/cia5rhes501ychkjxmuxdm6gn/cia5rhes501ydhkjxk9cu7gzp/cia5rhes501yehkjxt9czxhe8/cia5rhes501yfhkjxf1hpxe7k", + "https://example.com/cia5rhes501yghkjxzyfsx9ee/cia5rhes501yhhkjxoobntt4j/cia5rhes501yihkjxbv4l41i4/cia5rhes501yjhkjx9cg8i6yq", + "https://example.com/cia5rhes501ykhkjxb8micj52/cia5rhes501ylhkjxi810y8kg/cia5rhes501ymhkjx35pqd2dp/cia5rhes501ynhkjx411ay6w2", + "https://example.com/cia5rhes501yohkjxp230m8o4/cia5rhes501yphkjx85aei3f0/cia5rhes501yqhkjx39awmvdg/cia5rhes501yrhkjxabhea8z7", + "https://example.com/cia5rhes501yshkjxmy4w0zr0/cia5rhes501ythkjxxxtlmezs/cia5rhes501yuhkjx8mwm07hi/cia5rhes501yvhkjx1l5p3sr0", + "https://example.com/cia5rhes501ywhkjxdrcc28nn/cia5rhes501yxhkjxqcqd6ogs/cia5rhes501yyhkjxim858nwj/cia5rhes501yzhkjxpi5u68xr", + "https://example.com/cia5rhes501z0hkjxgxk8ryu8/cia5rhes501z1hkjx7jqdu67h/cia5rhes501z2hkjx21zj3rmt/cia5rhes501z3hkjxcq1lwavz", + "https://example.com/cia5rhes501z4hkjxgx1266ef/cia5rhes501z5hkjxi6uyr5et/cia5rhes501z6hkjxhr0eot9n/cia5rhes501z7hkjxyz2oyzjs", + "https://example.com/cia5rhes501z8hkjx5s5s200w/cia5rhes501z9hkjx67v1yb2z/cia5rhes501zahkjxw5u36sb5/cia5rhes501zbhkjxl17xibdr", + "https://example.com/cia5rhes501zchkjxpx05d6o1/cia5rhes501zdhkjxiiadtum2/cia5rhes501zehkjxoj9i56gl/cia5rhes501zfhkjxqcxmjy73", + "https://example.com/cia5rhes501zghkjxegc7tvdy/cia5rhes501zhhkjxqeeoq63e/cia5rhes501zihkjxysrggeqs/cia5rhes501zjhkjxf24x4w8j", + "https://example.com/cia5rhes501zkhkjx36w5g359/cia5rhes501zlhkjxuornb7pf/cia5rhes501zmhkjx4pvpci2q/cia5rhes501znhkjxbv1oa4fp", + "https://example.com/cia5rhes501zohkjxb6t1a9pz/cia5rhes501zphkjxg5ezhfdv/cia5rhes501zqhkjxl3efud9l/cia5rhes501zrhkjxcqb7r2sc", + "https://example.com/cia5rhes501zshkjxd7wcvoav/cia5rhes501zthkjxelhdxd7w/cia5rhes501zuhkjxh07pf32p/cia5rhes501zvhkjxgcxn3nvl", + "https://example.com/cia5rhes501zwhkjx95ri5zb5/cia5rhes501zxhkjxci9sujxb/cia5rhes501zyhkjx1hzc65ou/cia5rhes501zzhkjxf1kbgic9", + "https://example.com/cia5rhes50200hkjxphxlxmld/cia5rhes50201hkjx0sveusk8/cia5rhes50202hkjxg5822asq/cia5rhes50203hkjxxle2qnr4", + "https://example.com/cia5rhes50204hkjxswna3iww/cia5rhes50205hkjxo41y7z2t/cia5rhes50206hkjx1auwgf30/cia5rhes50207hkjx3vyiy15y", + "https://example.com/cia5rhes50208hkjx6n640dxz/cia5rhes50209hkjxxb3tliuh/cia5rhes5020ahkjxht8vaioj/cia5rhes5020bhkjxqjo5gr27", + "https://example.com/cia5rhes5020chkjxh9wu9gbv/cia5rhes5020dhkjxbrv63660/cia5rhes5020ehkjxbmozonad/cia5rhes5020fhkjxsek9b1wa", + "https://example.com/cia5rhes5020ghkjxrlfea9iv/cia5rhes5020hhkjxt7qh369y/cia5rhes5020ihkjxkn7yslxt/cia5rhes5020jhkjx2ge4xq51", + "https://example.com/cia5rhes5020khkjx2sp9c2gt/cia5rhes5020lhkjx1ks9juca/cia5rhes5020mhkjxrova7tax/cia5rhes5020nhkjxnaxah6tg", + "https://example.com/cia5rhes5020ohkjx9btins8g/cia5rhes5020phkjxy4or4s6u/cia5rhes5020qhkjxxrqcpd3n/cia5rhes5020rhkjxm6xw3z2x", + "https://example.com/cia5rhes5020shkjxz31fkpjb/cia5rhes5020thkjxsxivj1tx/cia5rhes5020uhkjx218dg3oe/cia5rhes5020vhkjxpxflwg9k", + "https://example.com/cia5rhes5020whkjx3xpogsrh/cia5rhes5020xhkjxv5k6yvhb/cia5rhes5020yhkjxmg5wu4xg/cia5rhes5020zhkjx49u1376r", + "https://example.com/cia5rhes50210hkjxu07iog9j/cia5rhes50211hkjxe2zq097b/cia5rhes50212hkjx7d2n5bis/cia5rhes50213hkjx98z0f1wd", + "https://example.com/cia5rhes50214hkjxxz2fxal3/cia5rhes50215hkjx4cdss157/cia5rhes50216hkjxgemb403b/cia5rhes50217hkjxcx1to7hv", + "https://example.com/cia5rhes50218hkjxlm8ctocp/cia5rhes50219hkjx1fcacxy3/cia5rhes5021ahkjxx59gdemf/cia5rhes5021bhkjxa8w89mbs", + "https://example.com/cia5rhes5021chkjxgbgtxsby/cia5rhes5021dhkjxpsb7jlci/cia5rhes5021ehkjxo8ytwukr/cia5rhes5021fhkjxtpoy84xh", + "https://example.com/cia5rhes5021ghkjxyk2hucae/cia5rhes5021hhkjxyiywhstb/cia5rhes5021ihkjx1sdmxxsc/cia5rhes5021jhkjxp5btccgt", + "https://example.com/cia5rhes5021khkjxav298li6/cia5rhes5021lhkjx4ba0mhnf/cia5rhes5021mhkjxngkomyhl/cia5rhes5021nhkjxxtqqmtir", + "https://example.com/cia5rhes5021ohkjxbavqb4tz/cia5rhes5021phkjx1f18irux/cia5rhes5021qhkjxgef61ilr/cia5rhes5021rhkjxeh1y04kj", + "https://example.com/cia5rhes5021shkjxr4s9i0ob/cia5rhes5021thkjxfocdh5vi/cia5rhes5021uhkjxjcwajris/cia5rhes5021vhkjxitwdjshb", + "https://example.com/cia5rhes5021whkjxwhlm3an5/cia5rhes5021xhkjx5dcoj15s/cia5rhes5021yhkjxy9biyupr/cia5rhes5021zhkjx6wit7c1p", + "https://example.com/cia5rhes50220hkjxco3srhrz/cia5rhes50221hkjxn8kb150i/cia5rhes50222hkjxcfl48mla/cia5rhes50223hkjx5wzddel7", + "https://example.com/cia5rhes50224hkjxv4kbq0bu/cia5rhes50225hkjxdlcujhtv/cia5rhes50226hkjx0nm0ncdj/cia5rhes50227hkjx4hnvg7w9", + "https://example.com/cia5rhes50228hkjxn2hoexz0/cia5rhes50229hkjx5a0zae0n/cia5rhes5022ahkjx7kw3lf0v/cia5rhes5022bhkjx9uaqp2w5", + "https://example.com/cia5rhes5022chkjxmllq37r4/cia5rhes5022dhkjxuogvq5kp/cia5rhes5022ehkjxegsxagw5/cia5rhes5022fhkjx25d5a5z8", + "https://example.com/cia5rhes5022ghkjxwwoecae0/cia5rhes5022hhkjxli8zm9vs/cia5rhes5022ihkjxzxcky0jv/cia5rhes5022jhkjxvsb9g2qa", + "https://example.com/cia5rhes5022khkjxhwpswkll/cia5rhes5022lhkjxow1y1vc4/cia5rhes5022mhkjxh0o8b4r5/cia5rhes5022nhkjxjsyoo9le", + "https://example.com/cia5rhes5022ohkjx50pmnu22/cia5rhes5022phkjxfdh1jhl2/cia5rhes5022qhkjxh67gv4up/cia5rhes5022rhkjxmpux301t", + "https://example.com/cia5rhes5022shkjxmgm2q2tv/cia5rhes5022thkjx7ivn1k01/cia5rhes5022uhkjxs4j1z1st/cia5rhes5022vhkjxh3y1ak61", + "https://example.com/cia5rhes5022whkjxy2vkf9qu/cia5rhes5022xhkjxotujbeup/cia5rhes5022yhkjx5qiu2ujp/cia5rhes5022zhkjxluajf32y", + "https://example.com/cia5rhes50230hkjxk7stw4db/cia5rhes60231hkjxf7aj9i0m/cia5rhes60232hkjxziydwog0/cia5rhes60233hkjxx3x1fbuc", + "https://example.com/cia5rhes60234hkjxg2uqu0ml/cia5rhes60235hkjxq7n4gpgv/cia5rhes60236hkjxolpslbdw/cia5rhes60237hkjxyn1lp5ir", + "https://example.com/cia5rhes60238hkjxwis4nirx/cia5rhes60239hkjxaiqtx5n6/cia5rhes6023ahkjxsgrablt0/cia5rhes6023bhkjxc06147lu", + "https://example.com/cia5rhes6023chkjxxge8xmjn/cia5rhes6023dhkjx5j31jwgd/cia5rhes6023ehkjxwuz388j6/cia5rhes6023fhkjx3pdltokg", + "https://example.com/cia5rhes6023ghkjx6dffsn9x/cia5rhes6023hhkjxzjoqqtor/cia5rhes6023ihkjx3bz79voa/cia5rhes6023jhkjxa7bb04th", + "https://example.com/cia5rhes6023khkjxhg5ub876/cia5rhes6023lhkjxrklzuro9/cia5rhes6023mhkjx8xmhpdqm/cia5rhes6023nhkjxch1jn490", + "https://example.com/cia5rhes6023ohkjxhad7g229/cia5rhes6023phkjx4zaksvdn/cia5rhes6023qhkjxx6ko1cpf/cia5rhes6023rhkjx0vireriy", + "https://example.com/cia5rhes6023shkjxhvae8jtn/cia5rhes6023thkjxw4de6xi4/cia5rhes6023uhkjxzfqht8ml/cia5rhes6023vhkjxs8ul3zvc", + "https://example.com/cia5rhes6023whkjxdsyyu08r/cia5rhes6023xhkjxhddko66j/cia5rhes6023yhkjxnfhgsx6b/cia5rhes6023zhkjxt63bqpbs", + "https://example.com/cia5rhes60240hkjxa7oafjex/cia5rhes60241hkjx74x1e2f3/cia5rhes60242hkjxiaptta0r/cia5rhes60243hkjxingpv6qf", + "https://example.com/cia5rhes60244hkjx832w9v0m/cia5rhes60245hkjxbtb4g19e/cia5rhes60246hkjxahthge6j/cia5rhes60247hkjxhqj3m07o", + "https://example.com/cia5rhes60248hkjxcf7nc4li/cia5rhes60249hkjxyaeee0po/cia5rhes6024ahkjxz0zbl31v/cia5rhes6024bhkjxyli25oi7", + "https://example.com/cia5rhes6024chkjxqymyzh67/cia5rhes6024dhkjx41mtrlwg/cia5rhes6024ehkjxupbohin3/cia5rhes6024fhkjx1wtwax3q", + "https://example.com/cia5rhes6024ghkjxbhnnx8qm/cia5rhes6024hhkjx330f907k/cia5rhes6024ihkjxt8kevs6h/cia5rhes6024jhkjx6fz60hhj", + "https://example.com/cia5rhes6024khkjx6jh6byd0/cia5rhes6024lhkjxnqak5lqd/cia5rhes6024mhkjx6qi3ka0d/cia5rhes6024nhkjxmydiqa1w", + "https://example.com/cia5rhes6024ohkjx1wzyvp8g/cia5rhes6024phkjxcpe4crtr/cia5rhes6024qhkjx5k672peu/cia5rhes6024rhkjxrgc14c0o", + "https://example.com/cia5rhes6024shkjxt3phdd6y/cia5rhes6024thkjxrcolx8rw/cia5rhes6024uhkjx1m8lrl96/cia5rhes6024vhkjx1ub0usjq", + "https://example.com/cia5rhes6024whkjx30q3vye6/cia5rhes6024xhkjxqhicyl5l/cia5rhes6024yhkjxewkiuvcd/cia5rhes6024zhkjxpi0s95q6", + "https://example.com/cia5rhes60250hkjx7x45wchz/cia5rhes60251hkjx29nj5yrn/cia5rhes60252hkjxmjtv4j8t/cia5rhes60253hkjx62flt3ct", + "https://example.com/cia5rhes60254hkjxj24tyltz/cia5rhes60255hkjxu43vfkjt/cia5rhes60256hkjxorb3l17v/cia5rhes60257hkjxuusa9260", + "https://example.com/cia5rhes60258hkjx2mtr4h7o/cia5rhes60259hkjxfni1laoe/cia5rhes6025ahkjxi8p6cxws/cia5rhes6025bhkjxms0v3mvk", + "https://example.com/cia5rhes6025chkjxak2ehrye/cia5rhes6025dhkjxkkwv08j7/cia5rhes6025ehkjxmviua90r/cia5rhes6025fhkjxxz5403tq", + "https://example.com/cia5rhes6025ghkjxw2zi9e42/cia5rhes6025hhkjxcpaquver/cia5rhes6025ihkjxdza15efa/cia5rhes6025jhkjxj10ftcde", + "https://example.com/cia5rhes6025khkjxzdgyklzu/cia5rhes6025lhkjxepec48wo/cia5rhes6025mhkjxrr0rxhsw/cia5rhes6025nhkjxbx5apxib", + "https://example.com/cia5rhes6025ohkjxmw1aiv3f/cia5rhes6025phkjxf2m420e9/cia5rhes6025qhkjxjiwth0yz/cia5rhes6025rhkjxrmxufevy", + "https://example.com/cia5rhes6025shkjxusdiwv01/cia5rhes6025thkjxds425t8m/cia5rhes6025uhkjxuqrtt7if/cia5rhes6025vhkjxowk5zvf3", + "https://example.com/cia5rhes6025whkjxh652j091/cia5rhes6025xhkjxg7n9opan/cia5rhes6025yhkjxhx4aysaj/cia5rhes6025zhkjxu82h4n54", + "https://example.com/cia5rhes60260hkjxi674w0z0/cia5rhes60261hkjxojs9dwc5/cia5rhes60262hkjx9zme8232/cia5rhes60263hkjxg3tduw2q", + "https://example.com/cia5rhes60264hkjxen5f1emm/cia5rhes60265hkjx9wlrydmg/cia5rhes60266hkjxyk0z00l1/cia5rhes60267hkjxim57nlkk", + "https://example.com/cia5rhes60268hkjx0dxjfg9r/cia5rhes60269hkjxvsd7fx55/cia5rhes6026ahkjxr4wv79py/cia5rhes6026bhkjxbtuynf74", + "https://example.com/cia5rhes6026chkjx0hbrlens/cia5rhes6026dhkjx4oarjdzi/cia5rhes6026ehkjxcfh9kh1i/cia5rhes6026fhkjxdvhhj9ps", + "https://example.com/cia5rhes6026ghkjxzbxwxiwi/cia5rhes6026hhkjx10dmy3ck/cia5rhes6026ihkjxrh57qzib/cia5rhes6026jhkjxa6wqf4ro", + "https://example.com/cia5rhes6026khkjxw4rqjhaq/cia5rhes6026lhkjxuc55dmgp/cia5rhes6026mhkjxlv6a6sz0/cia5rhes6026nhkjxwxm1u6cu", + "https://example.com/cia5rhes6026ohkjxcezmtk1t/cia5rhes6026phkjxt8hncf2i/cia5rhes6026qhkjxuxprl91o/cia5rhes6026rhkjx9ujzo2je", + "https://example.com/cia5rhes6026shkjxxutau6ka/cia5rhes6026thkjxa2hy9mje/cia5rhes6026uhkjxr2vho147/cia5rhes6026vhkjx7h70z8i9", + "https://example.com/cia5rhes6026whkjx1nagxk22/cia5rhes6026xhkjxke02jgeq/cia5rhes6026yhkjxhemx0l0x/cia5rhes6026zhkjx8uhw94o4", + "https://example.com/cia5rhes60270hkjxtpo8z0gx/cia5rhes60271hkjxaldlng02/cia5rhes60272hkjxi6u6vyos/cia5rhes60273hkjx8t4gz8q3", + "https://example.com/cia5rhes60274hkjxzetzmgfp/cia5rhes60275hkjxqtd9rh66/cia5rhes60276hkjxo38ak1v6/cia5rhes60277hkjx3t2grzdi", + "https://example.com/cia5rhes60278hkjxssjf92tp/cia5rhes60279hkjxtdiimuwo/cia5rhes6027ahkjxv7i327um/cia5rhes6027bhkjx34iyiwau", + "https://example.com/cia5rhes6027chkjxsalv7vq1/cia5rhes6027dhkjxj1qa0eqe/cia5rhes6027ehkjxdstykpct/cia5rhes6027fhkjxep1lg57f", + "https://example.com/cia5rhes6027ghkjxir6tvp5r/cia5rhes6027hhkjx37mwtxmp/cia5rhes6027ihkjxajh8kdk0/cia5rhes6027jhkjxprxxf6bf", + "https://example.com/cia5rhes6027khkjxtx8rt4eg/cia5rhes6027lhkjx6stckrq2/cia5rhes6027mhkjxbp2scl06/cia5rhes6027nhkjx5tcodm70", + "https://example.com/cia5rhes6027ohkjx02hq4e4i/cia5rhes6027phkjxpj98682x/cia5rhes6027qhkjxi6t9w6j8/cia5rhes6027rhkjxdoo5aitq", + "https://example.com/cia5rhes6027shkjxq61ipcpf/cia5rhes6027thkjx4c95chxk/cia5rhes6027uhkjx5yp65br8/cia5rhes6027vhkjxgaj3cw9t", + "https://example.com/cia5rhes6027whkjxx18if78t/cia5rhes6027xhkjxeruuk14w/cia5rhes6027yhkjxzur0jh40/cia5rhes6027zhkjx2zxmcdyy", + "https://example.com/cia5rhes60280hkjxrh298dzu/cia5rhes60281hkjx5m40ppz3/cia5rhes60282hkjxfak6x0vp/cia5rhes60283hkjxcokmxlit", + "https://example.com/cia5rhes60284hkjx58dts12q/cia5rhes60285hkjx7hgaud95/cia5rhes60286hkjxdycu90lv/cia5rhes60287hkjxjj4cgdk8", + "https://example.com/cia5rhes60288hkjxai7gc5c8/cia5rhes60289hkjxbnomezv6/cia5rhes6028ahkjxw7wxahj2/cia5rhes6028bhkjx1smzie0j", + "https://example.com/cia5rhes6028chkjxa57aiiju/cia5rhes6028dhkjxs1etgvw7/cia5rhes6028ehkjxtsbz6p0z/cia5rhes6028fhkjxmo1vsspv", + "https://example.com/cia5rhes6028ghkjxieobtxp5/cia5rhes6028hhkjx9ragsscj/cia5rhes6028ihkjx385kpk1h/cia5rhes6028jhkjxotj68l1k", + "https://example.com/cia5rhes6028khkjxea5reemm/cia5rhes6028lhkjx0kwzwbyo/cia5rhes6028mhkjx4nqjjcde/cia5rhes6028nhkjxzrrex5ue", + "https://example.com/cia5rhes6028ohkjx7t2lhe7z/cia5rhes6028phkjx46qyubif/cia5rhes6028qhkjxjolbuqus/cia5rhes6028rhkjx8r7ii6z7", + "https://example.com/cia5rhes6028shkjxilpnvd7j/cia5rhes6028thkjxof8m415p/cia5rhes6028uhkjxjp4mywli/cia5rhes6028vhkjxcw58yxw0", + "https://example.com/cia5rhes6028whkjxhya97tqs/cia5rhes6028xhkjxpezwz1pe/cia5rhes6028yhkjxx59c4igt/cia5rhes6028zhkjxdjv35rpr", + "https://example.com/cia5rhes60290hkjxnthanean/cia5rhes60291hkjxni7pjxv4/cia5rhes60292hkjx0flrl74n/cia5rhes60293hkjxm9x63zo7", + "https://example.com/cia5rhes60294hkjxpnfmclsw/cia5rhes60295hkjx56ccc80r/cia5rhes60296hkjx4s91lrwv/cia5rhes60297hkjxf132ofl7", + "https://example.com/cia5rhes60298hkjxl3mctpt0/cia5rhes60299hkjxvlg5nt62/cia5rhes6029ahkjx336mdt5q/cia5rhes6029bhkjxx1be21if", + "https://example.com/cia5rhes6029chkjxo22y49m7/cia5rhes6029dhkjx1llimb0p/cia5rhes6029ehkjxt13ucuxv/cia5rhes6029fhkjxh2xoljln", + "https://example.com/cia5rhes6029ghkjx68wd962d/cia5rhes6029hhkjx387d5swn/cia5rhes6029ihkjxh34aue0p/cia5rhes6029jhkjxfh61fg9l", + "https://example.com/cia5rhes6029khkjxuz53ttqc/cia5rhes6029lhkjxvrp7a6bu/cia5rhes6029mhkjx5ug57g8j/cia5rhes6029nhkjxiv7fjxr3", + "https://example.com/cia5rhes6029ohkjx2im4dkbc/cia5rhes6029phkjxk2vkitw7/cia5rhes6029qhkjx1g18697q/cia5rhes6029rhkjxu7cv0cp5", + "https://example.com/cia5rhes6029shkjxzfgxcfx5/cia5rhes6029thkjx6bi4op1u/cia5rhes6029uhkjx57v7j2tp/cia5rhes6029vhkjxqsn3ros1", + "https://example.com/cia5rhes7029whkjx33b3346i/cia5rhes7029xhkjxnhbvzlyl/cia5rhes7029yhkjxhofpksax/cia5rhes7029zhkjxpckp9le4", + "https://example.com/cia5rhes702a0hkjx6pzs7e5d/cia5rhes702a1hkjxp2x65zqo/cia5rhes702a2hkjxu66pcizj/cia5rhes702a3hkjx7o8r0f06", + "https://example.com/cia5rhes702a4hkjxs3nk500n/cia5rhes702a5hkjxg0rbzm6k/cia5rhes702a6hkjx234c6g7e/cia5rhes702a7hkjx9ocd54xq", + "https://example.com/cia5rhes702a8hkjxhv3xsjpp/cia5rhes702a9hkjxofxw9mdy/cia5rhes702aahkjxwtmyec4h/cia5rhes702abhkjxly8sn8hi", + "https://example.com/cia5rhes702achkjx7zlau40c/cia5rhes702adhkjx6i9t1hdm/cia5rhes702aehkjx3w115jp6/cia5rhes702afhkjx3spdsa1v", + "https://example.com/cia5rhes702aghkjxd4i1f3k7/cia5rhes702ahhkjx1o7338m9/cia5rhes702aihkjx3issv8lp/cia5rhes702ajhkjxkkpxy74s", + "https://example.com/cia5rhes702akhkjxdng2ft24/cia5rhes702alhkjxvf0nimyo/cia5rhes702amhkjxubx3l0hc/cia5rhes702anhkjxjdg78083", + "https://example.com/cia5rhes702aohkjxb6np3w0m/cia5rhes702aphkjxbmp49sgd/cia5rhes702aqhkjx3wm23ff0/cia5rhes702arhkjx9ht9wc86", + "https://example.com/cia5rhes702ashkjxw56jbjfz/cia5rhes702athkjx6js735z5/cia5rhes702auhkjxucfu5lpt/cia5rhes702avhkjxbyglt9ex", + "https://example.com/cia5rhes702awhkjx18s0uu13/cia5rhes702axhkjxi3zrv40h/cia5rhes702ayhkjx3a8cp916/cia5rhes702azhkjxczqrzngo", + "https://example.com/cia5rhes702b0hkjxglj4n5o7/cia5rhes702b1hkjx63bg4kb1/cia5rhes702b2hkjx60relgsi/cia5rhes702b3hkjxiol0e8ym", + "https://example.com/cia5rhes702b4hkjxjfpk1sg5/cia5rhes702b5hkjxk428e7bk/cia5rhes702b6hkjxr97qxcy0/cia5rhes702b7hkjxdyz4rzzn", + "https://example.com/cia5rhes702b8hkjx7vylah33/cia5rhes702b9hkjxinhs95fl/cia5rhes702bahkjxpengba9m/cia5rhes702bbhkjxh5smj013", + "https://example.com/cia5rhes702bchkjxqce1aoab/cia5rhes702bdhkjxaiyf10a3/cia5rhes702behkjx5yqopkqf/cia5rhes702bfhkjx3hiu4jp5", + "https://example.com/cia5rhes702bghkjx27997nof/cia5rhes702bhhkjxh131a1mu/cia5rhes702bihkjxdv7jmcf7/cia5rhes702bjhkjxu56c6np2", + "https://example.com/cia5rhes702bkhkjxqpt1iswl/cia5rhes702blhkjxxvuevm79/cia5rhes702bmhkjxlb6egm5v/cia5rhes702bnhkjx0frya4zv", + "https://example.com/cia5rhes702bohkjx62rqvbxx/cia5rhes702bphkjxn5543qcw/cia5rhes702bqhkjxo6xrcl3m/cia5rhes702brhkjxxiyxytk6", + "https://example.com/cia5rhes702bshkjxtupz79qv/cia5rhes702bthkjx46tmi8da/cia5rhes702buhkjxa076ev9b/cia5rhes702bvhkjxwzgfevcu", + "https://example.com/cia5rhes702bwhkjxwmx0x18a/cia5rhes702bxhkjxpq4el7be/cia5rhes702byhkjxwlypdgqk/cia5rhes702bzhkjxf16uiqj9", + "https://example.com/cia5rhes702c0hkjx0oylz3z7/cia5rhes702c1hkjxnka3undy/cia5rhes702c2hkjx9pvadq7q/cia5rhes702c3hkjxubumi03d", + "https://example.com/cia5rhes702c4hkjxv1je61d0/cia5rhes702c5hkjx3gud1w7h/cia5rhes702c6hkjxhbputn4m/cia5rhes702c7hkjx2fwamiyv", + "https://example.com/cia5rhes702c8hkjxbgkmje13/cia5rhes702c9hkjxlumxva5q/cia5rhes702cahkjxmiet3v1x/cia5rhes702cbhkjx8ibo8t0v", + "https://example.com/cia5rhes702cchkjxyl6aj596/cia5rhes702cdhkjxuk4jdais/cia5rhes702cehkjxznkrhgcf/cia5rhes702cfhkjxedld1xxc", + "https://example.com/cia5rhes702cghkjxc2ry2vt4/cia5rhes702chhkjxahplgyzs/cia5rhes702cihkjxdfgeirre/cia5rhes702cjhkjx5k6zbwnv", + "https://example.com/cia5rhes702ckhkjxt8jo94yh/cia5rhes702clhkjxsjs9l544/cia5rhes702cmhkjxob8bd0zc/cia5rhes702cnhkjx6cfcl3n9", + "https://example.com/cia5rhes702cohkjxb9cd9ogj/cia5rhes702cphkjxpoorw1yg/cia5rhes702cqhkjxykcpxjap/cia5rhes702crhkjx3469lxlp", + "https://example.com/cia5rhes702cshkjxmwi9wm5t/cia5rhes702cthkjx8tmzifvh/cia5rhes702cuhkjx4l68blak/cia5rhes702cvhkjxdxodcgpw", + "https://example.com/cia5rhes702cwhkjx0tbp18xa/cia5rhes702cxhkjxa9e95679/cia5rhes702cyhkjxpunm4oge/cia5rhes702czhkjxsxewphj9", + "https://example.com/cia5rhes702d0hkjx1a2yy8af/cia5rhes702d1hkjx4f2cssht/cia5rhes702d2hkjxa1d631y5/cia5rhes702d3hkjx5isc7bl5", + "https://example.com/cia5rhes702d4hkjxxf0dzxl4/cia5rhes702d5hkjxxnd097v7/cia5rhes702d6hkjx98mpvdya/cia5rhes702d7hkjx284luop7", + "https://example.com/cia5rhes702d8hkjxy6hghmfk/cia5rhes702d9hkjxr4ozxswm/cia5rhes702dahkjx4aemrdzl/cia5rhes702dbhkjx3b9om3gn", + "https://example.com/cia5rhes702dchkjx2q559yuu/cia5rhes702ddhkjxr1frvgb5/cia5rhes702dehkjx59to46ip/cia5rhes702dfhkjxtjmix0kn", + "https://example.com/cia5rhes702dghkjxk4m6a2s0/cia5rhes702dhhkjxfwaeszqy/cia5rhes702dihkjx4zf8y4ca/cia5rhes702djhkjxvhfrquil", + "https://example.com/cia5rhes702dkhkjx2orxsnm3/cia5rhes702dlhkjx47rdcwpv/cia5rhes702dmhkjx8j62q07m/cia5rhes702dnhkjxt3qftg4a", + "https://example.com/cia5rhes702dohkjxer57v1ky/cia5rhes702dphkjxjbishjq1/cia5rhes702dqhkjxt8r2fmuw/cia5rhes702drhkjx8etd1xkq", + "https://example.com/cia5rhes702dshkjxwbjmsogs/cia5rhes702dthkjxzjt0f26i/cia5rhes702duhkjxrspfet0e/cia5rhes702dvhkjx24ih1puf", + "https://example.com/cia5rhes702dwhkjx4qx5ofni/cia5rhes702dxhkjxyxhxsw0c/cia5rhes702dyhkjx8mi9wbce/cia5rhes702dzhkjxr9gk1g19", + "https://example.com/cia5rhes702e0hkjxin8zq13k/cia5rhes702e1hkjxn5bq0ikw/cia5rhes702e2hkjxxb2qoxsk/cia5rhes702e3hkjxbco0q0qj", + "https://example.com/cia5rhes702e4hkjxhxbl6l43/cia5rhes702e5hkjx0zz697fh/cia5rhes702e6hkjxfdsk112c/cia5rhes702e7hkjxabbxyd7j", + "https://example.com/cia5rhes702e8hkjx3vnctynz/cia5rhes702e9hkjxg4zopm86/cia5rhes702eahkjxo3bg8ml3/cia5rhes702ebhkjxp3aeugu4", + "https://example.com/cia5rhes702echkjxal3j832h/cia5rhes702edhkjx1lyibi15/cia5rhes702eehkjxstdtwkp6/cia5rhes702efhkjxdnbnyno0", + "https://example.com/cia5rhes702eghkjx55wp2mw0/cia5rhes702ehhkjxwmxwjl29/cia5rhes702eihkjxg7t126ld/cia5rhes702ejhkjx15qdziu1", + "https://example.com/cia5rhes702ekhkjxc0im9wy4/cia5rhes702elhkjxh2jd7hzr/cia5rhes702emhkjxcu8r9pzm/cia5rhes702enhkjx9jbgidf1", + "https://example.com/cia5rhes702eohkjxhlu6h4ep/cia5rhes702ephkjx4mwoc3ql/cia5rhes702eqhkjxe2bwkjv6/cia5rhes702erhkjxh8shrs32", + "https://example.com/cia5rhes702eshkjxs8w53l9b/cia5rhes702ethkjx1xsjdbbm/cia5rhes702euhkjxjrkym5vf/cia5rhes702evhkjxsuode17c", + "https://example.com/cia5rhes702ewhkjxj1bzme2d/cia5rhes702exhkjx88mzjzre/cia5rhes702eyhkjxst5flmg9/cia5rhes702ezhkjxdar3h55h", + "https://example.com/cia5rhes702f0hkjxrdjoki1j/cia5rhes702f1hkjx7iz1lpso/cia5rhes702f2hkjxvsyy2boh/cia5rhes702f3hkjxe4lwxkjq", + "https://example.com/cia5rhes702f4hkjxhsgvfwf9/cia5rhes702f5hkjxtemdddm6/cia5rhes702f6hkjx8t7z5qmo/cia5rhes702f7hkjxgb9mzb5t", + "https://example.com/cia5rhes702f8hkjxen7vbt3a/cia5rhes702f9hkjxozpijk1f/cia5rhes702fahkjxh2l1f7h6/cia5rhes702fbhkjxxojzw7gn", + "https://example.com/cia5rhes702fchkjx0tvnzt2w/cia5rhes702fdhkjxbi6zt33e/cia5rhes702fehkjxd54fxgzx/cia5rhes702ffhkjxsayc02os", + "https://example.com/cia5rhes702fghkjxpygjjz89/cia5rhes702fhhkjxbct2ojjb/cia5rhes702fihkjxe46ngi4m/cia5rhes702fjhkjxq7azlfig", + "https://example.com/cia5rhes702fkhkjx1ff4tumn/cia5rhes702flhkjxosiemsy8/cia5rhes702fmhkjx0o6ktv9m/cia5rhes702fnhkjxj9yp67gs", + "https://example.com/cia5rhes702fohkjxqro5xqt8/cia5rhes702fphkjx6s3gi6y0/cia5rhes702fqhkjxkkab85zz/cia5rhes702frhkjxo03b56tw", + "https://example.com/cia5rhes702fshkjxlvaiv6rz/cia5rhes702fthkjxvkg1r7dy/cia5rhes702fuhkjx3txhokr4/cia5rhes702fvhkjxtvqvs9ei", + "https://example.com/cia5rhes702fwhkjx5mknq1w5/cia5rhes702fxhkjxrj6a3pub/cia5rhes702fyhkjxvhu05ms3/cia5rhes702fzhkjxjby42qra", + "https://example.com/cia5rhes702g0hkjxrcf4pcw1/cia5rhes702g1hkjxn081wq4r/cia5rhes702g2hkjxaf91n239/cia5rhes702g3hkjxxlcnut0h", + "https://example.com/cia5rhes702g4hkjxboifrcf9/cia5rhes702g5hkjxzdowoz5o/cia5rhes702g6hkjxukarx97t/cia5rhes702g7hkjxccz4m3ra", + "https://example.com/cia5rhes702g8hkjxcojon0ux/cia5rhes702g9hkjxldlady20/cia5rhes702gahkjxzy3fh4eg/cia5rhes702gbhkjxrbfe6e4i", + "https://example.com/cia5rhes702gchkjxk66e8nbf/cia5rhes702gdhkjxudeemkvv/cia5rhes702gehkjx3c3hpe66/cia5rhes702gfhkjxn9olbr7q", + "https://example.com/cia5rhes702gghkjxjutmvz9r/cia5rhes702ghhkjxevjnumc0/cia5rhes702gihkjxcsgdpbt7/cia5rhes702gjhkjxkajsb5n7", + "https://example.com/cia5rhes702gkhkjxpctjecch/cia5rhes702glhkjx4psglrrf/cia5rhes702gmhkjxqsa29brc/cia5rhes702gnhkjxtu5lc4me", + "https://example.com/cia5rhes702gohkjxy4ljuvei/cia5rhes702gphkjxscllm1ij/cia5rhes702gqhkjx1e8d9ndd/cia5rhes702grhkjxvt3mx80t", + "https://example.com/cia5rhes702gshkjxex3gg0nz/cia5rhes702gthkjxlonhrzjs/cia5rhes702guhkjxl4vdp4al/cia5rhes702gvhkjxvu5xtj65", + "https://example.com/cia5rhes702gwhkjx2eqa8s8p/cia5rhes702gxhkjxzs0d96f8/cia5rhes702gyhkjxh5qhyc6d/cia5rhes702gzhkjxit6h6kq6", + "https://example.com/cia5rhes702h0hkjxovyyxzzh/cia5rhes702h1hkjxumx2doq9/cia5rhes702h2hkjxe8rwx6ye/cia5rhes702h3hkjxd0biux3c", + "https://example.com/cia5rhes702h4hkjx0r9rhds4/cia5rhes802h5hkjxxe3pbik6/cia5rhes802h6hkjxkoyqybob/cia5rhes802h7hkjx0s2gcxkk", + "https://example.com/cia5rhes802h8hkjx1b212net/cia5rhes802h9hkjxhye14m2j/cia5rhes802hahkjxf87hamb3/cia5rhes802hbhkjxvqh1ek5s", + "https://example.com/cia5rhes802hchkjx5hdnwwul/cia5rhes802hdhkjxyc9ojtpr/cia5rhes802hehkjxdnrgdch1/cia5rhes802hfhkjxj6gwgjbt", + "https://example.com/cia5rhes802hghkjxehwpywy9/cia5rhes802hhhkjx4lbi6x6l/cia5rhes802hihkjxnpf2cz93/cia5rhes802hjhkjxv9bgej4e", + "https://example.com/cia5rhes802hkhkjxta1aj8pd/cia5rhes802hlhkjxqot5lx49/cia5rhes802hmhkjxs0uj77o1/cia5rhes802hnhkjx69uqlhl9", + "https://example.com/cia5rhes802hohkjxsetak465/cia5rhes802hphkjx7cc4cvnw/cia5rhes802hqhkjxyz2rd85f/cia5rhes802hrhkjxwwwj80zy", + "https://example.com/cia5rhes802hshkjxcxpfz2zy/cia5rhes802hthkjx0mg13xvr/cia5rhes802huhkjxl8tf2f1j/cia5rhes802hvhkjxkkdxui48", + "https://example.com/cia5rhes802hwhkjxnt5u3nhm/cia5rhes802hxhkjxbffb2x8l/cia5rhes802hyhkjxd0tm0h6e/cia5rhes802hzhkjxua697jh2", + "https://example.com/cia5rhes802i0hkjx5thy2y3q/cia5rhes802i1hkjx3jr1y269/cia5rhes802i2hkjxwwksi6eg/cia5rhes802i3hkjxor5nv0z2", + "https://example.com/cia5rhes802i4hkjx4ttg4je9/cia5rhes802i5hkjxqzq7w677/cia5rhes802i6hkjxeldnbsf2/cia5rhes802i7hkjxk8rmgjfv", + "https://example.com/cia5rhes802i8hkjx6eb7w4np/cia5rhes802i9hkjxstgvt28t/cia5rhes802iahkjx8b9vwdzr/cia5rhes802ibhkjx1pnrsc7b", + "https://example.com/cia5rhes802ichkjxvo1nrawf/cia5rhes802idhkjxgivthtjh/cia5rhes802iehkjxx967w9dk/cia5rhes802ifhkjxuu3hsee9", + "https://example.com/cia5rhes802ighkjxeijczff2/cia5rhes802ihhkjxer0knjjl/cia5rhes802iihkjx116p0tfc/cia5rhes802ijhkjxuomqb7a0", + "https://example.com/cia5rhes802ikhkjxzw0s6ejs/cia5rhes802ilhkjx1fgypntw/cia5rhes802imhkjx7jreimgw/cia5rhes802inhkjx3shm6234", + "https://example.com/cia5rhes802iohkjx28sv1ivu/cia5rhes802iphkjxr4p098ji/cia5rhes802iqhkjxdsotusgp/cia5rhes802irhkjx5kudhhd2", + "https://example.com/cia5rhes802ishkjxixiz6mp1/cia5rhes802ithkjxcagn4wzv/cia5rhes802iuhkjxnulj8edc/cia5rhes802ivhkjxkc73vmwx", + "https://example.com/cia5rhes802iwhkjx0gdh9w2o/cia5rhes802ixhkjxcl50g4e1/cia5rhes802iyhkjxrptys42g/cia5rhes802izhkjx4w62hqht", + "https://example.com/cia5rhes802j0hkjx692slfdu/cia5rhes802j1hkjxv3bwwytl/cia5rhes802j2hkjxx3gky18b/cia5rhes802j3hkjx3vsa5jra", + "https://example.com/cia5rhes802j4hkjx2xe9zlx3/cia5rhes802j5hkjx4f3wi9rl/cia5rhes802j6hkjx2qr5bzrp/cia5rhes802j7hkjxrw3fcfe5", + "https://example.com/cia5rhes802j8hkjx14k6emm1/cia5rhes802j9hkjxcll7rahj/cia5rhes802jahkjx6dmkabft/cia5rhes802jbhkjxj2d4kvm5", + "https://example.com/cia5rhes802jchkjxu3olmu84/cia5rhes802jdhkjx1kyxhqd9/cia5rhes802jehkjxmzlxuvus/cia5rhes802jfhkjxt7cvj5h1", + "https://example.com/cia5rhes802jghkjx80q77jzc/cia5rhes802jhhkjxtj8xxa1e/cia5rhes802jihkjxu4lnwkqf/cia5rhes802jjhkjx33w6a2yi", + "https://example.com/cia5rhes802jkhkjxefk60o55/cia5rhes802jlhkjx9yuz30ib/cia5rhes802jmhkjxhuhfcbcy/cia5rhes802jnhkjxf6wkt9ht", + "https://example.com/cia5rhes802johkjxbbd9f8zb/cia5rhes802jphkjxzk5mtk4f/cia5rhes802jqhkjxwb6eerfn/cia5rhes802jrhkjxpuyyqgqw", + "https://example.com/cia5rhes802jshkjxbgeqep0t/cia5rhes802jthkjxdkbxnh69/cia5rhes802juhkjx03vlhdpu/cia5rhes802jvhkjx8zhdnauu", + "https://example.com/cia5rhes802jwhkjxkw5bbsew/cia5rhes802jxhkjxgzdwzev2/cia5rhes802jyhkjxeutiz8ot/cia5rhes802jzhkjxeigt9qdf", + "https://example.com/cia5rhes802k0hkjxvmfunndw/cia5rhes802k1hkjx6j968gws/cia5rhes802k2hkjxjdz6yfk6/cia5rhes802k3hkjxnsoiuwfm", + "https://example.com/cia5rhes802k4hkjx7fezq8em/cia5rhes802k5hkjxsgmtvhig/cia5rhes802k6hkjx8h5r0ac5/cia5rhes802k7hkjxm0tczqr3", + "https://example.com/cia5rhes802k8hkjx3ej667en/cia5rhes802k9hkjxeta3mqrs/cia5rhes802kahkjxttm3mtbc/cia5rhes802kbhkjx08tnchxt", + "https://example.com/cia5rhes802kchkjxs9ys1d1h/cia5rhes802kdhkjxa7zfmkfh/cia5rhes802kehkjx0f1f3s5x/cia5rhes802kfhkjx5bkfptdv", + "https://example.com/cia5rhes802kghkjxfbx0j2be/cia5rhes802khhkjx2796rmnr/cia5rhes802kihkjxc8qjmfqv/cia5rhes802kjhkjx5l18ngbo", + "https://example.com/cia5rhes802kkhkjxuvxeycqp/cia5rhes802klhkjxt3xak01c/cia5rhes802kmhkjxsnbinf75/cia5rhes802knhkjxdke5f04u", + "https://example.com/cia5rhes802kohkjxlr2esas9/cia5rhes802kphkjxoi8bubek/cia5rhes802kqhkjx652tsdtk/cia5rhes802krhkjxx0d9sapx", + "https://example.com/cia5rhes802kshkjx01h9i4q2/cia5rhes802kthkjxnvp9j4x1/cia5rhes802kuhkjxyfb118if/cia5rhes802kvhkjxcajg7k7x", + "https://example.com/cia5rhes802kwhkjx1ahqqj6a/cia5rhes802kxhkjx576izsui/cia5rhes802kyhkjxdopj85lq/cia5rhes802kzhkjxrak3td4w", + "https://example.com/cia5rhes802l0hkjx4oalj3hp/cia5rhes802l1hkjxuiaufryz/cia5rhes802l2hkjx3yx8z13v/cia5rhes802l3hkjxql0nh4mw", + "https://example.com/cia5rhes802l4hkjxh2yk4att/cia5rhes802l5hkjx1ld7evsc/cia5rhes802l6hkjx64mt6pcs/cia5rhes802l7hkjxoa0hr513", + "https://example.com/cia5rhes802l8hkjxb8puz3pu/cia5rhes802l9hkjx40l0wzy4/cia5rhes802lahkjxvqaauxku/cia5rhes802lbhkjxxe2r13sb", + "https://example.com/cia5rhes802lchkjxp07dxy3z/cia5rhes802ldhkjx4p23lqcu/cia5rhes802lehkjxj1swfy96/cia5rhes802lfhkjxeppnm27y", + "https://example.com/cia5rhes802lghkjxowt32sxr/cia5rhes802lhhkjxr2wyl9ej/cia5rhes802lihkjx62orwsjq/cia5rhes802ljhkjxw99oj10o", + "https://example.com/cia5rhes802lkhkjxieavj07d/cia5rhes802llhkjxrglllbwb/cia5rhes802lmhkjxjbmalhyj/cia5rhes802lnhkjx54ff2569", + "https://example.com/cia5rhes802lohkjxgar3wut3/cia5rhes802lphkjx3y6byab9/cia5rhes802lqhkjx2ki1hks2/cia5rhes802lrhkjx867oulq5", + "https://example.com/cia5rhes802lshkjxb7hkzrqs/cia5rhes802lthkjxfu4yyljq/cia5rhes802luhkjxqswmaz83/cia5rhes802lvhkjxcgjxwpin", + "https://example.com/cia5rhes802lwhkjxnow6sr6f/cia5rhes802lxhkjxbtxn02ok/cia5rhes802lyhkjxtreu397w/cia5rhes802lzhkjx9fbk1l2s", + "https://example.com/cia5rhes802m0hkjxuov8bbjf/cia5rhes802m1hkjxfjxcjswu/cia5rhes802m2hkjxnuumriep/cia5rhes802m3hkjxv3abuieh", + "https://example.com/cia5rhes802m4hkjx7chzbj7m/cia5rhes802m5hkjxamwwacgg/cia5rhes802m6hkjxalw3n1b1/cia5rhes802m7hkjx4oobkiqi", + "https://example.com/cia5rhes802m8hkjx01qpkvwg/cia5rhes802m9hkjx8s23cjy5/cia5rhes802mahkjx7zklmzeg/cia5rhes802mbhkjxs9htzggq", + "https://example.com/cia5rhes802mchkjxn4kg4arq/cia5rhes802mdhkjxrlms5rxt/cia5rhes802mehkjx51y6d37q/cia5rhes802mfhkjxgq01e010", + "https://example.com/cia5rhes802mghkjxcuk2pmky/cia5rhes802mhhkjxcy28ajz9/cia5rhes802mihkjxm3xz72d2/cia5rhes802mjhkjxfecrsmb1", + "https://example.com/cia5rhes802mkhkjx976oo1q6/cia5rhes802mlhkjxt1d0ks1h/cia5rhes802mmhkjxlya2lnkr/cia5rhes802mnhkjxcjv6cg22", + "https://example.com/cia5rhes802mohkjxbmd2ljcc/cia5rhes802mphkjxqmpsf43e/cia5rhes802mqhkjx1018sa5h/cia5rhes802mrhkjx83lcx364", + "https://example.com/cia5rhes802mshkjxx317boaq/cia5rhes802mthkjxvgax0zmu/cia5rhes802muhkjxfgnulq5x/cia5rhes802mvhkjxg9czop5a", + "https://example.com/cia5rhes802mwhkjx3qru605e/cia5rhes802mxhkjxo14r6mbk/cia5rhes802myhkjxnxvtblhe/cia5rhes802mzhkjxp4fuyyvq", + "https://example.com/cia5rhes802n0hkjxta6r34nn/cia5rhes802n1hkjxn2des330/cia5rhes802n2hkjxut3wbscg/cia5rhes802n3hkjxi3sjsek8", + "https://example.com/cia5rhes802n4hkjxlrze879b/cia5rhes802n5hkjxl9d2bptv/cia5rhes802n6hkjxe2pyq523/cia5rhes802n7hkjx3d7uk0va", + "https://example.com/cia5rhes802n8hkjxsm87le7w/cia5rhes802n9hkjxilk0wcph/cia5rhes802nahkjx2n4ghjd4/cia5rhes802nbhkjx2z0n5kej", + "https://example.com/cia5rhes802nchkjxjyazwvt2/cia5rhes802ndhkjxg9kfrprk/cia5rhes802nehkjxbgdpif6f/cia5rhes802nfhkjxe0456keb", + "https://example.com/cia5rhes802nghkjxjhpr22o6/cia5rhes802nhhkjxplhnqcb8/cia5rhes802nihkjxzys0lxo2/cia5rhes802njhkjx2q01z427", + "https://example.com/cia5rhes802nkhkjx2bh3a4jg/cia5rhes802nlhkjxwr4hs5z6/cia5rhes802nmhkjxuj8y14q2/cia5rhes802nnhkjxuhl3zdhl", + "https://example.com/cia5rhes802nohkjxsghqd6qb/cia5rhes802nphkjxzwvmt5ut/cia5rhes802nqhkjxr3vatvee/cia5rhes802nrhkjx2bozv5k1", + "https://example.com/cia5rhes802nshkjx2d7r9wfy/cia5rhes802nthkjxcxj3kn6a/cia5rhes802nuhkjxdria7pkp/cia5rhes802nvhkjx6uliansr", + "https://example.com/cia5rhes802nwhkjx8nqjhhq1/cia5rhes802nxhkjxc2k2euc9/cia5rhes802nyhkjxdv6dq6vu/cia5rhes802nzhkjxidl9ujw8", + "https://example.com/cia5rhes802o0hkjxt3hs5pt1/cia5rhes802o1hkjxouvuo74k/cia5rhes802o2hkjx46xz3nds/cia5rhes802o3hkjxrrrkqadg", + "https://example.com/cia5rhes902o4hkjx2apdepej/cia5rhes902o5hkjxqqttkkkz/cia5rhes902o6hkjxh46l0jeu/cia5rhes902o7hkjxl7h17xdc", + "https://example.com/cia5rhes902o8hkjxbafzc6v5/cia5rhes902o9hkjxcuowkvn1/cia5rhes902oahkjxasvphtbh/cia5rhes902obhkjxgp6ckpu5", + "https://example.com/cia5rhes902ochkjxfb99zhss/cia5rhes902odhkjx0idz3cqv/cia5rhes902oehkjxy0f9nkn1/cia5rhes902ofhkjxnhrq2m1r", + "https://example.com/cia5rhes902oghkjx24kuk19k/cia5rhes902ohhkjx5hx5puqb/cia5rhes902oihkjxcaqprqtz/cia5rhes902ojhkjx3zh6ivhp", + "https://example.com/cia5rhes902okhkjxuk062elz/cia5rhes902olhkjxpv0ezkgb/cia5rhes902omhkjx6gkm3rj1/cia5rhes902onhkjxmckdzmmf", + "https://example.com/cia5rhes902oohkjx6667yepw/cia5rhes902ophkjxkhrilcux/cia5rhes902oqhkjxubgywv84/cia5rhes902orhkjxl2z5gfhv", + "https://example.com/cia5rhes902oshkjxwnznffds/cia5rhes902othkjx2nrd505l/cia5rhes902ouhkjxor8wvi62/cia5rhes902ovhkjxkknnf2c5", + "https://example.com/cia5rhes902owhkjx0xvzj6j4/cia5rhes902oxhkjxm8wjviav/cia5rhes902oyhkjxd48tw0nv/cia5rhes902ozhkjxy55fth1m", + "https://example.com/cia5rhes902p0hkjxhf2ln9fg/cia5rhes902p1hkjxn1kh849s/cia5rhes902p2hkjx7w18z1ij/cia5rhes902p3hkjx1iukw4f9", + "https://example.com/cia5rhes902p4hkjx94e9yno8/cia5rhes902p5hkjxgg7krrow/cia5rhes902p6hkjxs7qbcgio/cia5rhes902p7hkjxjy5ubg21", + "https://example.com/cia5rhes902p8hkjxc76syimq/cia5rhes902p9hkjxr4crms15/cia5rhes902pahkjxnijggak5/cia5rhes902pbhkjxzj7ajf4p", + "https://example.com/cia5rhes902pchkjxtq8dybc1/cia5rhes902pdhkjxwqg0v1ob/cia5rhes902pehkjxig150nfx/cia5rhes902pfhkjx4pn0r7va", + "https://example.com/cia5rhes902pghkjxg86s4zod/cia5rhes902phhkjxc16il6yq/cia5rhes902pihkjx25j53w11/cia5rhes902pjhkjxar484o36", + "https://example.com/cia5rhes902pkhkjxqjtgnf6o/cia5rhes902plhkjxx22y2p6c/cia5rhes902pmhkjxu72lfdom/cia5rhes902pnhkjxv7bb9e9q", + "https://example.com/cia5rhes902pohkjxxb029uj1/cia5rhes902pphkjx4ujdzzo5/cia5rhes902pqhkjxx4lnnhw7/cia5rhes902prhkjx6x2u79ck", + "https://example.com/cia5rhes902pshkjxd1hhakk6/cia5rhes902pthkjxmpu8mcyi/cia5rhes902puhkjxzpcbicof/cia5rhes902pvhkjxij383b25", + "https://example.com/cia5rhes902pwhkjxiur6rdbh/cia5rhes902pxhkjxyhkhpxrq/cia5rhes902pyhkjx29a11uyj/cia5rhes902pzhkjxf1p8g30r", + "https://example.com/cia5rhes902q0hkjxotowbqgb/cia5rhes902q1hkjxmb7p5sr6/cia5rhes902q2hkjx378apexd/cia5rhes902q3hkjxjkglr1c4", + "https://example.com/cia5rhes902q4hkjxxcw4jsq6/cia5rhes902q5hkjxqenj7c97/cia5rhes902q6hkjx2ye8s3q1/cia5rhes902q7hkjxtxm7sdya", + "https://example.com/cia5rhes902q8hkjxkc9vstb2/cia5rhes902q9hkjxzwok7ng9/cia5rhes902qahkjx8ygp04d1/cia5rhes902qbhkjx4qux7aki", + "https://example.com/cia5rhes902qchkjx57dfmt8h/cia5rhes902qdhkjx9b0035cy/cia5rhes902qehkjxvebgxts8/cia5rhes902qfhkjxu6yi37mb", + "https://example.com/cia5rhes902qghkjx6xi3dyjx/cia5rhes902qhhkjx9x0aclfr/cia5rhes902qihkjx7mxvg28t/cia5rhes902qjhkjx9q2wphpa", + "https://example.com/cia5rhes902qkhkjxu6s6q2q0/cia5rhes902qlhkjxlgcpgxpx/cia5rhes902qmhkjxc1dxbvr1/cia5rhes902qnhkjx6bvhf6hr", + "https://example.com/cia5rhes902qohkjx1tm0hkvs/cia5rhes902qphkjx3pu22rbr/cia5rhes902qqhkjxdjcth8ug/cia5rhes902qrhkjxwhg6mr88", + "https://example.com/cia5rhes902qshkjxji4arcck/cia5rhes902qthkjx5t1kjk9o/cia5rhes902quhkjx88zgme2o/cia5rhes902qvhkjxhs22agoc", + "https://example.com/cia5rhes902qwhkjxkqdfy8em/cia5rhes902qxhkjxo11waca0/cia5rhes902qyhkjxthqzds0b/cia5rhes902qzhkjx890jrftn", + "https://example.com/cia5rhes902r0hkjx2scv74kv/cia5rhes902r1hkjxhczgr5iw/cia5rhes902r2hkjxd9v3ewx1/cia5rhes902r3hkjxx4tpj5xh", + "https://example.com/cia5rhes902r4hkjx8217649m/cia5rhes902r5hkjx954lwwvc/cia5rhes902r6hkjxzczxe9o4/cia5rhes902r7hkjxadtvbtm3", + "https://example.com/cia5rhes902r8hkjx1su72qpn/cia5rhes902r9hkjxexh45oq0/cia5rhes902rahkjxah76ntxr/cia5rhes902rbhkjxnnfojf19", + "https://example.com/cia5rhes902rchkjxc85n1zzu/cia5rhes902rdhkjxix5w6nkz/cia5rhes902rehkjxogcyyb50/cia5rhes902rfhkjx3r7glwov", + "https://example.com/cia5rhes902rghkjxtck9dhwc/cia5rhes902rhhkjxru36hy4a/cia5rhes902rihkjxmmyc9tpx/cia5rhes902rjhkjxmbsypxaq", + "https://example.com/cia5rhes902rkhkjx2fo040pu/cia5rhes902rlhkjxr65jltb9/cia5rhes902rmhkjxnzf86rqg/cia5rhes902rnhkjxca9gnhfv", + "https://example.com/cia5rhes902rohkjx3t12qfew/cia5rhes902rphkjx0uy0q6x0/cia5rhes902rqhkjxytr1mozv/cia5rhes902rrhkjxti5cpfhq", + "https://example.com/cia5rhes902rshkjxtzuesbvw/cia5rhes902rthkjx4imx7yq2/cia5rhes902ruhkjxv5rwbdfw/cia5rhes902rvhkjxx9dyruvh", + "https://example.com/cia5rhes902rwhkjx80skj5fy/cia5rhes902rxhkjxs2roo0or/cia5rhes902ryhkjx0f0egqew/cia5rhes902rzhkjx2qyobgwd", + "https://example.com/cia5rhes902s0hkjxwzjb0ibj/cia5rhes902s1hkjxthhdzgdb/cia5rhes902s2hkjxmp0am5hc/cia5rhes902s3hkjxou8fe0bw", + "https://example.com/cia5rhes902s4hkjxy807y0wz/cia5rhes902s5hkjxyi0ucjpj/cia5rhes902s6hkjx57r4913i/cia5rhes902s7hkjx5zyg25co", + "https://example.com/cia5rhes902s8hkjxtv0y9qsr/cia5rhes902s9hkjxmara3sln/cia5rhes902sahkjx16zbww31/cia5rhes902sbhkjxk3yfnqrf", + "https://example.com/cia5rhes902schkjxmqs7wb8e/cia5rhes902sdhkjxbzqsikjf/cia5rhes902sehkjxifnkxd42/cia5rhes902sfhkjxeslnix9t", + "https://example.com/cia5rhes902sghkjx9csqi025/cia5rhes902shhkjx03m41rdk/cia5rhes902sihkjx7o16p436/cia5rhes902sjhkjxuopqyoaf", + "https://example.com/cia5rhes902skhkjxkj9lox0l/cia5rhes902slhkjx4siwdfz6/cia5rhes902smhkjxkz6smrk5/cia5rhes902snhkjxbydhx9sr", + "https://example.com/cia5rhes902sohkjxmt7rn0m7/cia5rhes902sphkjxbr1rrero/cia5rhes902sqhkjxgsa5faxo/cia5rhes902srhkjxkaypi7hq", + "https://example.com/cia5rhes902sshkjxcdabjgaq/cia5rhes902sthkjxdj1l2sdw/cia5rhes902suhkjx4w18whjz/cia5rhes902svhkjx00bsy24i", + "https://example.com/cia5rhes902swhkjx1sxzd3bs/cia5rhes902sxhkjxwihbb32s/cia5rhes902syhkjxjv82ql1y/cia5rhes902szhkjxhx2p1tjw", + "https://example.com/cia5rhes902t0hkjxlq1v45l8/cia5rhes902t1hkjxpcb65x6c/cia5rhes902t2hkjxqd79lp9t/cia5rhes902t3hkjxzlu0vgsq", + "https://example.com/cia5rhes902t4hkjxchoh1xz9/cia5rhes902t5hkjxi7ja8w34/cia5rhes902t6hkjxcibihy5j/cia5rhes902t7hkjxzxhj2llf", + "https://example.com/cia5rhes902t8hkjxe6kjteus/cia5rhes902t9hkjxct0osy9c/cia5rhes902tahkjxkpn37x26/cia5rhes902tbhkjxs1i3y06r", + "https://example.com/cia5rhes902tchkjxkijsvrry/cia5rhes902tdhkjx478e7b15/cia5rhes902tehkjxe15r2zp0/cia5rhes902tfhkjx0xdr6u4g", + "https://example.com/cia5rhes902tghkjxre727axs/cia5rhes902thhkjx8tjhkncn/cia5rhes902tihkjxfwe9moa8/cia5rhes902tjhkjxrw37is68", + "https://example.com/cia5rhes902tkhkjx1vha7oxy/cia5rhes902tlhkjxorgrss4a/cia5rhes902tmhkjx5v2vjvpc/cia5rhes902tnhkjxe13xjwvn", + "https://example.com/cia5rhes902tohkjxhh415ghg/cia5rhes902tphkjxewddudgl/cia5rhes902tqhkjxlt904su4/cia5rhes902trhkjxvox3ueb9", + "https://example.com/cia5rhes902tshkjx565cdwgu/cia5rhes902tthkjx7v8dxnp1/cia5rhes902tuhkjx9lkhhc8x/cia5rhes902tvhkjxet30fwnm", + "https://example.com/cia5rhes902twhkjx50zbd0gj/cia5rhes902txhkjxcxmjzp6i/cia5rhes902tyhkjx4wwog6sc/cia5rhes902tzhkjxl5k35m8y", + "https://example.com/cia5rhes902u0hkjxmixf873e/cia5rhes902u1hkjxqkzx249g/cia5rhes902u2hkjxq1h6e73c/cia5rhes902u3hkjxy0raorlv", + "https://example.com/cia5rhes902u4hkjxp7qu708r/cia5rhes902u5hkjxjq511roe/cia5rhes902u6hkjx24zsjlw7/cia5rhes902u7hkjxxao19ibw", + "https://example.com/cia5rhes902u8hkjxjj8e4qjy/cia5rhes902u9hkjxnpfmyyee/cia5rhesa02uahkjx2cbqjqlj/cia5rhesa02ubhkjx2sb57ho9", + "https://example.com/cia5rhesa02uchkjxi7stcunb/cia5rhesa02udhkjxf13m1va9/cia5rhesa02uehkjxbrbvzlts/cia5rhesa02ufhkjxv7c5sg8p", + "https://example.com/cia5rhesa02ughkjxc4bg17mm/cia5rhesa02uhhkjx1fodi8bu/cia5rhesa02uihkjx1pgynm8w/cia5rhesa02ujhkjx21oibarf", + "https://example.com/cia5rhesa02ukhkjx9l592wdv/cia5rhesa02ulhkjxvbp05nkt/cia5rhesa02umhkjxosf9qynb/cia5rhesa02unhkjx7bb91ukh", + "https://example.com/cia5rhesa02uohkjxkss6ccme/cia5rhesa02uphkjxvd93brv7/cia5rhesa02uqhkjxoc61qqcx/cia5rhesa02urhkjxawfbw41u", + "https://example.com/cia5rhesa02ushkjxgxz51hyw/cia5rhesa02uthkjxewu4vl7k/cia5rhesa02uuhkjxknapklva/cia5rhesa02uvhkjxmdf6weyv", + "https://example.com/cia5rhesa02uwhkjx9egqxjsi/cia5rhesa02uxhkjxfzhkd5yr/cia5rhesa02uyhkjx8hv3p08k/cia5rhesa02uzhkjx45psji1y", + "https://example.com/cia5rhesa02v0hkjx4l8lsl3u/cia5rhesa02v1hkjxngdy1ar6/cia5rhesa02v2hkjx6jo5h3qu/cia5rhesa02v3hkjxzbv3dpni", + "https://example.com/cia5rhesa02v4hkjxtcyq3hrj/cia5rhesa02v5hkjxi1yvcnxy/cia5rhesa02v6hkjxw7v5871t/cia5rhesa02v7hkjxe8a1rpaz", + "https://example.com/cia5rhesa02v8hkjx11e94e28/cia5rhesa02v9hkjxy79y9wsa/cia5rhesa02vahkjx4x1k6e7p/cia5rhesa02vbhkjx4nugadg0", + "https://example.com/cia5rhesa02vchkjxyff973f9/cia5rhesa02vdhkjxxeylqp99/cia5rhesa02vehkjxup7kbh2i/cia5rhesa02vfhkjxvik2bnru", + "https://example.com/cia5rhesa02vghkjxojpmez35/cia5rhesa02vhhkjxrsr1rbtw/cia5rhesa02vihkjxz51r21kh/cia5rhesa02vjhkjxnry87ysd", + "https://example.com/cia5rhesa02vkhkjxc7kcgnod/cia5rhesa02vlhkjxou8csehx/cia5rhesa02vmhkjx4g46j5vv/cia5rhesa02vnhkjx0xdxordo", + "https://example.com/cia5rhesa02vohkjxcpd7futv/cia5rhesa02vphkjxjhhvuq13/cia5rhesa02vqhkjx1jx0mwyq/cia5rhesa02vrhkjxatheqhre", + "https://example.com/cia5rhesa02vshkjxhoxrm7du/cia5rhesa02vthkjxp3j2d7pl/cia5rhesa02vuhkjxfajs3kp2/cia5rhesa02vvhkjx094w7t5z", + "https://example.com/cia5rhesa02vwhkjx8zsoc546/cia5rhesa02vxhkjxbbwesmgs/cia5rhesa02vyhkjxah7vbsl2/cia5rhesa02vzhkjxccc1osvb", + "https://example.com/cia5rhesa02w0hkjxilmp1gcj/cia5rhesa02w1hkjxpax3mj4u/cia5rhesa02w2hkjxl4830fix/cia5rhesa02w3hkjxushwofrd", + "https://example.com/cia5rhesa02w4hkjx0ayq3lna/cia5rhesa02w5hkjxtyfjinxi/cia5rhesa02w6hkjx6v3jk6np/cia5rhesa02w7hkjxb3kzmwfz", + "https://example.com/cia5rhesa02w8hkjxpfztdog3/cia5rhesa02w9hkjxxu1jj9ro/cia5rhesa02wahkjx9x02t4s6/cia5rhesa02wbhkjxmudm4let", + "https://example.com/cia5rhesa02wchkjxf8gwzm46/cia5rhesa02wdhkjxnogroqj5/cia5rhesa02wehkjxzcswjm19/cia5rhesa02wfhkjxd7sq70cn", + "https://example.com/cia5rhesa02wghkjxzl71wo9i/cia5rhesa02whhkjx4qzdc4en/cia5rhesa02wihkjxqcwczavg/cia5rhesa02wjhkjx5v1mo7io", + "https://example.com/cia5rhesa02wkhkjx17rwy1u4/cia5rhesa02wlhkjxupgzmlhz/cia5rhesa02wmhkjxy4guynpo/cia5rhesa02wnhkjx7hwclzmy", + "https://example.com/cia5rhesa02wohkjxbhrfwyae/cia5rhesa02wphkjxg54k6a1v/cia5rhesa02wqhkjxxx6ovpcu/cia5rhesa02wrhkjx7chazbg3", + "https://example.com/cia5rhesa02wshkjxsqxk0429/cia5rhesa02wthkjxjmhikrl3/cia5rhesa02wuhkjxojb6ebr1/cia5rhesa02wvhkjxrcv5ezdg", + "https://example.com/cia5rhesa02wwhkjx6oc3w8ov/cia5rhesa02wxhkjxm2i72vec/cia5rhesa02wyhkjx3fh9ne9a/cia5rhesa02wzhkjx0971hhm1", + "https://example.com/cia5rhesa02x0hkjxwenvo26l/cia5rhesa02x1hkjxtfilhs8a/cia5rhesa02x2hkjxpqvnoyqk/cia5rhesa02x3hkjx23vjztdc", + "https://example.com/cia5rhesa02x4hkjxsgmx5os7/cia5rhesa02x5hkjxgrehs28q/cia5rhesa02x6hkjxvxtnze9l/cia5rhesa02x7hkjx0vlv9z1s", + "https://example.com/cia5rhesa02x8hkjxge35kywm/cia5rhesa02x9hkjxw2tbefo5/cia5rhesa02xahkjxv137f9qt/cia5rhesa02xbhkjxnz9ep47k", + "https://example.com/cia5rhesa02xchkjx00anlyr6/cia5rhesa02xdhkjx79zjud7w/cia5rhesa02xehkjxrb6rk7rw/cia5rhesa02xfhkjxphslyr6m", + "https://example.com/cia5rhesa02xghkjxv656h0en/cia5rhesa02xhhkjxwt9sllti/cia5rhesa02xihkjxbblv9n51/cia5rhesa02xjhkjxoms9ldox", + "https://example.com/cia5rhesa02xkhkjxb0ljdnru/cia5rhesa02xlhkjxuysb8km6/cia5rhesa02xmhkjx7sdb3ap1/cia5rhesa02xnhkjx556d8gld", + "https://example.com/cia5rhesa02xohkjxmh07tx4r/cia5rhesa02xphkjxzekp4dcp/cia5rhesa02xqhkjxywdkdqe0/cia5rhesa02xrhkjxn2exhmk3", + "https://example.com/cia5rhesa02xshkjxot0nxe5o/cia5rhesa02xthkjxxns5dc6l/cia5rhesa02xuhkjxppx37eq1/cia5rhesa02xvhkjxa0n1ft75", + "https://example.com/cia5rhesa02xwhkjxdlz42y7u/cia5rhesa02xxhkjxyyrh6ehu/cia5rhesa02xyhkjxc2snloyu/cia5rhesa02xzhkjx7hniv0l2", + "https://example.com/cia5rhesa02y0hkjx1ufllm5d/cia5rhesa02y1hkjxyciovmxi/cia5rhesa02y2hkjxeecnhafz/cia5rhesa02y3hkjxka899vl8", + "https://example.com/cia5rhesa02y4hkjxas1scma5/cia5rhesa02y5hkjx7hx7cgry/cia5rhesa02y6hkjxt4r854cj/cia5rhesa02y7hkjxl2d58z7z", + "https://example.com/cia5rhesa02y8hkjx6uvep3v0/cia5rhesa02y9hkjxzkox5acn/cia5rhesa02yahkjxbsttsd42/cia5rhesa02ybhkjx9s0eooqr", + "https://example.com/cia5rhesa02ychkjxs38kr0ju/cia5rhesa02ydhkjxp7i0dm4v/cia5rhesa02yehkjxytdwg00m/cia5rhesa02yfhkjx6kojbk3h", + "https://example.com/cia5rhesa02yghkjxu47g5rne/cia5rhesa02yhhkjxp5jiqzck/cia5rhesa02yihkjx9n0k7a86/cia5rhesa02yjhkjxawsmggmg", + "https://example.com/cia5rhesa02ykhkjxpz5t3big/cia5rhesa02ylhkjxd7mv52ko/cia5rhesa02ymhkjxaq7e3qjp/cia5rhesa02ynhkjxx6n59eel", + "https://example.com/cia5rhesa02yohkjx151ccq4m/cia5rhesa02yphkjxs6vfdnyb/cia5rhesa02yqhkjxakcfphvj/cia5rhesa02yrhkjxrtgqqlkl", + "https://example.com/cia5rhesa02yshkjx5nthrj0p/cia5rhesa02ythkjxaoa6xfw1/cia5rhesa02yuhkjxsyk94k5s/cia5rhesa02yvhkjxp1fsbr6q", + "https://example.com/cia5rhesa02ywhkjxxrjgdzm9/cia5rhesa02yxhkjxmt4liicf/cia5rhesa02yyhkjxqnavwf3w/cia5rhesa02yzhkjxwb6efk2q", + "https://example.com/cia5rhesa02z0hkjxai4gwv4i/cia5rhesb02z1hkjxotmfrbh1/cia5rhesb02z2hkjxcorstega/cia5rhesb02z3hkjxmm6qrl72", + "https://example.com/cia5rhesb02z4hkjx119h63fe/cia5rhesb02z5hkjx51zhx95d/cia5rhesb02z6hkjx13iaxgj7/cia5rhesb02z7hkjxnhttadyh", + "https://example.com/cia5rhesb02z8hkjx7n279k7d/cia5rhesb02z9hkjxtyhtwvh4/cia5rhesb02zahkjxuxc8tnjw/cia5rhesb02zbhkjx9f5w1igd", + "https://example.com/cia5rhesb02zchkjxu6u03gpq/cia5rhesb02zdhkjxad5fmie8/cia5rhesb02zehkjx82hi1ubw/cia5rhesb02zfhkjxlz014bc9", + "https://example.com/cia5rhesb02zghkjxpcp41mh4/cia5rhesb02zhhkjxwtmgx1un/cia5rhesb02zihkjxvdlzs9gj/cia5rhesb02zjhkjxxnbuzjtx", + "https://example.com/cia5rhesb02zkhkjxszpr5g1b/cia5rhesb02zlhkjx5r4u5x2d/cia5rhesb02zmhkjxj9k1c9lb/cia5rhesb02znhkjx76dnsetw", + "https://example.com/cia5rhesb02zohkjxk9w5hbj0/cia5rhesb02zphkjxsz3yi7na/cia5rhesb02zqhkjxn35x1ss7/cia5rhesb02zrhkjxcmyedvkx", + "https://example.com/cia5rhesb02zshkjxhuw9xl6g/cia5rhesb02zthkjxnkalu85l/cia5rhesb02zuhkjx0nb3kn0f/cia5rhesb02zvhkjxekzyryxq", + "https://example.com/cia5rhesb02zwhkjxl2y2pyxt/cia5rhesb02zxhkjxeqoaa6v8/cia5rhesb02zyhkjxu5g8zso5/cia5rhesb02zzhkjx4t9www0y", + "https://example.com/cia5rhesb0300hkjxdl9hwbgu/cia5rhesb0301hkjxed3e759g/cia5rhesb0302hkjxhsksjcb6/cia5rhesb0303hkjx3fn4nqpg", + "https://example.com/cia5rhesb0304hkjx0j1w4t7r/cia5rhesb0305hkjx8v918aao/cia5rhesb0306hkjxvdvahvsq/cia5rhesb0307hkjxooa62xp7", + "https://example.com/cia5rhesb0308hkjx33xkvxg7/cia5rhesb0309hkjxws0twzy4/cia5rhesb030ahkjxtpn8o3r0/cia5rhesb030bhkjxh3wusg8f", + "https://example.com/cia5rhesb030chkjxt4rz9ksr/cia5rhesb030dhkjxklvupy26/cia5rhesb030ehkjxxf7wxi4e/cia5rhesb030fhkjxa11i0b7u", + "https://example.com/cia5rhesb030ghkjxc1beup56/cia5rhesb030hhkjxxcgs2kpz/cia5rhesb030ihkjxc7a9g482/cia5rhesb030jhkjxjb51smqy", + "https://example.com/cia5rhesb030khkjxu6a7pi1g/cia5rhesb030lhkjxo0qkosjn/cia5rhesb030mhkjxucjtxcjj/cia5rhesb030nhkjxbn92q469", + "https://example.com/cia5rhesb030ohkjxalklya9k/cia5rhesb030phkjx2s6kj633/cia5rhesb030qhkjx5dus8zhl/cia5rhesb030rhkjxexszt1qv", + "https://example.com/cia5rhesb030shkjxs325mfs5/cia5rhesb030thkjxhj36rw16/cia5rhesb030uhkjxzs6na9ek/cia5rhesb030vhkjxcahu5xdq", + "https://example.com/cia5rhesb030whkjxbq6vx29f/cia5rhesb030xhkjxzcefm4gv/cia5rhesb030yhkjxcx0o6obs/cia5rhesb030zhkjxy6a654xp", + "https://example.com/cia5rhesb0310hkjx74o7x2ol/cia5rhesb0311hkjxbaa60v2j/cia5rhesb0312hkjxws59xi68/cia5rhesb0313hkjxe024wl25", + "https://example.com/cia5rhesb0314hkjxsmo6r7qy/cia5rhesb0315hkjxyo4vsu02/cia5rhesb0316hkjxq7bv4sl0/cia5rhesb0317hkjxnu8xcpds", + "https://example.com/cia5rhesb0318hkjxfllzbwdp/cia5rhesb0319hkjx9jpu3qeb/cia5rhesb031ahkjx1yw2rxmr/cia5rhesb031bhkjx0kh6iq8e", + "https://example.com/cia5rhesb031chkjxdjrdwexa/cia5rhesb031dhkjxpdkelmj0/cia5rhesb031ehkjxo0q4659i/cia5rhesb031fhkjxqj6po4aw", + "https://example.com/cia5rhesb031ghkjxydbw4cnp/cia5rhesb031hhkjx7ak9k5ib/cia5rhesb031ihkjxshjg8guf/cia5rhesb031jhkjxgc7isxom", + "https://example.com/cia5rhesb031khkjxilu3xhrq/cia5rhesb031lhkjx7qa2tmqv/cia5rhesb031mhkjx2gqx175d/cia5rhesb031nhkjxcclzxfj5", + "https://example.com/cia5rhesb031ohkjxnqa5u0yu/cia5rhesb031phkjxvj65rgc0/cia5rhesb031qhkjxps94ct2m/cia5rhesb031rhkjx13vf7hqf", + "https://example.com/cia5rhesb031shkjx4c0rxkqe/cia5rhesb031thkjx2f1rlhtc/cia5rhesb031uhkjxc6mmo6r9/cia5rhesb031vhkjxmkdbf7tz", + "https://example.com/cia5rhesb031whkjx4wrcwah4/cia5rhesb031xhkjxjy564fgv/cia5rhesb031yhkjxel00ycv5/cia5rhesb031zhkjxwv42qge9", + "https://example.com/cia5rhesb0320hkjx579rdytw/cia5rhesb0321hkjxqwktfaa3/cia5rhesb0322hkjx5uguvgm4/cia5rhesb0323hkjxod265vm7", + "https://example.com/cia5rhesb0324hkjxo91kfm12/cia5rhesb0325hkjx7eeoo34p/cia5rhesb0326hkjxkcbju4fy/cia5rhesb0327hkjx9rgv9jej", + "https://example.com/cia5rhesb0328hkjxu29htifz/cia5rhesb0329hkjx833v3icl/cia5rhesb032ahkjxp93q4nqo/cia5rhesb032bhkjx4tktxa61", + "https://example.com/cia5rhesb032chkjxli18annx/cia5rhesb032dhkjxoin4rpsb/cia5rhesb032ehkjxkezkkq9n/cia5rhesb032fhkjxxq4syq15", + "https://example.com/cia5rhesb032ghkjxjr48ia8g/cia5rhesb032hhkjxaz6zhm4c/cia5rhesb032ihkjxriefifyj/cia5rhesb032jhkjxt06hn2ix", + "https://example.com/cia5rhesb032khkjxl0o2c8hq/cia5rhesb032lhkjxvlfg1dcu/cia5rhesb032mhkjxa6neghbc/cia5rhesb032nhkjxomcdlu3w", + "https://example.com/cia5rhesb032ohkjxnrefhx6j/cia5rhesb032phkjx05xbd8mi/cia5rhesb032qhkjx22ncbb1j/cia5rhesb032rhkjx8mqw8vvb", + "https://example.com/cia5rhesb032shkjxzw7wur7z/cia5rhesb032thkjxdybqu2ix/cia5rhesb032uhkjxjqrudsu0/cia5rhesb032vhkjx60p88zgu", + "https://example.com/cia5rhesb032whkjxsj2cgd7r/cia5rhesb032xhkjxjv4oyt79/cia5rhesb032yhkjxlzlkj3x2/cia5rhesb032zhkjxhkvllyb6", + "https://example.com/cia5rhesb0330hkjxhduykvat/cia5rhesb0331hkjxqg1x6769/cia5rhesb0332hkjx8scwhj5n/cia5rhesb0333hkjxous8ibmx", +]; diff --git a/dom/cache/test/mochitest/message_receiver.html b/dom/cache/test/mochitest/message_receiver.html new file mode 100644 index 0000000000..82cb587c72 --- /dev/null +++ b/dom/cache/test/mochitest/message_receiver.html @@ -0,0 +1,6 @@ +<!DOCTYPE html> +<script> + navigator.serviceWorker.onmessage = function(e) { + window.parent.postMessage(e.data, "*"); + }; +</script> diff --git a/dom/cache/test/mochitest/mirror.sjs b/dom/cache/test/mochitest/mirror.sjs new file mode 100644 index 0000000000..0006aba8f0 --- /dev/null +++ b/dom/cache/test/mochitest/mirror.sjs @@ -0,0 +1,5 @@ +function handleRequest(request, response) { + response.setStatusLine(request.httpVersion, 200, "OK"); + response.setHeader("Mirrored", request.getHeader("Mirror")); + response.write(request.getHeader("Mirror")); +} diff --git a/dom/cache/test/mochitest/mochitest.ini b/dom/cache/test/mochitest/mochitest.ini new file mode 100644 index 0000000000..51dc9917be --- /dev/null +++ b/dom/cache/test/mochitest/mochitest.ini @@ -0,0 +1,66 @@ +[DEFAULT] +tags = condprof +support-files = + test_cache.js + test_cache_add.js + worker_driver.js + worker_wrapper.js + frame.html + message_receiver.html + driver.js + serviceworker_driver.js + test_cache_match_request.js + test_cache_matchAll_request.js + test_cache_overwrite.js + mirror.sjs + test_cache_match_vary.js + vary.sjs + test_caches.js + test_cache_keys.js + test_cache_put.js + test_cache_requestCache.js + test_cache_delete.js + test_cache_put_reorder.js + test_cache_redirect.js + test_cache_https.js + large_url_list.js + empty.html + idle_worker.js + +[test_cache.html] +[test_cache_add.html] +[test_cache_match_request.html] +[test_cache_matchAll_request.html] +[test_cache_overwrite.html] +skip-if = + http3 +[test_cache_match_vary.html] +[test_caches.html] +[test_cache_keys.html] +[test_cache_put.html] +[test_cache_requestCache.html] +[test_cache_delete.html] +[test_cache_put_reorder.html] +[test_cache_https.html] +[test_cache_redirect.html] +[test_cache_restart.html] +[test_cache_shrink.html] +[test_cache_orphaned_cache.html] +[test_cache_orphaned_body.html] +scheme=https +skip-if = + os == "linux" && bits == 64 && debug # Bug 1749068 +[test_cache_padding.html] +skip-if = verify + os == 'android' # bug 1468434 for android + condprof #: "The opaque response should be removed by caches.delete() and cache.delete()" +[test_cache_untrusted.html] +skip-if = + http3 +[test_cache_updateUsage.html] +[test_chrome_constructor.html] +[test_cache_worker_gc.html] +scheme=https +skip-if = + os == "linux" && bits == 64 && debug # Bug 1749068 +[test_cache_tons_of_fd.html] diff --git a/dom/cache/test/mochitest/serviceworker_driver.js b/dom/cache/test/mochitest/serviceworker_driver.js new file mode 100644 index 0000000000..3406cdcc05 --- /dev/null +++ b/dom/cache/test/mochitest/serviceworker_driver.js @@ -0,0 +1,38 @@ +// Any copyright is dedicated to the Public Domain. +// http://creativecommons.org/publicdomain/zero/1.0/ + +function serviceWorkerTestExec(testFile) { + return new Promise(function (resolve, reject) { + function setupSW(registration) { + var worker = + registration.installing || registration.waiting || registration.active; + var iframe; + + window.addEventListener("message", function onMessage(event) { + if (event.data.context != "ServiceWorker") { + return; + } + if (event.data.type == "finish") { + window.removeEventListener("message", onMessage); + iframe.remove(); + registration.unregister().then(resolve).catch(reject); + } else if (event.data.type == "status") { + ok(event.data.status, event.data.context + ": " + event.data.msg); + } + }); + + worker.onerror = reject; + + iframe = document.createElement("iframe"); + iframe.src = "message_receiver.html"; + iframe.onload = function () { + worker.postMessage({ script: testFile }); + }; + document.body.appendChild(iframe); + } + + navigator.serviceWorker + .register("worker_wrapper.js", { scope: "." }) + .then(setupSW); + }); +} diff --git a/dom/cache/test/mochitest/test_cache.html b/dom/cache/test/mochitest/test_cache.html new file mode 100644 index 0000000000..1ac3447120 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache.html @@ -0,0 +1,20 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Validate Interfaces Exposed to Workers</title> + <script src="/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> + <script type="text/javascript" src="driver.js"></script> +</head> +<body> +<iframe id="frame"></iframe> +<script class="testbody" type="text/javascript"> + runTests("test_cache.js") + .then(function() { + SimpleTest.finish(); + }); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_cache.js b/dom/cache/test/mochitest/test_cache.js new file mode 100644 index 0000000000..7d6d01b34b --- /dev/null +++ b/dom/cache/test/mochitest/test_cache.js @@ -0,0 +1,195 @@ +/* global context testDone:true */ + +var c = null; +var request = "https://example.com/hmm?q=foobar" + context; +var response = new Response("This is some Response!"); +var name = "snafu" + context; +var foobar = "foobar" + context; + +ok(!!caches, "caches object should be available on global"); +caches + .open(name) + .then(function (openCache) { + ok( + openCache instanceof Cache, + "cache object should be resolved from caches.open" + ); + return caches.has(name); + }) + .then(function (hasResult) { + ok(hasResult, "caches.has() should resolve true"); + return caches.keys(); + }) + .then(function (keys) { + ok(!!keys, "caches.keys() should resolve to a truthy value"); + ok( + keys.length >= 1, + "caches.keys() should resolve to an array of length at least 1" + ); + ok( + keys.includes(name), + "caches.keys() should resolve to an array containing key" + ); + return caches.delete(name); + }) + .then(function (deleteResult) { + ok(deleteResult, "caches.delete() should resolve true"); + return caches.has(name); + }) + .then(function (hasMissingCache) { + ok(!hasMissingCache, "missing key should return false from has"); + }) + .then(function () { + return caches.open(name); + }) + .then(function (snafu) { + return snafu.keys(); + }) + .then(function (empty) { + is(0, empty.length, "cache.keys() should resolve to an array of length 0"); + }) + .then(function () { + return caches.open(name); + }) + .then(function (snafu) { + var req = "./cachekey"; + var res = new Response("Hello world"); + return snafu + .put("ftp://invalid", res) + .then(function () { + ok(false, "This should fail"); + }) + .catch(function (err) { + is( + err.name, + "TypeError", + "put() should throw TypeError for invalid scheme" + ); + return snafu.put(req, res); + }) + .then(function (v) { + return snafu; + }); + }) + .then(function (snafu) { + return Promise.all([snafu, snafu.keys()]); + }) + .then(function (args) { + var snafu = args[0]; + var keys = args[1]; + is(1, keys.length, "cache.keys() should resolve to an array of length 1"); + ok(keys[0] instanceof Request, "key should be a Request"); + ok(keys[0].url.match(/cachekey$/), "Request URL should match original"); + return Promise.all([ + snafu, + snafu.match(keys[0]), + snafu.match("ftp://invalid"), + ]); + }) + .then(function (args) { + var snafu = args[0]; + var res = args[1]; + ok(res instanceof Response, "value should be a Response"); + is(res.status, 200, "Response status should be 200"); + is( + undefined, + args[2], + "Match with invalid scheme should resolve undefined" + ); + return Promise.all([snafu, snafu.put("./cachekey2", res)]); + }) + .then(function (args) { + var snafu = args[0]; + return snafu.match("./cachekey2"); + }) + .then(function (res) { + return res.text().then(function (v) { + is(v, "Hello world", "Response body should match original"); + }); + }) + .then(function () { + // FIXME(nsm): Can't use a Request object for now since the operations + // consume it's 'body'. See + // https://github.com/slightlyoff/ServiceWorker/issues/510. + return caches.open(foobar); + }) + .then(function (openCache) { + c = openCache; + return c.put(request, response); + }) + .then(function (putResponse) { + is(putResponse, undefined, "The promise should resolve to undefined"); + return c.keys(request); + }) + .then(function (keys) { + ok(keys, "Valid keys object expected"); + is(keys.length, 1, "Only one key is expected"); + return c.keys(); + }) + .then(function (keys) { + ok(keys, "Valid keys object expected"); + is(keys.length, 1, "Only one key is expected"); + return c.matchAll(request); + }) + .then(function (matchAllResponses) { + ok(matchAllResponses, "matchAll should succeed"); + is(matchAllResponses.length, 1, "Only one match is expected"); + return c.match(request); + }) + .then(function (matchResponse) { + ok(matchResponse, "match should succeed"); + return caches.match(request); + }) + .then(function (storageMatchResponse) { + ok(storageMatchResponse, "storage match should succeed"); + return caches.match(request, { cacheName: foobar }); + }) + .then(function (storageMatchResponse) { + ok(storageMatchResponse, "storage match with cacheName should succeed"); + var request2 = new Request("https://example.com/hmm?q=snafu" + context); + return c.match(request2, { ignoreSearch: true }); + }) + .then(function (match2Response) { + ok(match2Response, "match should succeed"); + return c.delete(request); + }) + .then(function (deleteResult) { + ok(deleteResult, "delete should succeed"); + return c.keys(); + }) + .then(function (keys) { + ok(keys, "Valid keys object expected"); + is(keys.length, 0, "Zero keys is expected"); + return c.matchAll(request); + }) + .then(function (matchAll2Responses) { + ok(matchAll2Responses, "matchAll should succeed"); + is(matchAll2Responses.length, 0, "Zero matches is expected"); + return caches.has(foobar); + }) + .then(function (hasResult) { + ok(hasResult, "has should succeed"); + return caches.keys(); + }) + .then(function (keys) { + ok(keys, "Valid keys object expected"); + ok(keys.length >= 2, "At least two keys are expected"); + ok(keys.includes(name), "snafu should exist"); + ok( + keys.indexOf(foobar) >= keys.indexOf(name), + "foobar should come after it" + ); + return caches.delete(foobar); + }) + .then(function (deleteResult) { + ok(deleteResult, "delete should succeed"); + return caches.has(foobar); + }) + .then(function (hasMissingCache) { + ok(!hasMissingCache, "has should have a result"); + return caches.delete(name); + }) + .then(function (deleteResult) { + ok(deleteResult, "delete should succeed"); + testDone(); + }); diff --git a/dom/cache/test/mochitest/test_cache_add.html b/dom/cache/test/mochitest/test_cache_add.html new file mode 100644 index 0000000000..e8a141dd64 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_add.html @@ -0,0 +1,20 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Validate Interfaces Exposed to Workers</title> + <script src="/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> + <script type="text/javascript" src="driver.js"></script> +</head> +<body> +<iframe id="frame"></iframe> +<script class="testbody" type="text/javascript"> + runTests("test_cache_add.js") + .then(function() { + SimpleTest.finish(); + }); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_cache_add.js b/dom/cache/test/mochitest/test_cache_add.js new file mode 100644 index 0000000000..9f96b985f0 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_add.js @@ -0,0 +1,75 @@ +/* global context testDone:true */ + +var singleUrl = "./test_cache_add.js"; +var urlList = ["./empty.html", "./frame.html", "./test_cache.js"]; +var cache; +var name = "adder" + context; +caches + .open(name) + .then(function (openCache) { + cache = openCache; + return cache.add("ftp://example.com/invalid" + context); + }) + .catch(function (err) { + is( + err.name, + "TypeError", + "add() should throw TypeError for invalid scheme" + ); + return cache.addAll([ + "https://example.com/valid" + context, + "ftp://example.com/invalid" + context, + ]); + }) + .catch(function (err) { + is( + err.name, + "TypeError", + "addAll() should throw TypeError for invalid scheme" + ); + var promiseList = urlList.map(function (url) { + return cache.match(url); + }); + promiseList.push(cache.match(singleUrl)); + return Promise.all(promiseList); + }) + .then(function (resultList) { + is(urlList.length + 1, resultList.length, "Expected number of results"); + resultList.every(function (result) { + is(undefined, result, "URLs should not already be in the cache"); + }); + return cache.add(singleUrl); + }) + .then(function (result) { + is(undefined, result, "Successful add() should resolve undefined"); + return cache.addAll(urlList); + }) + .then(function (result) { + is(undefined, result, "Successful addAll() should resolve undefined"); + var promiseList = urlList.map(function (url) { + return cache.match(url); + }); + promiseList.push(cache.match(singleUrl)); + return Promise.all(promiseList); + }) + .then(function (resultList) { + is(urlList.length + 1, resultList.length, "Expected number of results"); + resultList.every(function (result) { + ok(!!result, "Responses should now be in cache for each URL."); + }); + return cache.matchAll(); + }) + .then(function (resultList) { + is(urlList.length + 1, resultList.length, "Expected number of results"); + resultList.every(function (result) { + ok(!!result, "Responses should now be in cache for each URL."); + }); + return caches.delete(name); + }) + .then(function () { + testDone(); + }) + .catch(function (err) { + ok(false, "Caught error: " + err); + testDone(); + }); diff --git a/dom/cache/test/mochitest/test_cache_delete.html b/dom/cache/test/mochitest/test_cache_delete.html new file mode 100644 index 0000000000..3ec7a9666c --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_delete.html @@ -0,0 +1,20 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Validate the Cache.delete() method</title> + <script src="/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> + <script type="text/javascript" src="driver.js"></script> +</head> +<body> +<iframe id="frame"></iframe> +<script class="testbody" type="text/javascript"> + runTests("test_cache_delete.js") + .then(function() { + SimpleTest.finish(); + }); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_cache_delete.js b/dom/cache/test/mochitest/test_cache_delete.js new file mode 100644 index 0000000000..67ded1affa --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_delete.js @@ -0,0 +1,132 @@ +/* global context testDone:true */ + +var name = "delete" + context; + +function setupTest(reqs) { + return new Promise(function (resolve, reject) { + var cache; + caches + .open(name) + .then(function (c) { + cache = c; + return c.addAll(reqs); + }) + .then(function () { + resolve(cache); + }) + .catch(function (err) { + reject(err); + }); + }); +} + +function testBasics() { + var tests = [ + "//mochi.test:8888/?foo" + context, + "//mochi.test:8888/?bar" + context, + ]; + var cache; + return setupTest(tests) + .then(function (c) { + cache = c; + return cache.delete("//mochi.test:8888/?baz"); + }) + .then(function (deleted) { + ok(!deleted, "Deleting a non-existing entry should fail"); + return cache.keys(); + }) + .then(function (keys) { + is(keys.length, 2, "No entries from the cache should be deleted"); + return cache.delete(tests[0]); + }) + .then(function (deleted) { + ok(deleted, "Deleting an existing entry should succeed"); + return cache.keys(); + }) + .then(function (keys) { + is(keys.length, 1, "Only one entry should exist now"); + ok(keys[0].url.includes(tests[1]), "The correct entry must be deleted"); + }); +} + +function testFragment() { + var tests = [ + "//mochi.test:8888/?foo" + context, + "//mochi.test:8888/?bar" + context, + "//mochi.test:8888/?baz" + context + "#fragment", + ]; + var cache; + return setupTest(tests) + .then(function (c) { + cache = c; + return cache.delete(tests[0] + "#fragment"); + }) + .then(function (deleted) { + ok(deleted, "Deleting an existing entry should succeed"); + return cache.keys(); + }) + .then(function (keys) { + is(keys.length, 2, "Only one entry should exist now"); + ok(keys[0].url.includes(tests[1]), "The correct entry must be deleted"); + ok( + keys[1].url.includes(tests[2].replace("#fragment", "")), + "The correct entry must be deleted" + ); + // Now, delete a request that was added with a fragment + return cache.delete("//mochi.test:8888/?baz" + context); + }) + .then(function (deleted) { + ok(deleted, "Deleting an existing entry should succeed"); + return cache.keys(); + }) + .then(function (keys) { + is(keys.length, 1, "Only one entry should exist now"); + ok(keys[0].url.includes(tests[1]), "3The correct entry must be deleted"); + }); +} + +function testInterleaved() { + var tests = [ + "//mochi.test:8888/?foo" + context, + "//mochi.test:8888/?bar" + context, + ]; + var newURL = "//mochi.test:8888/?baz" + context; + var cache; + return setupTest(tests) + .then(function (c) { + cache = c; + // Simultaneously add and delete a request + return Promise.all([cache.delete(newURL), cache.add(newURL)]); + }) + .then(function (result) { + ok(!result[1], "deletion should fail"); + return cache.keys(); + }) + .then(function (keys) { + is(keys.length, 3, "Tree entries should still exist"); + ok(keys[0].url.includes(tests[0]), "The correct entry must be deleted"); + ok(keys[1].url.includes(tests[1]), "The correct entry must be deleted"); + ok( + keys[2].url.includes(newURL), + "The new entry should be correctly inserted" + ); + }); +} + +// Make sure to clean up after each test step. +function step(testPromise) { + return testPromise.then(function () { + caches.delete(name); + }); +} + +step(testBasics()) + .then(function () { + return step(testFragment()); + }) + .then(function () { + return step(testInterleaved()); + }) + .then(function () { + testDone(); + }); diff --git a/dom/cache/test/mochitest/test_cache_https.html b/dom/cache/test/mochitest/test_cache_https.html new file mode 100644 index 0000000000..82a2823379 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_https.html @@ -0,0 +1,20 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Validate Interfaces Exposed to Workers</title> + <script src="/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> + <script type="text/javascript" src="driver.js"></script> +</head> +<body> +<iframe id="frame"></iframe> +<script class="testbody" type="text/javascript"> + runTests("test_cache_https.js") + .then(function() { + SimpleTest.finish(); + }); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_cache_https.js b/dom/cache/test/mochitest/test_cache_https.js new file mode 100644 index 0000000000..682938a664 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_https.js @@ -0,0 +1,36 @@ +/* global context testDone:true */ + +var cache = null; +var name = "https_" + context; +var urlBase = "https://example.com/tests/dom/cache/test/mochitest"; +var url1 = urlBase + "/test_cache.js"; +var url2 = urlBase + "/test_cache_add.js"; + +function addOpaque(c, url) { + return fetch(new Request(url, { mode: "no-cors" })).then(function (response) { + return c.put(url, response); + }); +} + +caches + .open(name) + .then(function (c) { + cache = c; + return Promise.all([addOpaque(cache, url1), addOpaque(cache, url2)]); + }) + .then(function () { + return cache.delete(url1); + }) + .then(function (result) { + ok(result, "Cache entry should be deleted"); + return cache.delete(url2); + }) + .then(function (result) { + ok(result, "Cache entry should be deleted"); + cache = null; + return caches.delete(name); + }) + .then(function (result) { + ok(result, "Cache should be deleted"); + testDone(); + }); diff --git a/dom/cache/test/mochitest/test_cache_keys.html b/dom/cache/test/mochitest/test_cache_keys.html new file mode 100644 index 0000000000..30aa5c9d19 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_keys.html @@ -0,0 +1,20 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Validate the Cache.keys() method</title> + <script src="/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> + <script type="text/javascript" src="driver.js"></script> +</head> +<body> +<iframe id="frame"></iframe> +<script class="testbody" type="text/javascript"> + runTests("test_cache_keys.js") + .then(function() { + SimpleTest.finish(); + }); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_cache_keys.js b/dom/cache/test/mochitest/test_cache_keys.js new file mode 100644 index 0000000000..1eda988af2 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_keys.js @@ -0,0 +1,98 @@ +/* global context testDone:true */ + +var name = "keys" + context; +var c; + +var tests = [ + "//mochi.test:8888/?page" + context, + "//mochi.test:8888/?another" + context, +]; + +caches + .open(name) + .then(function (cache) { + c = cache; + return c.addAll(tests); + }) + .then(function () { + // Add another cache entry using Cache.add + var another = "//mochi.test:8888/?yetanother" + context; + tests.push(another); + return c.add(another); + }) + .then(function () { + // Add another cache entry with URL fragment using Cache.add + var anotherWithFragment = + "//mochi.test:8888/?fragment" + context + "#fragment"; + tests.push(anotherWithFragment); + return c.add(anotherWithFragment); + }) + .then(function () { + return c.keys(); + }) + .then(function (keys) { + is(keys.length, tests.length, "Same number of elements"); + // Verify both the insertion order of the requests and their validity. + keys.forEach(function (r, i) { + ok(r instanceof Request, "Valid request object"); + ok(r.url.includes(tests[i]), "Valid URL"); + }); + // Try searching for just one request + return c.keys(tests[1]); + }) + .then(function (keys) { + is(keys.length, 1, "One match should be found"); + ok(keys[0].url.includes(tests[1]), "Valid URL"); + // Try to see if ignoreSearch works as expected. + return c.keys(new Request("//mochi.test:8888/?foo"), { + ignoreSearch: true, + }); + }) + .then(function (key_arr) { + is(key_arr.length, tests.length, "Same number of elements"); + key_arr.forEach(function (r, i) { + ok(r instanceof Request, "Valid request object"); + ok(r.url.includes(tests[i]), "Valid URL"); + }); + // Try to see if ignoreMethod works as expected + return Promise.all( + ["POST", "PUT", "DELETE", "OPTIONS"].map(function (method) { + var req = new Request(tests[2], { method }); + return c + .keys(req) + .then(function (keys) { + is( + keys.length, + 0, + "No request should be matched without ignoreMethod" + ); + return c.keys(req, { ignoreMethod: true }); + }) + .then(function (keys) { + is(keys.length, 1, "One match should be found"); + ok(keys[0].url.includes(tests[2]), "Valid URL"); + }); + }) + ); + }) + .then(function () { + // But HEAD should be allowed only when ignoreMethod is ture + return c.keys(new Request(tests[0], { method: "HEAD" }), { + ignoreMethod: true, + }); + }) + .then(function (keys) { + is(keys.length, 1, "One match should be found"); + ok(keys[0].url.includes(tests[0]), "Valid URL"); + // Make sure cacheName is ignored. + return c.keys(tests[0], { cacheName: "non-existing-cache" }); + }) + .then(function (keys) { + is(keys.length, 1, "One match should be found"); + ok(keys[0].url.includes(tests[0]), "Valid URL"); + return caches.delete(name); + }) + .then(function (deleted) { + ok(deleted, "The cache should be successfully deleted"); + testDone(); + }); diff --git a/dom/cache/test/mochitest/test_cache_matchAll_request.html b/dom/cache/test/mochitest/test_cache_matchAll_request.html new file mode 100644 index 0000000000..2536f75bb6 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_matchAll_request.html @@ -0,0 +1,20 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Validate calling matchAll with a Request object</title> + <script src="/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> + <script type="text/javascript" src="driver.js"></script> +</head> +<body> +<iframe id="frame"></iframe> +<script class="testbody" type="text/javascript"> + runTests("test_cache_matchAll_request.js") + .then(function() { + SimpleTest.finish(); + }); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_cache_matchAll_request.js b/dom/cache/test/mochitest/test_cache_matchAll_request.js new file mode 100644 index 0000000000..615a69b6c6 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_matchAll_request.js @@ -0,0 +1,273 @@ +/* global context testDone:true */ + +var req1 = new Request("//mochi.test:8888/?1&" + context + "#fragment"); +var req2 = new Request("//mochi.test:8888/?2&" + context); +var req3 = new Request("//mochi.test:8888/?3&" + context); +var requestWithAltQS = new Request("//mochi.test:8888/?queryString"); +var unknownRequest = new Request( + "//mochi.test:8888/non/existing/path?" + context +); +var response1, response3; +var c; +var response1Text, response3Text; +var name = "matchAll-request" + context; + +function checkResponse(r, response, responseText) { + ok(r !== response, "The objects should not be the same"); + is( + r.url, + response.url.replace("#fragment", ""), + "The URLs should be the same" + ); + is(r.status, response.status, "The status codes should be the same"); + is(r.type, response.type, "The response types should be the same"); + is(r.ok, response.ok, "Both responses should have succeeded"); + is( + r.statusText, + response.statusText, + "Both responses should have the same status text" + ); + return r.text().then(function (text) { + // Avoid dumping out the large response text to the log if they're equal. + if (text !== responseText) { + is(text, responseText, "The response body should be correct"); + } + }); +} + +fetch(new Request(req1)) + .then(function (r) { + response1 = r; + return response1.text(); + }) + .then(function (text) { + response1Text = text; + return fetch(new Request(req3)); + }) + .then(function (r) { + response3 = r; + return response3.text(); + }) + .then(function (text) { + response3Text = text; + return testRequest( + req1, + req2, + req3, + unknownRequest, + requestWithAltQS, + req1.url.replace("#fragment", "#other") + ); + }) + .then(function () { + return testRequest( + req1.url, + req2.url, + req3.url, + unknownRequest.url, + requestWithAltQS.url, + req1.url.replace("#fragment", "#other") + ); + }) + .then(function () { + testDone(); + }); + +// The request arguments can either be a URL string, or a Request object. +function testRequest( + request1, + request2, + request3, + unknownReq, + requestWithAlternateQueryString, + requestWithDifferentFragment +) { + return caches + .open(name) + .then(function (cache) { + c = cache; + return c.add(request1); + }) + .then(function () { + return c.add(request3); + }) + .then(function () { + return Promise.all( + ["HEAD", "POST", "PUT", "DELETE", "OPTIONS"].map(function (method) { + var r = new Request(request1, { method }); + return c.add(r).then( + function () { + ok(false, "Promise should be rejected"); + }, + function (err) { + is( + err.name, + "TypeError", + "Adding a request with type '" + method + "' should fail" + ); + } + ); + }) + ); + }) + .then(function () { + return c.matchAll(request1); + }) + .then(function (r) { + is(r.length, 1, "Should only find 1 item"); + return checkResponse(r[0], response1, response1Text); + }) + .then(function () { + return c.matchAll(new Request(request1, { method: "HEAD" })); + }) + .then(function (r) { + is(r.length, 0, "Should be an empty array when match a HEAD request."); + return c.matchAll(new Request(request1, { method: "HEAD" }), { + ignoreMethod: true, + }); + }) + .then(function (r) { + is(r.length, 1, "Should only find 1 item"); + return checkResponse(r[0], response1, response1Text); + }) + .then(function () { + return Promise.all( + ["HEAD", "POST", "PUT", "DELETE", "OPTIONS"].map(function (method) { + var req = new Request(request1, { method }); + return c + .matchAll(req) + .then(function (r) { + is( + r.length, + 0, + "Searching for a request with a non-GET method should not succeed" + ); + return c.matchAll(req, { ignoreMethod: true }); + }) + .then(function (r) { + is(r.length, 1, "Should only find 1 item"); + return checkResponse(r[0], response1, response1Text); + }); + }) + ); + }) + .then(function () { + return c.matchAll(requestWithDifferentFragment); + }) + .then(function (r) { + is(r.length, 1, "Should only find 1 item"); + return checkResponse(r[0], response1, response1Text); + }) + .then(function () { + return c.matchAll(requestWithAlternateQueryString, { + ignoreSearch: true, + }); + }) + .then(function (r) { + is(r.length, 2, "Should find 2 items"); + return Promise.all([ + checkResponse(r[0], response1, response1Text), + checkResponse(r[1], response3, response3Text), + ]); + }) + .then(function () { + return c.matchAll(request3); + }) + .then(function (r) { + is(r.length, 1, "Should only find 1 item"); + return checkResponse(r[0], response3, response3Text); + }) + .then(function () { + return c.matchAll(); + }) + .then(function (r) { + is(r.length, 2, "Should find 2 items"); + return Promise.all([ + checkResponse(r[0], response1, response1Text), + checkResponse(r[1], response3, response3Text), + ]); + }) + .then(function () { + return caches + .match(request1, { cacheName: name + "mambojambo" }) + .then(function () { + is( + typeof r, + "undefined", + "Searching in the wrong cache should resolve to undefined" + ); + return caches.has(name + "mambojambo"); + }) + .then(function (hasCache) { + ok(!hasCache, "The wrong cache should still not exist"); + }); + }) + .then(function () { + return c.matchAll(unknownReq); + }) + .then(function (r) { + is(r.length, 0, "Searching for an unknown request should not succeed"); + return caches.match(unknownReq, { cacheName: name }); + }) + .then(function (r) { + is( + typeof r, + "undefined", + "Searching for an unknown request should not succeed" + ); + // Make sure that cacheName is ignored on Cache + return c.matchAll(request1, { cacheName: name + "mambojambo" }); + }) + .then(function (r) { + is(r.length, 1, "Should only find 1 item"); + return checkResponse(r[0], response1, response1Text); + }) + .then(function () { + return caches.delete(name); + }) + .then(function (success) { + ok(success, "We should be able to delete the cache successfully"); + // Make sure that the cache is still usable after deletion. + return c.matchAll(request1); + }) + .then(function (r) { + is(r.length, 1, "Should only find 1 item"); + return checkResponse(r[0], response1, response1Text); + }) + .then(function () { + return c.matchAll(request3); + }) + .then(function (r) { + is(r.length, 1, "Should only find 1 item"); + return checkResponse(r[0], response3, response3Text); + }) + .then(function () { + return c.matchAll(); + }) + .then(function (r) { + is(r.length, 2, "Should find 2 items"); + return Promise.all([ + checkResponse(r[0], response1, response1Text), + checkResponse(r[1], response3, response3Text), + ]); + }) + .then(function () { + // Now, drop the cache, reopen and verify that we can't find the request any more. + c = null; + return caches.open(name); + }) + .then(function (cache) { + return cache.matchAll(); + }) + .then(function (r) { + is( + r.length, + 0, + "Searching in the cache after deletion should not succeed" + ); + return caches.delete(name); + }) + .then(function (deleted) { + ok(deleted, "The cache should be deleted successfully"); + }); +} diff --git a/dom/cache/test/mochitest/test_cache_match_request.html b/dom/cache/test/mochitest/test_cache_match_request.html new file mode 100644 index 0000000000..2cd6132b11 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_match_request.html @@ -0,0 +1,20 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Validate calling match with a Request object</title> + <script src="/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> + <script type="text/javascript" src="driver.js"></script> +</head> +<body> +<iframe id="frame"></iframe> +<script class="testbody" type="text/javascript"> + runTests("test_cache_match_request.js") + .then(function() { + SimpleTest.finish(); + }); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_cache_match_request.js b/dom/cache/test/mochitest/test_cache_match_request.js new file mode 100644 index 0000000000..f7a7001444 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_match_request.js @@ -0,0 +1,238 @@ +/* global context testDone:true */ + +var req = new Request("//mochi.test:8888/?" + context + "#fragment"); +var requestWithAltQS = new Request("//mochi.test:8888/?queryString"); +var unknownReq = new Request("//mochi.test:8888/non/existing/path?" + context); +var response; +var c; +var responseText; +var name = "match-request" + context; + +function checkResponse(r, expectedBody) { + if (expectedBody === undefined) { + expectedBody = responseText; + } + ok(r !== response, "The objects should not be the same"); + is( + r.url, + response.url.replace("#fragment", ""), + "The URLs should be the same" + ); + is(r.status, response.status, "The status codes should be the same"); + is(r.type, response.type, "The response types should be the same"); + is(r.ok, response.ok, "Both responses should have succeeded"); + is( + r.statusText, + response.statusText, + "Both responses should have the same status text" + ); + return r.text().then(function (text) { + // Avoid dumping out the large response text to the log if they're equal. + if (text !== expectedBody) { + is(text, responseText, "The response body should be correct"); + } + }); +} +fetch(new Request(req)) + .then(function (r) { + response = r; + return response.text(); + }) + .then(function (text) { + responseText = text; + return testRequest( + req, + unknownReq, + requestWithAltQS, + req.url.replace("#fragment", "#other") + ); + }) + .then(function () { + return testRequest( + req.url, + unknownReq.url, + requestWithAltQS.url, + req.url.replace("#fragment", "#other") + ); + }) + .then(function () { + testDone(); + }); +// The request argument can either be a URL string, or a Request object. +function testRequest( + request, + unknownRequest, + requestWithAlternateQueryString, + requestWithDifferentFragment +) { + return caches + .open(name) + .then(function (cache) { + c = cache; + return c.add(request); + }) + .then(function () { + return Promise.all( + ["HEAD", "POST", "PUT", "DELETE", "OPTIONS"].map(function (method) { + var r = new Request(request, { method }); + return c.add(r).then( + function () { + ok(false, "Promise should be rejected"); + }, + function (err) { + is( + err.name, + "TypeError", + "Adding a request with type '" + method + "' should fail" + ); + } + ); + }) + ); + }) + .then(function () { + return c.match(request); + }) + .then(function (r) { + return checkResponse(r); + }) + .then(function () { + return c.match(new Request(request, { method: "HEAD" })); + }) + .then(function (r) { + is( + typeof r, + "undefined", + "Searching for an HEAD request should not succeed" + ); + return c.match(new Request(request, { method: "HEAD" }), { + ignoreMethod: true, + }); + }) + .then(function (r) { + return checkResponse(r); + }) + .then(function () { + return Promise.all( + ["HEAD", "POST", "PUT", "DELETE", "OPTIONS"].map(function (method) { + var req1 = new Request(request, { method }); + return c + .match(req1) + .then(function (r) { + is( + typeof r, + "undefined", + "Searching for a request with a non-GET method should not succeed" + ); + return c.match(req1, { ignoreMethod: true }); + }) + .then(function (r) { + return checkResponse(r); + }); + }) + ); + }) + .then(function () { + return caches.match(request); + }) + .then(function (r) { + return checkResponse(r); + }) + .then(function () { + return caches.match(requestWithDifferentFragment); + }) + .then(function (r) { + return checkResponse(r); + }) + .then(function () { + return caches.match(requestWithAlternateQueryString, { + ignoreSearch: true, + cacheName: name, + }); + }) + .then(function (r) { + return checkResponse(r); + }) + .then(function () { + return caches.match(request, { cacheName: name }); + }) + .then(function (r) { + return checkResponse(r); + }) + .then(function () { + return caches + .match(request, { cacheName: name + "mambojambo" }) + .then(function (result) { + is( + typeof r, + "undefined", + "Searching in the wrong cache should resolve to undefined" + ); + return caches.has(name + "mambojambo"); + }) + .then(function (hasCache) { + ok(!hasCache, "The wrong cache should still not exist"); + }); + }) + .then(function () { + // Make sure that cacheName is ignored on Cache + return c.match(request, { cacheName: name + "mambojambo" }); + }) + .then(function (r) { + return checkResponse(r); + }) + .then(function () { + return c.match(unknownRequest); + }) + .then(function (r) { + is( + typeof r, + "undefined", + "Searching for an unknown request should not succeed" + ); + return caches.match(unknownRequest); + }) + .then(function (r) { + is( + typeof r, + "undefined", + "Searching for an unknown request should not succeed" + ); + return caches.match(unknownRequest, { cacheName: name }); + }) + .then(function (r) { + is( + typeof r, + "undefined", + "Searching for an unknown request should not succeed" + ); + return caches.delete(name); + }) + .then(function (success) { + ok(success, "We should be able to delete the cache successfully"); + // Make sure that the cache is still usable after deletion. + return c.match(request); + }) + .then(function (r) { + return checkResponse(r); + }) + .then(function () { + // Now, drop the cache, reopen and verify that we can't find the request any more. + c = null; + return caches.open(name); + }) + .then(function (cache) { + return cache.match(request); + }) + .then(function (r) { + is( + typeof r, + "undefined", + "Searching in the cache after deletion should not succeed" + ); + return caches.delete(name); + }) + .then(function (deleted) { + ok(deleted, "The cache should be deleted successfully"); + }); +} diff --git a/dom/cache/test/mochitest/test_cache_match_vary.html b/dom/cache/test/mochitest/test_cache_match_vary.html new file mode 100644 index 0000000000..17b7a4a9a5 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_match_vary.html @@ -0,0 +1,20 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Validate calling match with requests involving the Vary header</title> + <script src="/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> + <script type="text/javascript" src="driver.js"></script> +</head> +<body> +<iframe id="frame"></iframe> +<script class="testbody" type="text/javascript"> + runTests("test_cache_match_vary.js") + .then(function() { + SimpleTest.finish(); + }); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_cache_match_vary.js b/dom/cache/test/mochitest/test_cache_match_vary.js new file mode 100644 index 0000000000..87e21826db --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_match_vary.js @@ -0,0 +1,545 @@ +/* global context testDone:true */ + +var requestURL = + "//mochi.test:8888/tests/dom/cache/test/mochitest/vary.sjs?" + context; +var name = "match-vary" + context; + +function checkResponse(r, response, responseText) { + ok(r !== response, "The objects should not be the same"); + is( + r.url, + response.url.replace("#fragment", ""), + "The URLs should be the same" + ); + is(r.status, response.status, "The status codes should be the same"); + is(r.type, response.type, "The response types should be the same"); + is(r.ok, response.ok, "Both responses should have succeeded"); + is( + r.statusText, + response.statusText, + "Both responses should have the same status text" + ); + is( + r.headers.get("Vary"), + response.headers.get("Vary"), + "Both responses should have the same Vary header" + ); + return r.text().then(function (text) { + is(text, responseText, "The response body should be correct"); + }); +} + +// Returns a Promise that will be resolved to an object with the following +// properties: +// * cache: A Cache object that contains one entry fetched with headers. +// * response: A Response object which is the result of fetching a request +// with the specified headers. +// * responseText: The body of the above response object. +function setupTest(headers) { + return setupTestMultipleEntries([headers]).then(function (test) { + return { + response: test.response[0], + responseText: test.responseText[0], + cache: test.cache, + }; + }); +} +function setupTestMultipleEntries(headers) { + ok(Array.isArray(headers), "headers should be an array"); + return new Promise(function (resolve, reject) { + var response, responseText, cache; + Promise.all( + headers.map(function (h) { + return fetch(requestURL, { headers: h }); + }) + ) + .then(function (res) { + response = res; + return Promise.all( + response.map(function (r) { + return r.text(); + }) + ); + }) + .then(function (text) { + responseText = text; + return caches.open(name); + }) + .then(function (c) { + cache = c; + return Promise.all( + headers.map(function (h) { + return c.add(new Request(requestURL, { headers: h })); + }) + ); + }) + .then(function () { + resolve({ response, responseText, cache }); + }) + .catch(function (err) { + reject(err); + }); + }); +} + +function testBasics() { + var test; + return setupTest({ WhatToVary: "Custom" }) + .then(function (t) { + test = t; + // Ensure that searching without specifying a Custom header succeeds. + return test.cache.match(requestURL); + }) + .then(function (r) { + return checkResponse(r, test.response, test.responseText); + }) + .then(function () { + // Ensure that searching with a non-matching value for the Custom header fails. + return test.cache.match( + new Request(requestURL, { headers: { Custom: "foo=bar" } }) + ); + }) + .then(function (r) { + is( + typeof r, + "undefined", + "Searching for a request with an unknown Vary header should not succeed" + ); + // Ensure that searching with a non-matching value for the Custom header but with ignoreVary set succeeds. + return test.cache.match( + new Request(requestURL, { headers: { Custom: "foo=bar" } }), + { ignoreVary: true } + ); + }) + .then(function (r) { + return checkResponse(r, test.response, test.responseText); + }); +} + +function testBasicKeys() { + function checkRequest(reqs) { + is(reqs.length, 1, "One request expected"); + ok(reqs[0].url.includes(requestURL), "The correct request expected"); + ok( + reqs[0].headers.get("WhatToVary"), + "Custom", + "The correct request headers expected" + ); + } + var test; + return setupTest({ WhatToVary: "Custom" }) + .then(function (t) { + test = t; + // Ensure that searching without specifying a Custom header succeeds. + return test.cache.keys(requestURL); + }) + .then(function (r) { + return checkRequest(r); + }) + .then(function () { + // Ensure that searching with a non-matching value for the Custom header fails. + return test.cache.keys( + new Request(requestURL, { headers: { Custom: "foo=bar" } }) + ); + }) + .then(function (r) { + is( + r.length, + 0, + "Searching for a request with an unknown Vary header should not succeed" + ); + // Ensure that searching with a non-matching value for the Custom header but with ignoreVary set succeeds. + return test.cache.keys( + new Request(requestURL, { headers: { Custom: "foo=bar" } }), + { ignoreVary: true } + ); + }) + .then(function (r) { + return checkRequest(r); + }); +} + +function testStar() { + function ensurePromiseRejected(promise) { + return promise.then( + function () { + ok(false, "Promise should be rejected"); + }, + function (err) { + is( + err.name, + "TypeError", + "Attempting to store a Response with a Vary:* header must fail" + ); + } + ); + } + return new Promise(function (resolve, reject) { + var cache; + caches.open(name).then(function (c) { + cache = c; + Promise.all([ + ensurePromiseRejected( + cache.add( + new Request(requestURL + "1", { headers: { WhatToVary: "*" } }) + ) + ), + ensurePromiseRejected( + cache.addAll([ + new Request(requestURL + "2", { headers: { WhatToVary: "*" } }), + requestURL + "3", + ]) + ), + ensurePromiseRejected( + fetch( + new Request(requestURL + "4", { headers: { WhatToVary: "*" } }) + ).then(function (response) { + return cache.put(requestURL + "4", response); + }) + ), + ensurePromiseRejected( + cache.add( + new Request(requestURL + "5", { + headers: { WhatToVary: "*,User-Agent" }, + }) + ) + ), + ensurePromiseRejected( + cache.addAll([ + new Request(requestURL + "6", { + headers: { WhatToVary: "*,User-Agent" }, + }), + requestURL + "7", + ]) + ), + ensurePromiseRejected( + fetch( + new Request(requestURL + "8", { + headers: { WhatToVary: "*,User-Agent" }, + }) + ).then(function (response) { + return cache.put(requestURL + "8", response); + }) + ), + ensurePromiseRejected( + cache.add( + new Request(requestURL + "9", { + headers: { WhatToVary: "User-Agent,*" }, + }) + ) + ), + ensurePromiseRejected( + cache.addAll([ + new Request(requestURL + "10", { + headers: { WhatToVary: "User-Agent,*" }, + }), + requestURL + "10", + ]) + ), + ensurePromiseRejected( + fetch( + new Request(requestURL + "11", { + headers: { WhatToVary: "User-Agent,*" }, + }) + ).then(function (response) { + return cache.put(requestURL + "11", response); + }) + ), + ]).then(reject, resolve); + }); + }); +} + +function testMatch() { + var test; + return setupTest({ WhatToVary: "Custom", Custom: "foo=bar" }) + .then(function (t) { + test = t; + // Ensure that searching with a different Custom header fails. + return test.cache.match( + new Request(requestURL, { headers: { Custom: "bar=baz" } }) + ); + }) + .then(function (r) { + is( + typeof r, + "undefined", + "Searching for a request with a non-matching Custom header should not succeed" + ); + // Ensure that searching with the same Custom header succeeds. + return test.cache.match( + new Request(requestURL, { headers: { Custom: "foo=bar" } }) + ); + }) + .then(function (r) { + return checkResponse(r, test.response, test.responseText); + }); +} + +function testInvalidHeaderName() { + var test; + return setupTest({ WhatToVary: "Foo/Bar, Custom-User-Agent" }) + .then(function (t) { + test = t; + // Ensure that searching with a different User-Agent header fails. + return test.cache.match( + new Request(requestURL, { headers: { "Custom-User-Agent": "MyUA" } }) + ); + }) + .then(function (r) { + is( + typeof r, + "undefined", + "Searching for a request with a non-matching Custom-User-Agent header should not succeed" + ); + // Ensure that searching with a different Custom-User-Agent header but with ignoreVary succeeds. + return test.cache.match( + new Request(requestURL, { headers: { "Custom-User-Agent": "MyUA" } }), + { ignoreVary: true } + ); + }) + .then(function (r) { + return checkResponse(r, test.response, test.responseText); + }) + .then(function () { + // Ensure that we do not mistakenly recognize the tokens in the invalid header name. + return test.cache.match( + new Request(requestURL, { headers: { Foo: "foobar" } }) + ); + }) + .then(function (r) { + return checkResponse(r, test.response, test.responseText); + }); +} + +function testMultipleHeaders() { + var test; + return setupTest({ WhatToVary: "Custom-Referer,\tCustom-Accept-Encoding" }) + .then(function (t) { + test = t; + // Ensure that searching with a different Referer header fails. + return test.cache.match( + new Request(requestURL, { + headers: { "Custom-Referer": "https://somesite.com/" }, + }) + ); + }) + .then(function (r) { + is( + typeof r, + "undefined", + "Searching for a request with a non-matching Custom-Referer header should not succeed" + ); + // Ensure that searching with a different Custom-Referer header but with ignoreVary succeeds. + return test.cache.match( + new Request(requestURL, { + headers: { "Custom-Referer": "https://somesite.com/" }, + }), + { ignoreVary: true } + ); + }) + .then(function (r) { + return checkResponse(r, test.response, test.responseText); + }) + .then(function () { + // Ensure that searching with a different Custom-Accept-Encoding header fails. + return test.cache.match( + new Request(requestURL, { + headers: { "Custom-Accept-Encoding": "myencoding" }, + }) + ); + }) + .then(function (r) { + is( + typeof r, + "undefined", + "Searching for a request with a non-matching Custom-Accept-Encoding header should not succeed" + ); + // Ensure that searching with a different Custom-Accept-Encoding header but with ignoreVary succeeds. + return test.cache.match( + new Request(requestURL, { + headers: { "Custom-Accept-Encoding": "myencoding" }, + }), + { ignoreVary: true } + ); + }) + .then(function (r) { + return checkResponse(r, test.response, test.responseText); + }) + .then(function () { + // Ensure that searching with an empty Custom-Referer header succeeds. + return test.cache.match( + new Request(requestURL, { headers: { "Custom-Referer": "" } }) + ); + }) + .then(function (r) { + return checkResponse(r, test.response, test.responseText); + }) + .then(function () { + // Ensure that searching with an empty Custom-Accept-Encoding header succeeds. + return test.cache.match( + new Request(requestURL, { headers: { "Custom-Accept-Encoding": "" } }) + ); + }) + .then(function (r) { + return checkResponse(r, test.response, test.responseText); + }) + .then(function () { + // Ensure that searching with an empty Custom-Referer header but with a different Custom-Accept-Encoding header fails. + return test.cache.match( + new Request(requestURL, { + headers: { + "Custom-Referer": "", + "Custom-Accept-Encoding": "myencoding", + }, + }) + ); + }) + .then(function (r) { + is( + typeof r, + "undefined", + "Searching for a request with a non-matching Custom-Accept-Encoding header should not succeed" + ); + // Ensure that searching with an empty Custom-Referer header but with a different Custom-Accept-Encoding header and ignoreVary succeeds. + return test.cache.match( + new Request(requestURL, { + headers: { + "Custom-Referer": "", + "Custom-Accept-Encoding": "myencoding", + }, + }), + { ignoreVary: true } + ); + }) + .then(function (r) { + return checkResponse(r, test.response, test.responseText); + }); +} + +function testMultipleCacheEntries() { + var test; + return setupTestMultipleEntries([ + { WhatToVary: "Accept-Language", "Accept-Language": "en-US" }, + { WhatToVary: "Accept-Language", "Accept-Language": "en-US, fa-IR" }, + ]) + .then(function (t) { + test = t; + return test.cache.matchAll(); + }) + .then(function (r) { + is(r.length, 2, "Two cache entries should be stored in the DB"); + // Ensure that searching without specifying an Accept-Language header fails. + return test.cache.matchAll(requestURL); + }) + .then(function (r) { + is( + r.length, + 0, + "Searching for a request without specifying an Accept-Language header should not succeed" + ); + // Ensure that searching without specifying an Accept-Language header but with ignoreVary succeeds. + return test.cache.matchAll(requestURL, { ignoreVary: true }); + }) + .then(function (r) { + return Promise.all([ + checkResponse(r[0], test.response[0], test.responseText[0]), + checkResponse(r[1], test.response[1], test.responseText[1]), + ]); + }) + .then(function () { + // Ensure that searching with Accept-Language: en-US succeeds. + return test.cache.matchAll( + new Request(requestURL, { headers: { "Accept-Language": "en-US" } }) + ); + }) + .then(function (r) { + is(r.length, 1, "One cache entry should be found"); + return checkResponse(r[0], test.response[0], test.responseText[0]); + }) + .then(function () { + // Ensure that searching with Accept-Language: en-US,fa-IR succeeds. + return test.cache.matchAll( + new Request(requestURL, { + headers: { "Accept-Language": "en-US, fa-IR" }, + }) + ); + }) + .then(function (r) { + is(r.length, 1, "One cache entry should be found"); + return checkResponse(r[0], test.response[1], test.responseText[1]); + }) + .then(function () { + // Ensure that searching with a valid Accept-Language header but with ignoreVary returns both entries. + return test.cache.matchAll( + new Request(requestURL, { headers: { "Accept-Language": "en-US" } }), + { ignoreVary: true } + ); + }) + .then(function (r) { + return Promise.all([ + checkResponse(r[0], test.response[0], test.responseText[0]), + checkResponse(r[1], test.response[1], test.responseText[1]), + ]); + }) + .then(function () { + // Ensure that searching with Accept-Language: fa-IR fails. + return test.cache.matchAll( + new Request(requestURL, { headers: { "Accept-Language": "fa-IR" } }) + ); + }) + .then(function (r) { + is( + r.length, + 0, + "Searching for a request with a different Accept-Language header should not succeed" + ); + // Ensure that searching with Accept-Language: fa-IR but with ignoreVary should succeed. + return test.cache.matchAll( + new Request(requestURL, { headers: { "Accept-Language": "fa-IR" } }), + { ignoreVary: true } + ); + }) + .then(function (r) { + is(r.length, 2, "Two cache entries should be found"); + return Promise.all([ + checkResponse(r[0], test.response[0], test.responseText[0]), + checkResponse(r[1], test.response[1], test.responseText[1]), + ]); + }); +} + +// Make sure to clean up after each test step. +function step(testPromise) { + return testPromise.then( + function () { + caches.delete(name); + }, + function () { + caches.delete(name); + } + ); +} + +step(testBasics()) + .then(function () { + return step(testBasicKeys()); + }) + .then(function () { + return step(testStar()); + }) + .then(function () { + return step(testMatch()); + }) + .then(function () { + return step(testInvalidHeaderName()); + }) + .then(function () { + return step(testMultipleHeaders()); + }) + .then(function () { + return step(testMultipleCacheEntries()); + }) + .then(function () { + testDone(); + }); diff --git a/dom/cache/test/mochitest/test_cache_orphaned_body.html b/dom/cache/test/mochitest/test_cache_orphaned_body.html new file mode 100644 index 0000000000..337fa075c1 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_orphaned_body.html @@ -0,0 +1,241 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Test Cache with QuotaManager Restart</title> + <script src="/tests/SimpleTest/SimpleTest.js"></script> + <script type="text/javascript" src="large_url_list.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> +</head> +<body> +<script class="testbody" type="text/javascript"> +function setupTestIframe() { + return new Promise(function(resolve) { + var iframe = document.createElement("iframe"); + iframe.src = "empty.html"; + iframe.onload = function() { + window.caches = iframe.contentWindow.caches; + resolve(); + }; + document.body.appendChild(iframe); + }); +} + +function clearStorage() { + return new Promise(function(resolve, reject) { + var qms = SpecialPowers.Services.qms; + var principal = SpecialPowers.wrap(document).nodePrincipal; + var request = qms.clearStoragesForPrincipal(principal); + var cb = SpecialPowers.wrapCallback(resolve); + request.callback = cb; + }); +} + +function storageUsage() { + return new Promise(function(resolve, reject) { + var qms = SpecialPowers.Services.qms; + var principal = SpecialPowers.wrap(document).nodePrincipal; + var cb = SpecialPowers.wrapCallback(function(request) { + var result = request.result; + resolve(result.usage); + }); + qms.getUsageForPrincipal(principal, cb); + }); +} + +function groupUsage() { + return new Promise(function(resolve, reject) { + navigator.storage.estimate().then(storageEstimation => { + resolve(storageEstimation.usage, 0); + }); + }); +} + +function workerGroupUsage() { + return new Promise(function(resolve, reject) { + function workerScript() { + navigator.storage.estimate().then(storageEstimation => { + postMessage(storageEstimation.usage); + }); + } + + let url = + URL.createObjectURL(new Blob(["(", workerScript.toString(), ")()"])); + + let worker = new Worker(url); + worker.onmessage = function(e) { + resolve(e.data, 0); + }; + }); +} + +function resetStorage() { + return new Promise(function(resolve, reject) { + var qms = SpecialPowers.Services.qms; + var request = qms.reset(); + var cb = SpecialPowers.wrapCallback(resolve); + request.callback = cb; + }); +} + +function gc() { + return new Promise(function(resolve, reject) { + SpecialPowers.exactGC(resolve); + }); +} + +SimpleTest.waitForExplicitFinish(); +SpecialPowers.pushPrefEnv({ + "set": [["dom.caches.enabled", true], + ["dom.caches.testing.enabled", true], + ["dom.quotaManager.testing", true], + ["dom.storageManager.enabled", true], + ["privacy.partition.always_partition_third_party_non_cookie_storage", false]], +}, async function() { + // Bug 1746646: Make mochitests work with TCP enabled (cookieBehavior = 5) + // Acquire storage access permission here so that the Cache API is avaialable + SpecialPowers.wrap(document).notifyUserGestureActivation(); + await SpecialPowers.addPermission("storageAccessAPI", true, window.location.href); + await SpecialPowers.wrap(document).requestStorageAccess(); + + var name = "orphanedBodyOwner"; + var cache = null; + var response = null; + var initialUsage = 0; + var fullUsage = 0; + var endUsage = 0; + var url = "test_cache_add.js"; + + // start from a fresh origin directory so other tests do not influence our + // results + setupTestIframe().then(function() { + return clearStorage(); + }).then(function() { + return storageUsage(); + }).then(function(usage) { + is(0, usage, "disk usage should be zero to start"); + }) + + // Initialize and populate an initial cache to get the base sqlite pages + // and directory structure allocated. + .then(function() { + return caches.open(name); + }).then(function(c) { + return c.add(url); + }).then(function() { + return gc(); + }).then(function() { + return caches.delete(name); + }).then(function(deleted) { + ok(deleted, "cache should be deleted"); + + // This is a bit superfluous, but its necessary to make sure the Cache is + // fully deleted before we proceed. The deletion actually takes place in + // two async steps. We don't want to resetStorage() until the second step + // has taken place. This extra Cache operation ensure that all the + // runnables have been flushed through the threads, etc. + return caches.has(name); + }) + + // Now measure initial disk usage + .then(function() { + return resetStorage(); + }).then(function() { + return storageUsage(); + }).then(function(usage) { + initialUsage = usage; + }) + + // Now re-populate the Cache object + .then(function() { + return caches.open(name); + }).then(function(c) { + cache = c; + return cache.add(url); + }) + + // Get a reference to the body we've stored in the Cache. + .then(function() { + return cache.match(url); + }).then(function(r) { + response = r; + return cache.delete(url); + }).then(function(result) { + ok(result, "Cache entry should be deleted"); + }) + + // Reset the quota dir while the cache entry is deleted, but still referenced + // from the DOM. This forces the body to be orphaned. + .then(function() { + return resetStorage(); + }).then(function() { + return storageUsage(); + }).then(function(usage) { + fullUsage = usage; + ok(fullUsage > initialUsage, "disk usage should have grown"); + }) + + // Test groupUsage() + .then(function() { + return resetStorage(); + }).then(function() { + return groupUsage(); + }).then(function(usage) { + fullUsage = usage; + ok(fullUsage > initialUsage, "disk group usage should have grown"); + }) + + // Test workerGroupUsage() + .then(function() { + return resetStorage(); + }).then(function() { + return workerGroupUsage(); + }).then(function(usage) { + fullUsage = usage; + ok(fullUsage > initialUsage, "disk group usage on worker should have grown"); + }) + + // Now perform a new Cache operation that will reopen the origin. This + // should clean up the orphaned body. + .then(function() { + return caches.match(url); + }).then(function(r) { + ok(!r, "response should not exist in storage"); + }) + + // Finally, verify orphaned data was cleaned up by re-checking the disk + // usage. Reset the storage first to ensure any WAL transaction files + // are flushed before measuring the usage. + .then(function() { + return resetStorage(); + }).then(function() { + return storageUsage(); + }).then(function(usage) { + endUsage = usage; + dump("### ### initial:" + initialUsage + ", full:" + fullUsage + + ", end:" + endUsage + "\n"); + ok(endUsage < fullUsage, "disk usage should have shrank"); + is(endUsage, initialUsage, "disk usage should return to original"); + }) + + // Verify that the stale, orphaned response cannot be put back into + // the cache. + .then(function() { + ok(!response.bodyUsed, "response body should not be considered used"); + return cache.put(url, response).then(function() { + ok(false, "Should not be able to store stale orphaned body."); + }).catch(function(e) { + is(e.name, "TypeError", "storing a stale orphaned body should throw TypeError"); + }); + }).then(function() { + ok(response.bodyUsed, "attempting to store response should mark body used"); + }) + + .then(function() { + SimpleTest.finish(); + }); +}); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_cache_orphaned_cache.html b/dom/cache/test/mochitest/test_cache_orphaned_cache.html new file mode 100644 index 0000000000..8f32e83b2a --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_orphaned_cache.html @@ -0,0 +1,171 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Test Cache with QuotaManager Restart</title> + <script src="/tests/SimpleTest/SimpleTest.js"></script> + <script type="text/javascript" src="large_url_list.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> +</head> +<body> +<script class="testbody" type="text/javascript"> +function setupTestIframe() { + return new Promise(function(resolve) { + var iframe = document.createElement("iframe"); + iframe.src = "empty.html"; + iframe.onload = function() { + window.caches = iframe.contentWindow.caches; + resolve(); + }; + document.body.appendChild(iframe); + }); +} + +function clearStorage() { + return new Promise(function(resolve, reject) { + var qms = SpecialPowers.Services.qms; + var principal = SpecialPowers.wrap(document).nodePrincipal; + var request = qms.clearStoragesForPrincipal(principal); + var cb = SpecialPowers.wrapCallback(resolve); + request.callback = cb; + }); +} + +function storageUsage() { + return new Promise(function(resolve, reject) { + var qms = SpecialPowers.Services.qms; + var principal = SpecialPowers.wrap(document).nodePrincipal; + var cb = SpecialPowers.wrapCallback(function(request) { + var result = request.result; + resolve(result.usage); + }); + qms.getUsageForPrincipal(principal, cb); + }); +} + +function resetStorage() { + return new Promise(function(resolve, reject) { + var qms = SpecialPowers.Services.qms; + var request = qms.reset(); + var cb = SpecialPowers.wrapCallback(resolve); + request.callback = cb; + }); +} + +function gc() { + return new Promise(function(resolve, reject) { + SpecialPowers.exactGC(resolve); + }); +} + +SimpleTest.waitForExplicitFinish(); +SpecialPowers.pushPrefEnv({ + "set": [["dom.caches.enabled", true], + ["dom.caches.testing.enabled", true], + ["dom.quotaManager.testing", true], + ["privacy.partition.always_partition_third_party_non_cookie_storage", false]], +}, async function() { + // Bug 1746646: Make mochitests work with TCP enabled (cookieBehavior = 5) + // Acquire storage access permission here so that the Cache API is avaialable + SpecialPowers.wrap(document).notifyUserGestureActivation(); + await SpecialPowers.addPermission("storageAccessAPI", true, window.location.href); + await SpecialPowers.wrap(document).requestStorageAccess(); + + var name = "toBeOrphaned"; + var cache = null; + var initialUsage = 0; + var fullUsage = 0; + var endUsage = 0; + var url = "test_cache_add.js"; + + // start from a fresh origin directory so other tests do not influence our + // results + setupTestIframe().then(function() { + return clearStorage(); + }).then(function() { + return storageUsage(); + }).then(function(usage) { + is(0, usage, "disk usage should be zero to start"); + }) + + // Initialize and populate an initial cache to get the base sqlite pages + // and directory structure allocated. + .then(function() { + return caches.open(name); + }).then(function(c) { + return c.add(url); + }).then(function() { + return gc(); + }).then(function() { + return caches.delete(name); + }).then(function(deleted) { + ok(deleted, "cache should be deleted"); + + // This is a bit superfluous, but its necessary to make sure the Cache is + // fully deleted before we proceed. The deletion actually takes place in + // two async steps. We don't want to resetStorage() until the second step + // has taken place. This extra Cache operation ensure that all the + // runnables have been flushed through the threads, etc. + return caches.has(name); + }) + + // Now measure initial disk usage + .then(function() { + return resetStorage(); + }).then(function() { + return storageUsage(); + }).then(function(usage) { + initialUsage = usage; + }) + + // Now re-populate the Cache object + .then(function() { + return caches.open(name); + }).then(function(c) { + cache = c; + return cache.add(url); + }).then(function() { + return caches.delete(name); + }).then(function(deleted) { + ok(deleted, "cache should be deleted"); + }) + + // Reset the quota dir while the cache is deleted, but still referenced + // from the DOM. This forces it to be orphaned. + .then(function() { + return resetStorage(); + }).then(function() { + return storageUsage(); + }).then(function(usage) { + fullUsage = usage; + ok(fullUsage > initialUsage, "disk usage should have grown"); + }) + + // Now perform a new Cache operation that will reopen the origin. This + // should clean up the orphaned Cache data. + .then(function() { + return caches.has(name); + }).then(function(result) { + ok(!result, "cache should not exist in storage"); + }) + + // Finally, verify orphaned data was cleaned up by re-checking the disk + // usage. Reset the storage first to ensure any WAL transaction files + // are flushed before measuring the usage. + .then(function() { + return resetStorage(); + }).then(function() { + return storageUsage(); + }).then(function(usage) { + endUsage = usage; + dump("### ### initial:" + initialUsage + ", full:" + fullUsage + + ", end:" + endUsage + "\n"); + ok(endUsage < fullUsage, "disk usage should have shrank"); + is(endUsage, initialUsage, "disk usage should return to original"); + SimpleTest.finish(); + }); +}); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_cache_overwrite.html b/dom/cache/test/mochitest/test_cache_overwrite.html new file mode 100644 index 0000000000..7d2b9c797a --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_overwrite.html @@ -0,0 +1,20 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Test what happens when you overwrite a cache entry</title> + <script src="/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> + <script type="text/javascript" src="driver.js"></script> +</head> +<body> +<iframe id="frame"></iframe> +<script class="testbody" type="text/javascript"> + runTests("test_cache_overwrite.js") + .then(function() { + SimpleTest.finish(); + }); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_cache_overwrite.js b/dom/cache/test/mochitest/test_cache_overwrite.js new file mode 100644 index 0000000000..dd6b3c5848 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_overwrite.js @@ -0,0 +1,71 @@ +/* global context testDone:true */ + +var requestURL = + "//mochi.test:8888/tests/dom/cache/test/mochitest/mirror.sjs?" + context; +var response; +var c; +var responseText; +var name = "match-mirror" + context; + +function checkResponse(r) { + ok(r !== response, "The objects should not be the same"); + is( + r.url, + response.url.replace("#fragment", ""), + "The URLs should be the same" + ); + is(r.status, response.status, "The status codes should be the same"); + is(r.type, response.type, "The response types should be the same"); + is(r.ok, response.ok, "Both responses should have succeeded"); + is( + r.statusText, + response.statusText, + "Both responses should have the same status text" + ); + is( + r.headers.get("Mirrored"), + response.headers.get("Mirrored"), + "Both responses should have the same Mirrored header" + ); + return r.text().then(function (text) { + is(text, responseText, "The response body should be correct"); + }); +} + +fetch(new Request(requestURL, { headers: { Mirror: "bar" } })) + .then(function (r) { + is( + r.headers.get("Mirrored"), + "bar", + "The server should give back the correct header" + ); + response = r; + return response.text(); + }) + .then(function (text) { + responseText = text; + return caches.open(name); + }) + .then(function (cache) { + c = cache; + return c.add(new Request(requestURL, { headers: { Mirror: "foo" } })); + }) + .then(function () { + // Overwrite the request, to replace the entry stored in response_headers + // with a different value. + return c.add(new Request(requestURL, { headers: { Mirror: "bar" } })); + }) + .then(function () { + return c.matchAll(); + }) + .then(function (r) { + is(r.length, 1, "Only one request should be in the cache"); + return checkResponse(r[0]); + }) + .then(function () { + return caches.delete(name); + }) + .then(function (deleted) { + ok(deleted, "The cache should be deleted successfully"); + testDone(); + }); diff --git a/dom/cache/test/mochitest/test_cache_padding.html b/dom/cache/test/mochitest/test_cache_padding.html new file mode 100644 index 0000000000..b05dadb304 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_padding.html @@ -0,0 +1,203 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Test Cache generate padding size for opaque repsonse</title> + <script src="/tests/SimpleTest/SimpleTest.js"></script> + <script type="text/javascript" src="large_url_list.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> +</head> +<body> +<script class="testbody" type="text/javascript"> +function setupTestIframe() { + return new Promise(function(resolve) { + var iframe = document.createElement("iframe"); + iframe.src = "empty.html"; + iframe.onload = function() { + window.caches = iframe.contentWindow.caches; + resolve(); + }; + document.body.appendChild(iframe); + }); +} + +function clearStorage() { + return new Promise(function(resolve, reject) { + var qms = SpecialPowers.Services.qms; + var principal = SpecialPowers.wrap(document).nodePrincipal; + var request = qms.clearStoragesForPrincipal(principal); + var cb = SpecialPowers.wrapCallback(resolve); + request.callback = cb; + }); +} + +function resetStorage() { + return new Promise(function(resolve, reject) { + var qms = SpecialPowers.Services.qms; + var request = qms.reset(); + var cb = SpecialPowers.wrapCallback(resolve); + request.callback = cb; + }); +} + +function getStorageUsage(fromMemory) { + return new Promise(function(resolve, reject) { + var qms = SpecialPowers.Services.qms; + var principal = SpecialPowers.wrap(document).nodePrincipal; + var cb = SpecialPowers.wrapCallback(function(request) { + var result = request.result; + resolve(result.usage); + }); + + // Actually, the flag is used to distingulish getting group usage and origin + // usage, but we utilize this to get usage from in-memory and the disk. + // Default value for "fromMemory" is false. + qms.getUsageForPrincipal(principal, cb, !!fromMemory); + }); +} + +async function verifyUsage() { + // Although it returns group usage when passing true, it calculate the usage + // from tracking usage object (in-memory object) in QuotaManager. + let memoryUsage = await getStorageUsage(/* fromMemory */ true); + // This will returns the origin usage by re-calculating usage from directory. + let diskUsage = await getStorageUsage(/* fromMemory */ false); + + is(memoryUsage, diskUsage, + "In-memory usage and disk usage should be the same."); + return memoryUsage; +} + +async function waitForIOToComplete(cache, request) { + info("Wait for IO complete."); + // The following lines ensure we've deleted orphaned body. + // First, wait for cache operation delete the orphaned body. + await cache.match(request); + + // Finally, wait for -wal file finish its job. + return resetStorage(); +} + +function fetchOpaqueResponse(url) { + return fetch(url, { mode: "no-cors" }); +} + +SimpleTest.waitForExplicitFinish(); +SpecialPowers.pushPrefEnv({ + "set": [["dom.caches.enabled", true], + ["dom.caches.testing.enabled", true], + ["dom.quotaManager.testing", true], + ["privacy.partition.always_partition_third_party_non_cookie_storage", false]], +}, async function() { + // This test is mainly to verify we only generate different padding size for + // the opaque response which is comming from netwrok. + // Besides, this test utilizes verifyUsage() to ensure Cache Acions does + // update thier usage/padding size to the QM, does record padding size to + // the directory padding file and does do above two things synchronously. + // So that, opaque response's size is bigger than the normal response's size + // and we always have the same usage bewteen from in-memory and from + // the file-system. + // Note: For the cloned and cached opaque response, the padding size shouldn't + // be changed. Thus, it makes the attacker harder to get the padding size. + + // Bug 1746646: Make mochitests work with TCP enabled (cookieBehavior = 5) + // Acquire storage access permission here so that the Cache API is avaialable + SpecialPowers.wrap(document).notifyUserGestureActivation(); + await SpecialPowers.addPermission("storageAccessAPI", true, window.location.href); + await SpecialPowers.wrap(document).requestStorageAccess(); + + const name = "cachePadding"; + const other_name = "cachePaddingOther"; + const cors_base = "https://example.com/tests/dom/cache/test/mochitest/"; + const url = "test_cache_add.js"; + + await setupTestIframe(); + + info("Stage 1: Clean storage."); + await clearStorage(); + + let cache = await caches.open(name); + await waitForIOToComplete(cache, url); + let usage1 = await verifyUsage(); + + info("Stage 2: Verify opaque responses have padding."); + cache = await caches.open(name); + await cache.add(url); + await waitForIOToComplete(cache, url); + let usage2 = await verifyUsage(); + let sizeForNormalResponse = usage2 - usage1; + + let opaqueResponse = await fetchOpaqueResponse(cors_base + url); + cache = await caches.open(name); + await cache.put(cors_base + url, opaqueResponse.clone()); + await waitForIOToComplete(cache, url); + let usage3 = await verifyUsage(); + let sizeForOpaqueResponse = usage3 - usage2; + ok(sizeForOpaqueResponse > sizeForNormalResponse, + "The opaque response should have larger size than the normal response."); + + info("Stage 3: Verify the cloned response has the same size."); + cache = await caches.open(name); + await cache.put(cors_base + url, opaqueResponse.clone()); + await waitForIOToComplete(cache, url); + let usage4 = await verifyUsage(); + // Since we put the same request and response again, the size should be the + // same (DOM Cache removes the previous cached request and response) + ok(usage4 == usage3, + "We won't generate different padding for cloned response"); + + info("Stage 4: Verify the cached response has the same size."); + cache = await caches.open(name); + opaqueResponse = await cache.match(cors_base + url); + ok(opaqueResponse); + + await cache.put(cors_base + url, opaqueResponse); + await waitForIOToComplete(cache, url); + let usage5 = await verifyUsage(); + ok(usage5 == usage3, + "We won't generate different padding for cached response"); + + info("Stage 5: Verify padding size may changes in different fetch()s."); + let paddingSizeChange = false; + // Since we randomly generate padding size and rounding the overall size up, + // we will probably have the same size. So, fetch it multiple times. + for (let i = 0; i < 10; i++) { + opaqueResponse = await fetchOpaqueResponse(cors_base + url); + cache = await caches.open(name); + await cache.put(cors_base + url, opaqueResponse); + await waitForIOToComplete(cache, url); + let usage6 = await verifyUsage(); + if (usage6 != usage5) { + paddingSizeChange = true; + break; + } + } + ok(paddingSizeChange, + "We should generate different padding size for fetching response"); + + info("Stage 6: Verify the padding is removed once on caches.delete() and " + + "cache.delete()."); + // Add an opauqe response on other cache storage and then delete that storage. + cache = await caches.open(other_name); + opaqueResponse = await fetchOpaqueResponse(cors_base + url); + await cache.put(cors_base + url, opaqueResponse); + await caches.delete(other_name); + await caches.has(other_name); + // Force remove orphaned cached in the next action + await resetStorage(); + + // Delete the opauqe repsonse on current cache storage. + cache = await caches.open(name); + await cache.delete(cors_base + url); + await waitForIOToComplete(cache, url); + let usage7 = await verifyUsage(); + ok(usage7 == usage2, + "The opaque response should be removed by caches.delete() and " + + "cache.delete()"); + + await SimpleTest.finish(); +}); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_cache_put.html b/dom/cache/test/mochitest/test_cache_put.html new file mode 100644 index 0000000000..19b67f6c55 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_put.html @@ -0,0 +1,20 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Validate Interfaces Exposed to Workers</title> + <script src="/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> + <script type="text/javascript" src="driver.js"></script> +</head> +<body> +<iframe id="frame"></iframe> +<script class="testbody" type="text/javascript"> + runTests("test_cache_put.js") + .then(function() { + SimpleTest.finish(); + }); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_cache_put.js b/dom/cache/test/mochitest/test_cache_put.js new file mode 100644 index 0000000000..a9229a46da --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_put.js @@ -0,0 +1,72 @@ +/* global context testDone:true */ + +var url = "test_cache.js"; +var cache; +var fetchResponse; +Promise.all([fetch(url), caches.open("putter" + context)]) + .then(function (results) { + fetchResponse = results[0]; + cache = results[1]; + return cache.put(url, fetchResponse.clone()); + }) + .then(function (result) { + is(undefined, result, "Successful put() should resolve undefined"); + return cache.match(url); + }) + .then(function (response) { + ok(response, "match() should find resppnse that was previously put()"); + ok( + response.url.endsWith(url), + "matched response should match original url" + ); + return Promise.all([fetchResponse.text(), response.text()]); + }) + .then(function (results) { + // suppress large assert spam unless it's relevent + if (results[0] !== results[1]) { + is(results[0], results[1], "stored response body should match original"); + } + + // Now, try to overwrite the request with a different response object. + return cache.put(url, new Response("overwritten")); + }) + .then(function () { + return cache.matchAll(url); + }) + .then(function (result) { + is(result.length, 1, "Only one entry should exist"); + return result[0].text(); + }) + .then(function (body) { + is( + body, + "overwritten", + "The cache entry should be successfully overwritten" + ); + + // Now, try to write a URL with a fragment + return cache.put(url + "#fragment", new Response("more overwritten")); + }) + .then(function () { + return cache.matchAll(url + "#differentFragment"); + }) + .then(function (result) { + is(result.length, 1, "Only one entry should exist"); + return result[0].text(); + }) + .then(function (body) { + is( + body, + "more overwritten", + "The cache entry should be successfully overwritten" + ); + + // TODO: Verify that trying to store a response with an error raises a TypeError + // when bug 1147178 is fixed. + + return caches.delete("putter" + context); + }) + .then(function (deleted) { + ok(deleted, "The cache should be deleted successfully"); + testDone(); + }); diff --git a/dom/cache/test/mochitest/test_cache_put_reorder.html b/dom/cache/test/mochitest/test_cache_put_reorder.html new file mode 100644 index 0000000000..f9b360f137 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_put_reorder.html @@ -0,0 +1,20 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Ensure that using Cache.put to overwrite an entry will change its insertion order</title> + <script src="/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> + <script type="text/javascript" src="driver.js"></script> +</head> +<body> +<iframe id="frame"></iframe> +<script class="testbody" type="text/javascript"> + runTests("test_cache_put_reorder.js") + .then(function() { + SimpleTest.finish(); + }); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_cache_put_reorder.js b/dom/cache/test/mochitest/test_cache_put_reorder.js new file mode 100644 index 0000000000..c5875c2c9d --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_put_reorder.js @@ -0,0 +1,47 @@ +/* global context testDone:true */ + +var name = "putreorder" + context; +var c; + +var reqs = [ + "//mochi.test:8888/?foo" + context, + "//mochi.test:8888/?bar" + context, + "//mochi.test:8888/?baz" + context, +]; + +caches + .open(name) + .then(function (cache) { + c = cache; + return c.addAll(reqs); + }) + .then(function () { + return c.put(reqs[1], new Response("overwritten")); + }) + .then(function () { + return c.keys(); + }) + .then(function (keys) { + is(keys.length, 3, "Correct number of entries expected"); + ok(keys[0].url.includes(reqs[0]), "The first entry should be untouched"); + ok( + keys[2].url.includes(reqs[1]), + "The second entry should be moved to the end" + ); + ok( + keys[1].url.includes(reqs[2]), + "The third entry should now be the second one" + ); + return c.match(reqs[1]); + }) + .then(function (r) { + return r.text(); + }) + .then(function (body) { + is(body, "overwritten", "The body should be overwritten"); + return caches.delete(name); + }) + .then(function (deleted) { + ok(deleted, "The cache should be deleted successfully"); + testDone(); + }); diff --git a/dom/cache/test/mochitest/test_cache_redirect.html b/dom/cache/test/mochitest/test_cache_redirect.html new file mode 100644 index 0000000000..50d040141e --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_redirect.html @@ -0,0 +1,20 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Validate Cache storage of redirect responses</title> + <script src="/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> + <script type="text/javascript" src="driver.js"></script> +</head> +<body> +<iframe id="frame"></iframe> +<script class="testbody" type="text/javascript"> + runTests("test_cache_redirect.js") + .then(function() { + SimpleTest.finish(); + }); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_cache_redirect.js b/dom/cache/test/mochitest/test_cache_redirect.js new file mode 100644 index 0000000000..7d62970b7b --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_redirect.js @@ -0,0 +1,20 @@ +/* global context testDone:true */ + +let cache; +let url = "foo.html"; +let redirectURL = "https://example.com/foo-bar.html"; +caches + .open("redirect-" + context) + .then(c => { + cache = c; + var response = Response.redirect(redirectURL); + is(response.headers.get("Location"), redirectURL); + return cache.put(url, response); + }) + .then(_ => { + return cache.match(url); + }) + .then(response => { + is(response.headers.get("Location"), redirectURL); + testDone(); + }); diff --git a/dom/cache/test/mochitest/test_cache_requestCache.html b/dom/cache/test/mochitest/test_cache_requestCache.html new file mode 100644 index 0000000000..8f991ec2d3 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_requestCache.html @@ -0,0 +1,20 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Validate the Cache.keys() method</title> + <script src="/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> + <script type="text/javascript" src="driver.js"></script> +</head> +<body> +<iframe id="frame"></iframe> +<script class="testbody" type="text/javascript"> + runTests("test_cache_requestCache.js") + .then(function() { + SimpleTest.finish(); + }); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_cache_requestCache.js b/dom/cache/test/mochitest/test_cache_requestCache.js new file mode 100644 index 0000000000..a6ae2a3418 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_requestCache.js @@ -0,0 +1,35 @@ +/* global context testDone:true */ + +var name = "requestCache" + context; +var c; + +var reqWithoutCache = new Request("//mochi.test:8888/?noCache" + context); +var reqWithCache = new Request("//mochi.test:8888/?withCache" + context, { + cache: "force-cache", +}); + +// Sanity check +is(reqWithoutCache.cache, "default", "Correct default value"); +is(reqWithCache.cache, "force-cache", "Correct value set by the ctor"); + +caches + .open(name) + .then(function (cache) { + c = cache; + return c.addAll([reqWithoutCache, reqWithCache]); + }) + .then(function () { + return c.keys(); + }) + .then(function (keys) { + is(keys.length, 2, "Correct number of requests"); + is(keys[0].url, reqWithoutCache.url, "Correct URL"); + is(keys[0].cache, reqWithoutCache.cache, "Correct cache attribute"); + is(keys[1].url, reqWithCache.url, "Correct URL"); + is(keys[1].cache, reqWithCache.cache, "Correct cache attribute"); + return caches.delete(name); + }) + .then(function (deleted) { + ok(deleted, "The cache should be successfully deleted"); + testDone(); + }); diff --git a/dom/cache/test/mochitest/test_cache_restart.html b/dom/cache/test/mochitest/test_cache_restart.html new file mode 100644 index 0000000000..b122ba5e3c --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_restart.html @@ -0,0 +1,70 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Test Cache with QuotaManager Restart</title> + <script src="/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> +</head> +<body> +<script class="testbody" type="text/javascript"> +function setupTestIframe() { + return new Promise(function(resolve) { + var iframe = document.createElement("iframe"); + iframe.src = "empty.html"; + iframe.onload = function() { + window.caches = iframe.contentWindow.caches; + resolve(); + }; + document.body.appendChild(iframe); + }); +} + +function resetStorage() { + return new Promise(function(resolve, reject) { + var qms = SpecialPowers.Services.qms; + var request = qms.reset(); + var cb = SpecialPowers.wrapCallback(resolve); + request.callback = cb; + }); +} + +SimpleTest.waitForExplicitFinish(); +SpecialPowers.pushPrefEnv({ + "set": [["dom.caches.enabled", true], + ["dom.caches.testing.enabled", true], + ["dom.quotaManager.testing", true]], +}, function() { + var name = "foo"; + var url = "./test_cache_add.js"; + var cache; + setupTestIframe().then(function() { + return caches.open(name); + }).then(function(c) { + cache = c; + return cache.add(url); + }).then(function() { + return resetStorage(); + }).then(function() { + return cache.match(url).then(function(resp) { + ok(false, "old cache reference should not work after reset"); + }).catch(function(err) { + ok(true, "old cache reference should not work after reset"); + }); + }).then(function() { + return caches.open(name); + }).then(function(c) { + cache = c; + return cache.match(url); + }).then(function(resp) { + ok(!!resp, "cache should work after QM reset"); + return caches.delete(name); + }).then(function(success) { + ok(success, "cache should be deleted"); + SimpleTest.finish(); + }); +}); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_cache_shrink.html b/dom/cache/test/mochitest/test_cache_shrink.html new file mode 100644 index 0000000000..387d180fef --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_shrink.html @@ -0,0 +1,139 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Test Cache with QuotaManager Restart</title> + <script src="/tests/SimpleTest/SimpleTest.js"></script> + <script type="text/javascript" src="large_url_list.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> +</head> +<body> +<script class="testbody" type="text/javascript"> +function setupTestIframe() { + return new Promise(function(resolve) { + var iframe = document.createElement("iframe"); + iframe.src = "empty.html"; + iframe.onload = function() { + window.caches = iframe.contentWindow.caches; + resolve(); + }; + document.body.appendChild(iframe); + }); +} + +function clearStorage() { + return new Promise(function(resolve, reject) { + var qms = SpecialPowers.Services.qms; + var principal = SpecialPowers.wrap(document).nodePrincipal; + var request = qms.clearStoragesForPrincipal(principal); + var cb = SpecialPowers.wrapCallback(resolve); + request.callback = cb; + }); +} + +function storageUsage() { + return new Promise(function(resolve, reject) { + var qms = SpecialPowers.Services.qms; + var principal = SpecialPowers.wrap(document).nodePrincipal; + var cb = SpecialPowers.wrapCallback(function(request) { + var result = request.result; + resolve(result.usage); + }); + qms.getUsageForPrincipal(principal, cb); + }); +} + +function resetStorage() { + return new Promise(function(resolve, reject) { + var qms = SpecialPowers.Services.qms; + var request = qms.reset(); + var cb = SpecialPowers.wrapCallback(resolve); + request.callback = cb; + }); +} + +function gc() { + return new Promise(function(resolve, reject) { + SpecialPowers.exactGC(resolve); + }); +} + +SimpleTest.waitForExplicitFinish(); +SpecialPowers.pushPrefEnv({ + "set": [["dom.caches.enabled", true], + ["dom.caches.testing.enabled", true], + ["dom.quotaManager.testing", true], + ["privacy.partition.always_partition_third_party_non_cookie_storage", false]], +}, async function() { + // Bug 1746646: Make mochitests work with TCP enabled (cookieBehavior = 5) + // Acquire storage access permission here so that the Cache API is avaialable + SpecialPowers.wrap(document).notifyUserGestureActivation(); + await SpecialPowers.addPermission("storageAccessAPI", true, window.location.href); + await SpecialPowers.wrap(document).requestStorageAccess(); + + var name = "foo"; + var cache = null; + var initialUsage = 0; + var fullUsage = 0; + var endUsage = 0; + // start from a fresh origin directory so other tests do not influence our + // results + setupTestIframe().then(function() { + return clearStorage(); + }).then(function() { + return storageUsage(); + }).then(function(usage) { + is(0, usage, "disk usage should be zero to start"); + return caches.open(name); + }).then(function(c) { + cache = c; + return storageUsage(); + }).then(function(usage) { + initialUsage = usage; + return Promise.all(largeUrlList.map(function(url) { + return cache.put(new Request(url), new Response()); + })); + }).then(function() { + return cache.keys(); + }).then(function(keyList) { + is(keyList.length, largeUrlList.length, "Large URL list is stored in cache"); + cache = null; + // Ensure the Cache DOM object is gone before proceeding. If its alive + // it will keep the related entries on-disk as well. + return gc(); + }).then(function() { + // reset the quota manager storage to ensure the DB connection is flushed + return resetStorage(); + }).then(function() { + return storageUsage(); + }).then(function(usage) { + fullUsage = usage; + ok(fullUsage > initialUsage, "disk usage should have grown"); + return caches.delete(name); + }).then(function(result) { + ok(result, "cache should be deleted"); + // This is a bit superfluous, but its necessary to make sure the Cache is + // fully deleted before we proceed. The deletion actually takes place in + // two async steps. We don't want to resetStorage() until the second step + // has taken place. This extra Cache operation ensure that all the + // runnables have been flushed through the threads, etc. + return caches.has(name); + }).then(function(result) { + ok(!result, "cache should not exist in storage"); + // reset the quota manager storage to ensure the DB connection is flushed + return resetStorage(); + }).then(function() { + return storageUsage(); + }).then(function(usage) { + endUsage = usage; + dump("### ### initial:" + initialUsage + ", full:" + fullUsage + + ", end:" + endUsage + "\n"); + ok(endUsage < (fullUsage / 2), "disk usage should have shrank significantly"); + ok(endUsage > initialUsage, "disk usage should not shrink back to orig size"); + SimpleTest.finish(); + }); +}); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_cache_tons_of_fd.html b/dom/cache/test/mochitest/test_cache_tons_of_fd.html new file mode 100644 index 0000000000..59d24769f6 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_tons_of_fd.html @@ -0,0 +1,118 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Test cache to create tons of fds</title> + <script src="/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> + <script type="text/javascript" src="driver.js"></script> +</head> +<body> +<script class="testbody" type="text/javascript"> + function setupTestIframe() { + return new Promise(function(resolve) { + var iframe = document.createElement("iframe"); + iframe.src = "empty.html"; + iframe.onload = function() { + window.caches = iframe.contentWindow.caches; + resolve(); + }; + document.body.appendChild(iframe); + }); +} + +function clearStorage() { + return new Promise(function(resolve, reject) { + var qms = SpecialPowers.Services.qms; + var principal = SpecialPowers.wrap(document).nodePrincipal; + var request = qms.clearStoragesForPrincipal(principal); + var cb = SpecialPowers.wrapCallback(resolve); + request.callback = cb; + }); +} + +async function testCreateTonsOfFD() { + const number_of_fd = 5120; + const name = "cacheTonsOfFD"; + const url = "foo.com"; + const body = "This is a body"; + + info("Stage A: Cached a Request/Response pairs"); + let cache = await caches.open(name); + let request = new Request(url); + let response = new Response(body); + await cache.put(request, response); + + info("Stage B: Read the cached response mutliple times"); + let promise_array = []; + for (let i = 0; i < number_of_fd; ++i) { + let promise = cache.match(request); + promise_array.push(promise); + } + let cached_response_array = []; + try { + cached_response_array = await Promise.all(promise_array); + } catch (e) { + throw new Error("Fail to open tons of files with error: " + e); + } + + if (cached_response_array.length != number_of_fd) { + throw new Error("Fail to cache.match the cached responses"); + } + + info("Stage C: Consume the cached body"); + for (let i = 0; i < number_of_fd; ++i) { + if (!cached_response_array[i]) { + // Reduce the checking message. + throw new Error("The cached response doesn't exist"); + } + + let bodyText = ""; + try { + bodyText = await cached_response_array[i].text(); + } catch (e) { + throw new Error("Fail to consume the cached response's body with error: " + e); + } + + if (bodyText != body) { + // Reduce the checking message. + throw new Error("The cached body doeen't be the same as original one"); + } + } + + ok(true, "Doesn't crash or timeout"); + return Promise.resolve(); +} + +SimpleTest.waitForExplicitFinish(); +SpecialPowers.pushPrefEnv({ + "set": [["dom.caches.enabled", true], + ["dom.caches.testing.enabled", true], + ["dom.quotaManager.testing", true], + ["privacy.partition.always_partition_third_party_non_cookie_storage", false]], +}, async function() { + // Bug 1746646: Make mochitests work with TCP enabled (cookieBehavior = 5) + // Acquire storage access permission here so that the Cache API is avaialable + SpecialPowers.wrap(document).notifyUserGestureActivation(); + await SpecialPowers.addPermission("storageAccessAPI", true, window.location.href); + await SpecialPowers.wrap(document).requestStorageAccess(); + + await setupTestIframe(); + + info("Stage 1: Clean storage."); + await clearStorage(); + + info("Stage 2: Verify open lots of files at the same time doesn't crash " + + "the browser"); + try { + await testCreateTonsOfFD(); + } catch (e) { + ok(false, e); + } + + await SimpleTest.finish(); +}); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_cache_untrusted.html b/dom/cache/test/mochitest/test_cache_untrusted.html new file mode 100644 index 0000000000..e835c3076b --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_untrusted.html @@ -0,0 +1,32 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Test Cache in untrusted context</title> + <script src="/tests/SimpleTest/SimpleTest.js"></script> + <script type="text/javascript" src="large_url_list.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> +</head> +<body> +<script class="testbody" type="text/javascript"> +function setupTestIframe() { + return new Promise(function(resolve) { + var iframe = document.createElement("iframe"); + iframe.src = "empty.html"; + iframe.onload = function() { + window.caches = iframe.contentWindow.caches; + resolve(); + }; + document.body.appendChild(iframe); + }); +} + +SimpleTest.waitForExplicitFinish(); +setupTestIframe().then(function() { + is(typeof window.caches, "undefined", "caches should not be usable in untrusted http origin"); + SimpleTest.finish(); +}); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_cache_updateUsage.html b/dom/cache/test/mochitest/test_cache_updateUsage.html new file mode 100644 index 0000000000..5abc080f14 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_updateUsage.html @@ -0,0 +1,189 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Test Cache update its usage to QuotaManager</title> + <script src="/tests/SimpleTest/SimpleTest.js"></script> + <script type="text/javascript" src="large_url_list.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> +</head> +<body> +<script class="testbody" type="text/javascript"> +function setupTestIframe() { + return new Promise(function(resolve) { + var iframe = document.createElement("iframe"); + iframe.src = "empty.html"; + iframe.onload = function() { + window.caches = iframe.contentWindow.caches; + resolve(); + }; + document.body.appendChild(iframe); + }); +} + +function clearStorage() { + return new Promise(function(resolve, reject) { + var qms = SpecialPowers.Services.qms; + var principal = SpecialPowers.wrap(document).nodePrincipal; + var request = qms.clearStoragesForPrincipal(principal); + var cb = SpecialPowers.wrapCallback(resolve); + request.callback = cb; + }); +} + +function resetStorage() { + return new Promise(function(resolve, reject) { + var qms = SpecialPowers.Services.qms; + var request = qms.reset(); + var cb = SpecialPowers.wrapCallback(resolve); + request.callback = cb; + }); +} + +function getStorageUsage(fromMemory) { + return new Promise(function(resolve, reject) { + var qms = SpecialPowers.Services.qms; + var principal = SpecialPowers.wrap(document).nodePrincipal; + var cb = SpecialPowers.wrapCallback(function(request) { + var result = request.result; + resolve(result.usage); + }); + + // Actually, the flag is used to distingulish getting group usage and origin + // usage, but we utilize this to get usage from in-memory and the disk. + // Default value for "fromMemory" is false. + qms.getUsageForPrincipal(principal, cb, !!fromMemory); + }); +} + +async function verifyUsage() { + // Although it returns group usage when passing true, it calculate the usage + // from tracking usage object (in-memory object) in QuotaManager. + let memoryUsage = await getStorageUsage(/* fromMemory */ true); + // This will returns the origin usage by re-calculating usage from directory. + let diskUsage = await getStorageUsage(/* fromMemory */ false); + + is(memoryUsage, diskUsage, + "In-memory usage and disk usage should be the same."); + return memoryUsage; +} + +async function waitForIOToComplete(cache, request) { + info("Wait for IO complete."); + // The following lines ensure we've deleted orphaned body. + // First, wait for cache operation delete the orphaned body. + await cache.match(request); + + // Finally, wait for -wal file finish its job. + return resetStorage(); +} + +SimpleTest.waitForExplicitFinish(); +SpecialPowers.pushPrefEnv({ + "set": [["dom.caches.enabled", true], + ["dom.caches.testing.enabled", true], + ["dom.quotaManager.testing", true], + ["privacy.partition.always_partition_third_party_non_cookie_storage", false]], +}, async function() { + // Bug 1746646: Make mochitests work with TCP enabled (cookieBehavior = 5) + // Acquire storage access permission here so that the Cache API is avaialable + SpecialPowers.wrap(document).notifyUserGestureActivation(); + await SpecialPowers.addPermission("storageAccessAPI", true, window.location.href); + await SpecialPowers.wrap(document).requestStorageAccess(); + + const name = "cacheUpdateUsage"; + const url = "test_cache_add.js"; + const other_url = "test_cache_put.js"; + + // This test mainly ensure DOM Cache updates its usage to QuotaManager when + // executing an operation which creates/deletes files. To do this, we verify + // usage by calling getUsageFromPrincipal twice with different flag(aGroup). + // The reason is we get group usage by collecting in-memory data, and getting + // origin usage by collecting storage usage from files. + + await setupTestIframe(); + + info("Stage 1: Clean storage."); + await clearStorage(); + await verifyUsage(); + + info("Stage 2: Verify CacheStorage."); + info("Stage 2.1: Verify caches.open."); + await caches.open(name); + await verifyUsage(); + + info("Stage 2.2: Verify caches.delete."); + var deleted = await caches.delete(name); + ok(deleted, "Cache storage should be deleted"); + // Reference from test_cache_orphanced_body.html. It ensures that all + // the runnables have been flushed through the threads. + await caches.has(name); + await resetStorage(); + await verifyUsage(); + + info("Stage 3: Verify Cache."); + let cache = await caches.open(name); + info("Stage 3.1: Verify cache.addAll."); + await cache.addAll([url, other_url]); + await verifyUsage(); + info("Stage 3.1.1: Delete all cached requests."); + await cache.delete(url); + await cache.delete(other_url); + await waitForIOToComplete(cache, other_url); + let emptyUsage1 = await verifyUsage(); + + info("Stage 3.2: Verify cache.add."); + cache = await caches.open(name); + await cache.add(url); + await verifyUsage(); + info("Stage 3.2.1: Delete cache."); + await cache.delete(url); + await waitForIOToComplete(cache, url); + let emptyUsage2 = await verifyUsage(); + + info("Stage 3.3: Verify cache.put."); + cache = await caches.open(name); + let response = await fetch(other_url); + await cache.put(other_url, response); + await verifyUsage(); + info("Stage 3.3.1: Delete cache."); + await cache.delete(other_url); + await waitForIOToComplete(cache, other_url); + let emptyUsage3 = await verifyUsage(); + + // Adding same requests twice will make Cache create morgue file twice, and + // then delete the previous one. + info("Stage 4: Add same request twice to make removing a morgue file."); + cache = await caches.open(name); + info("Stage 4.1: First cache.add."); + await cache.add(url); + await verifyUsage(); + + info("Stage 4.2: Second cache.add."); + await cache.add(url); + + // Since we trigger the action to delete orphaned body, we need to wait for + // the action is done. + await waitForIOToComplete(cache, url); + await verifyUsage(); + cache = await caches.open(name); + info("Stage 4.2.1: cache.delete."); + await cache.delete(url); + await waitForIOToComplete(cache, url); + let emptyUsage4 = await verifyUsage(); + + info("Stage 5: Clean caches."); + await caches.delete(name); + + info("Stage 6: Final Check."); + ok(emptyUsage1 == emptyUsage2 && + emptyUsage1 == emptyUsage3 && + emptyUsage1 == emptyUsage4, + "Empty usages should be the same"); + + await SimpleTest.finish(); +}); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_cache_worker_gc.html b/dom/cache/test/mochitest/test_cache_worker_gc.html new file mode 100644 index 0000000000..798bafaf9b --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_worker_gc.html @@ -0,0 +1,58 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Test CacheStorage does not prevent worker GC</title> + <script src="/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> + <script type="text/javascript" src="driver.js"></script> +</head> +<body> +<iframe id="frame"></iframe> +<script class="testbody" type="text/javascript"> +function startWorker() { + return new Promise(resolve => { + var w = new Worker("idle_worker.js"); + w.addEventListener("message", function onMessage(evt) { + if (evt.data === "LOADED") { + w.removeEventListener("message", onMessage); + resolve(w); + } + }); + }); +} + +function gc() { + return new Promise(resolve => { + SpecialPowers.exactGC(resolve); + }); +} + +SimpleTest.waitForExplicitFinish(); + +async function test() { + let w = await startWorker(); + let weakWorker = SpecialPowers.Cu.getWeakReference(w); + ok(weakWorker, "worker supports weak reference"); + w = null; + await gc(); + await gc(); + ok(!weakWorker.get(), "worker weak reference should be garbage collected"); + SimpleTest.finish(); +} +// Bug 1746646: Make mochitests work with TCP enabled (cookieBehavior = 5) +// Acquire storage access permission here so that the Cache API is avaialable +SpecialPowers.wrap(document).notifyUserGestureActivation(); +SpecialPowers.addPermission("storageAccessAPI", true, window.location.href).then(() => { + SpecialPowers.wrap(document).requestStorageAccess().then(() => { + SpecialPowers.pushPrefEnv({ + set: [["privacy.partition.always_partition_third_party_non_cookie_storage", false]], + }).then(() => { + test(); + }); + }); +}); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_caches.html b/dom/cache/test/mochitest/test_caches.html new file mode 100644 index 0000000000..63e43bc139 --- /dev/null +++ b/dom/cache/test/mochitest/test_caches.html @@ -0,0 +1,22 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Test the CacheStorage API</title> + <script src="/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> + <script type="text/javascript" src="driver.js"></script> +</head> +<body> +<iframe id="frame"></iframe> +<script class="testbody" type="text/javascript"> + // These tests can only be run in sequential mode because we need to be able + // to rely on the global state of the CacheStorage at all times. + runTests("test_caches.js", "sequential") + .then(function() { + SimpleTest.finish(); + }); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_caches.js b/dom/cache/test/mochitest/test_caches.js new file mode 100644 index 0000000000..c1d3abaff1 --- /dev/null +++ b/dom/cache/test/mochitest/test_caches.js @@ -0,0 +1,177 @@ +/* global context testDone:true */ + +function arraysHaveSameContent(arr1, arr2) { + if (arr1.length != arr2.length) { + return false; + } + return arr1.every(function (value, index) { + return arr2[index] === value; + }); +} + +function testHas() { + var name = "caches-has" + context; + return caches + .has(name) + .then(function (has) { + ok(!has, name + " should not exist yet"); + return caches.open(name); + }) + .then(function (c) { + return caches.has(name); + }) + .then(function (has) { + ok(has, name + " should now exist"); + return caches.delete(name); + }) + .then(function (deleted) { + ok(deleted, "The deletion should finish successfully"); + return caches.has(name); + }) + .then(function (has) { + ok(!has, name + " should not exist any more"); + }); +} + +function testKeys() { + var names = [ + // The names here are intentionally unsorted, to ensure the insertion order + // and make sure we don't confuse it with an alphabetically sorted list. + "caches-keys4" + context, + "caches-keys0" + context, + "caches-keys1" + context, + "caches-keys3" + context, + ]; + return caches + .keys() + .then(function (keys) { + is(keys.length, 0, "No keys should exist yet"); + return Promise.all( + names.map(function (name) { + return caches.open(name); + }) + ); + }) + .then(function () { + return caches.keys(); + }) + .then(function (keys) { + ok( + arraysHaveSameContent(keys, names), + "Keys must match in insertion order" + ); + return Promise.all( + names.map(function (name) { + return caches.delete(name); + }) + ); + }) + .then(function (deleted) { + ok( + arraysHaveSameContent(deleted, [true, true, true, true]), + "All deletions must succeed" + ); + return caches.keys(); + }) + .then(function (keys) { + is(keys.length, 0, "No keys should exist any more"); + }); +} + +function testMatchAcrossCaches() { + var tests = [ + // The names here are intentionally unsorted, to ensure the insertion order + // and make sure we don't confuse it with an alphabetically sorted list. + { + name: "caches-xmatch5" + context, + request: "//mochi.test:8888/?5" + context, + }, + { + name: "caches-xmatch2" + context, + request: + "//mochi.test:8888/tests/dom/cache/test/mochitest/test_caches.js?2" + + context, + }, + { + name: "caches-xmatch4" + context, + request: "//mochi.test:8888/?4" + context, + }, + ]; + return Promise.all( + tests.map(function (test) { + return caches.open(test.name).then(function (c) { + return c.add(test.request); + }); + }) + ) + .then(function () { + return caches.match("//mochi.test:8888/?5" + context, { + ignoreSearch: true, + }); + }) + .then(function (match) { + ok(match.url.indexOf("?5") > 0, "Match should come from the first cache"); + return caches.delete("caches-xmatch2" + context); // This should not change anything! + }) + .then(function (deleted) { + ok(deleted, "Deletion should finish successfully"); + return caches.match("//mochi.test:8888/?" + context, { + ignoreSearch: true, + }); + }) + .then(function (match) { + ok( + match.url.indexOf("?5") > 0, + "Match should still come from the first cache" + ); + return caches.delete("caches-xmatch5" + context); // This should eliminate the first match! + }) + .then(function (deleted) { + ok(deleted, "Deletion should finish successfully"); + return caches.match("//mochi.test:8888/?" + context, { + ignoreSearch: true, + }); + }) + .then(function (match) { + ok(match.url.indexOf("?4") > 0, "Match should come from the third cache"); + return caches.delete("caches-xmatch4" + context); // Game over! + }) + .then(function (deleted) { + ok(deleted, "Deletion should finish successfully"); + return caches.match("//mochi.test:8888/?" + context, { + ignoreSearch: true, + }); + }) + .then(function (match) { + is(typeof match, "undefined", "No matches should be found"); + }); +} + +function testDelete() { + return caches + .delete("delete" + context) + .then(function (deleted) { + ok(!deleted, "Attempting to delete a non-existing cache should fail"); + return caches.open("delete" + context); + }) + .then(function () { + return caches.delete("delete" + context); + }) + .then(function (deleted) { + ok(deleted, "Delete should now succeed"); + }); +} + +testHas() + .then(function () { + return testKeys(); + }) + .then(function () { + return testMatchAcrossCaches(); + }) + .then(function () { + return testDelete(); + }) + .then(function () { + testDone(); + }); diff --git a/dom/cache/test/mochitest/test_chrome_constructor.html b/dom/cache/test/mochitest/test_chrome_constructor.html new file mode 100644 index 0000000000..fbce2d0162 --- /dev/null +++ b/dom/cache/test/mochitest/test_chrome_constructor.html @@ -0,0 +1,44 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Validate Interfaces Exposed to Workers</title> + <script src="/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> +</head> +<body> +<script class="testbody" type="text/javascript"> + SimpleTest.waitForExplicitFinish(); + + SpecialPowers.pushPrefEnv({ + "set": [["dom.caches.enabled", true], + ["dom.caches.testing.enabled", true]], + }, function() { + // attach to a different origin's CacheStorage + var url = "https://example.com/"; + var storage = SpecialPowers.createChromeCache("content", url); + + // verify we can use the other origin's CacheStorage as normal + var req = new Request("https://example.com/index.html"); + var res = new Response("hello world"); + var cache; + storage.open("foo").then(function(c) { + cache = c; + ok(cache, "storage should create cache"); + return cache.put(req, res.clone()); + }).then(function() { + return cache.match(req); + }).then(function(foundResponse) { + return Promise.all([res.text(), foundResponse.text()]); + }).then(function(results) { + is(results[0], results[1], "cache should contain response"); + return storage.delete("foo"); + }).then(function(deleted) { + ok(deleted, "storage should delete cache"); + SimpleTest.finish(); + }); + }); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/vary.sjs b/dom/cache/test/mochitest/vary.sjs new file mode 100644 index 0000000000..6659ac852b --- /dev/null +++ b/dom/cache/test/mochitest/vary.sjs @@ -0,0 +1,9 @@ +function handleRequest(request, response) { + response.setStatusLine(request.httpVersion, 200, "OK"); + var header = "no WhatToVary header"; + if (request.hasHeader("WhatToVary")) { + header = request.getHeader("WhatToVary"); + response.setHeader("Vary", header); + } + response.write(header); +} diff --git a/dom/cache/test/mochitest/worker_driver.js b/dom/cache/test/mochitest/worker_driver.js new file mode 100644 index 0000000000..03535c9839 --- /dev/null +++ b/dom/cache/test/mochitest/worker_driver.js @@ -0,0 +1,81 @@ +// Any copyright is dedicated to the Public Domain. +// http://creativecommons.org/publicdomain/zero/1.0/ +// +// Utility script for writing worker tests. In your main document do: +// +// <script type="text/javascript" src="worker_driver.js"></script> +// <script type="text/javascript"> +// workerTestExec('myWorkerTestCase.js') +// </script> +// +// This will then spawn a worker, define some utility functions, and then +// execute the code in myWorkerTestCase.js. You can then use these +// functions in your worker-side test: +// +// ok() - like the SimpleTest assert +// is() - like the SimpleTest assert +// workerTestDone() - like SimpleTest.finish() indicating the test is complete +// +// There are also some functions for requesting information that requires +// SpecialPowers or other main-thread-only resources: +// +// workerTestGetPrefs() - request an array of prefs value from the main thread +// workerTestGetPermissions() - request an array permissions from the MT +// workerTestGetVersion() - request the current version string from the MT +// workerTestGetUserAgent() - request the user agent string from the MT +// +// For an example see test_worker_interfaces.html and test_worker_interfaces.js. + +function workerTestExec(script) { + return new Promise(function (resolve, reject) { + var worker = new Worker("worker_wrapper.js"); + worker.onmessage = function (event) { + is( + event.data.context, + "Worker", + "Correct context for messages received on the worker" + ); + if (event.data.type == "finish") { + worker.terminate(); + SpecialPowers.forceGC(); + resolve(); + } else if (event.data.type == "status") { + ok(event.data.status, event.data.context + ": " + event.data.msg); + } else if (event.data.type == "getPrefs") { + let result = {}; + event.data.prefs.forEach(function (pref) { + result[pref] = SpecialPowers.Services.prefs.getBoolPref(pref); + }); + worker.postMessage({ + type: "returnPrefs", + prefs: event.data.prefs, + result, + }); + } else if (event.data.type == "getPermissions") { + let result = {}; + event.data.permissions.forEach(function (permission) { + result[permission] = SpecialPowers.hasPermission( + permission, + window.document + ); + }); + worker.postMessage({ + type: "returnPermissions", + permissions: event.data.permissions, + result, + }); + } else if (event.data.type == "getUserAgent") { + worker.postMessage({ + type: "returnUserAgent", + result: navigator.userAgent, + }); + } + }; + + worker.onerror = function (event) { + reject("Worker had an error: " + event.data); + }; + + worker.postMessage({ script }); + }); +} diff --git a/dom/cache/test/mochitest/worker_wrapper.js b/dom/cache/test/mochitest/worker_wrapper.js new file mode 100644 index 0000000000..e18015b220 --- /dev/null +++ b/dom/cache/test/mochitest/worker_wrapper.js @@ -0,0 +1,141 @@ +/* eslint-env worker */ + +// Any copyright is dedicated to the Public Domain. +// http://creativecommons.org/publicdomain/zero/1.0/ +// +// ServiceWorker equivalent of worker_wrapper.js. + +var client; +var context; + +function ok(a, msg) { + client.postMessage({ + type: "status", + status: !!a, + msg: a + ": " + msg, + context, + }); +} + +function is(a, b, msg) { + client.postMessage({ + type: "status", + status: a === b, + msg: a + " === " + b + ": " + msg, + context, + }); +} + +function workerTestArrayEquals(a, b) { + if (!Array.isArray(a) || !Array.isArray(b) || a.length != b.length) { + return false; + } + for (var i = 0, n = a.length; i < n; ++i) { + if (a[i] !== b[i]) { + return false; + } + } + return true; +} + +function testDone() { + client.postMessage({ type: "finish", context }); +} + +function workerTestGetPrefs(prefs, cb) { + addEventListener("message", function workerTestGetPrefsCB(e) { + if ( + e.data.type != "returnPrefs" || + !workerTestArrayEquals(prefs, e.data.prefs) + ) { + return; + } + removeEventListener("message", workerTestGetPrefsCB); + cb(e.data.result); + }); + client.postMessage({ + type: "getPrefs", + context, + prefs, + }); +} + +function workerTestGetPermissions(permissions, cb) { + addEventListener("message", function workerTestGetPermissionsCB(e) { + if ( + e.data.type != "returnPermissions" || + !workerTestArrayEquals(permissions, e.data.permissions) + ) { + return; + } + removeEventListener("message", workerTestGetPermissionsCB); + cb(e.data.result); + }); + client.postMessage({ + type: "getPermissions", + context, + permissions, + }); +} + +function workerTestGetUserAgent(cb) { + addEventListener("message", function workerTestGetUserAgentCB(e) { + if (e.data.type !== "returnUserAgent") { + return; + } + removeEventListener("message", workerTestGetUserAgentCB); + cb(e.data.result); + }); + client.postMessage({ + context, + type: "getUserAgent", + }); +} + +var completeInstall = null; + +addEventListener("message", function workerWrapperOnMessage(e) { + removeEventListener("message", workerWrapperOnMessage); + var data = e.data; + function runScript() { + try { + importScripts(data.script); + } catch (err) { + client.postMessage({ + type: "status", + status: false, + context, + msg: + "worker failed to import " + data.script + "; error: " + err.message, + }); + } + } + if ("ServiceWorker" in self) { + self.clients + .matchAll({ includeUncontrolled: true }) + .then(function (clients) { + for (var i = 0; i < clients.length; ++i) { + if (clients[i].url.indexOf("message_receiver.html") > -1) { + client = clients[i]; + break; + } + } + if (!client) { + dump( + "We couldn't find the message_receiver window, the test will fail\n" + ); + } + context = "ServiceWorker"; + runScript(); + completeInstall(); + }); + } else { + client = self; + context = "Worker"; + runScript(); + } +}); + +addEventListener("install", e => { + e.waitUntil(new Promise(resolve => (completeInstall = resolve))); +}); diff --git a/dom/cache/test/xpcshell/bug1425146_profile.zip b/dom/cache/test/xpcshell/bug1425146_profile.zip Binary files differnew file mode 100644 index 0000000000..fc57bc1043 --- /dev/null +++ b/dom/cache/test/xpcshell/bug1425146_profile.zip diff --git a/dom/cache/test/xpcshell/head.js b/dom/cache/test/xpcshell/head.js new file mode 100644 index 0000000000..58717f0f68 --- /dev/null +++ b/dom/cache/test/xpcshell/head.js @@ -0,0 +1,208 @@ +/** + * Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ + * + * All images in schema_15_profile.zip are from https://github.com/mdn/sw-test/ + * and are CC licensed by https://www.flickr.com/photos/legofenris/. + */ + +// testSteps is expected to be defined by the file including this file. +/* global testSteps */ + +const NS_APP_USER_PROFILE_50_DIR = "ProfD"; +const osWindowsName = "WINNT"; +const pathDelimiter = "/"; + +const persistentPersistence = "persistent"; +const defaultPersistence = "default"; + +const storageDirName = "storage"; +const persistentPersistenceDirName = "permanent"; +const defaultPersistenceDirName = "default"; + +function cacheClientDirName() { + return "cache"; +} + +// services required be initialized in order to run CacheStorage +var ss = Cc["@mozilla.org/storage/service;1"].createInstance( + Ci.mozIStorageService +); +var sts = Cc["@mozilla.org/network/stream-transport-service;1"].getService( + Ci.nsIStreamTransportService +); +var hash = Cc["@mozilla.org/security/hash;1"].createInstance(Ci.nsICryptoHash); + +class RequestError extends Error { + constructor(resultCode, resultName) { + super(`Request failed (code: ${resultCode}, name: ${resultName})`); + this.name = "RequestError"; + this.resultCode = resultCode; + this.resultName = resultName; + } +} + +add_setup(function () { + do_get_profile(); + + enableTesting(); + + // Expose Cache and Fetch symbols on the global + Cu.importGlobalProperties(["caches", "fetch"]); + + registerCleanupFunction(resetTesting); +}); + +function enableTesting() { + Services.prefs.setBoolPref("dom.simpleDB.enabled", true); + Services.prefs.setBoolPref("dom.quotaManager.testing", true); +} + +function resetTesting() { + Services.prefs.clearUserPref("dom.quotaManager.testing"); + Services.prefs.clearUserPref("dom.simpleDB.enabled"); +} + +function initStorage() { + return Services.qms.init(); +} + +function initTemporaryStorage() { + return Services.qms.initTemporaryStorage(); +} + +function initPersistentOrigin(principal) { + return Services.qms.initializePersistentOrigin(principal); +} + +function initTemporaryOrigin(principal) { + return Services.qms.initializeTemporaryOrigin("default", principal); +} + +function clearOrigin(principal, persistence) { + let request = Services.qms.clearStoragesForPrincipal(principal, persistence); + + return request; +} + +function reset() { + return Services.qms.reset(); +} + +async function requestFinished(request) { + await new Promise(function (resolve) { + request.callback = function () { + resolve(); + }; + }); + + if (request.resultCode !== Cr.NS_OK) { + throw new RequestError(request.resultCode, request.resultName); + } + + return request.result; +} + +// Extract a zip file into the profile +function create_test_profile(zipFileName) { + var directoryService = Services.dirsvc; + + var profileDir = directoryService.get(NS_APP_USER_PROFILE_50_DIR, Ci.nsIFile); + var currentDir = directoryService.get("CurWorkD", Ci.nsIFile); + + var packageFile = currentDir.clone(); + packageFile.append(zipFileName); + + var zipReader = Cc["@mozilla.org/libjar/zip-reader;1"].createInstance( + Ci.nsIZipReader + ); + zipReader.open(packageFile); + + var entryNames = Array.from(zipReader.findEntries(null)); + entryNames.sort(); + + for (var entryName of entryNames) { + var zipentry = zipReader.getEntry(entryName); + + var file = profileDir.clone(); + entryName.split(pathDelimiter).forEach(function (part) { + file.append(part); + }); + + if (zipentry.isDirectory) { + file.create(Ci.nsIFile.DIRECTORY_TYPE, parseInt("0755", 8)); + } else { + var istream = zipReader.getInputStream(entryName); + + var ostream = Cc[ + "@mozilla.org/network/file-output-stream;1" + ].createInstance(Ci.nsIFileOutputStream); + ostream.init(file, -1, parseInt("0644", 8), 0); + + var bostream = Cc[ + "@mozilla.org/network/buffered-output-stream;1" + ].createInstance(Ci.nsIBufferedOutputStream); + bostream.init(ostream, 32 * 1024); + + bostream.writeFrom(istream, istream.available()); + + istream.close(); + bostream.close(); + } + } + + zipReader.close(); +} + +function getCacheDir() { + return getRelativeFile( + `${storageDirName}/${defaultPersistenceDirName}/chrome/${cacheClientDirName()}` + ); +} + +function getPrincipal(url, attrs) { + let uri = Services.io.newURI(url); + if (!attrs) { + attrs = {}; + } + return Services.scriptSecurityManager.createContentPrincipal(uri, attrs); +} + +function getDefaultPrincipal() { + return getPrincipal("http://example.com"); +} + +function getRelativeFile(relativePath) { + let file = Services.dirsvc + .get(NS_APP_USER_PROFILE_50_DIR, Ci.nsIFile) + .clone(); + + if (Services.appinfo.OS === osWindowsName) { + let winFile = file.QueryInterface(Ci.nsILocalFileWin); + winFile.useDOSDevicePathSyntax = true; + } + + relativePath.split(pathDelimiter).forEach(function (component) { + if (component == "..") { + file = file.parent; + } else { + file.append(component); + } + }); + + return file; +} + +function getSimpleDatabase(principal, persistence) { + let connection = Cc["@mozilla.org/dom/sdb-connection;1"].createInstance( + Ci.nsISDBConnection + ); + + if (!principal) { + principal = getDefaultPrincipal(); + } + + connection.init(principal, persistence); + + return connection; +} diff --git a/dom/cache/test/xpcshell/make_profile.js b/dom/cache/test/xpcshell/make_profile.js new file mode 100644 index 0000000000..b7b1a9042b --- /dev/null +++ b/dom/cache/test/xpcshell/make_profile.js @@ -0,0 +1,137 @@ +/** + * Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ + * + * All images in schema_15_profile.zip are from https://github.com/mdn/sw-test/ + * and are CC licensed by https://www.flickr.com/photos/legofenris/. + */ + +// Enumerate the directory tree and store results in entryList as +// +// { path: 'a/b/c', file: <nsIFile> } +// +// The algorithm starts with the first entry already in entryList. +function enumerate_tree(entryList) { + for (var index = 0; index < entryList.length; ++index) { + var path = entryList[index].path; + var file = entryList[index].file; + + if (file.isDirectory()) { + var dirList = file.directoryEntries; + while (dirList.hasMoreElements()) { + var dirFile = dirList.nextFile; + entryList.push({ path: path + "/" + dirFile.leafName, file: dirFile }); + } + } + } +} + +function zip_profile(zipFile, profileDir) { + var zipWriter = Cc["@mozilla.org/zipwriter;1"].createInstance( + Ci.nsIZipWriter + ); + zipWriter.open(zipFile, 0x04 | 0x08 | 0x20); + + var root = profileDir.clone(); + root.append("storage"); + root.append("default"); + root.append("chrome"); + + var entryList = [{ path: "storage/default/chrome", file: root }]; + enumerate_tree(entryList); + + entryList.forEach(function (entry) { + if (entry.file.isDirectory()) { + zipWriter.addEntryDirectory( + entry.path, + entry.file.lastModifiedTime, + false + ); + } else { + var istream = Cc[ + "@mozilla.org/network/file-input-stream;1" + ].createInstance(Ci.nsIFileInputStream); + istream.init(entry.file, -1, -1, 0); + zipWriter.addEntryStream( + entry.path, + entry.file.lastModifiedTime, + Ci.nsIZipWriter.COMPRESSION_DEFAULT, + istream, + false + ); + istream.close(); + } + }); + + zipWriter.close(); +} + +function exactGC() { + return new Promise(function (resolve) { + var count = 0; + function doPreciseGCandCC() { + function scheduleGCCallback() { + Cu.forceCC(); + + if (++count < 2) { + doPreciseGCandCC(); + } else { + resolve(); + } + } + Cu.schedulePreciseGC(scheduleGCCallback); + } + doPreciseGCandCC(); + }); +} + +function resetQuotaManager() { + return new Promise(function (resolve) { + var prefService = Services.prefs; + + // enable quota manager testing mode + var pref = "dom.quotaManager.testing"; + prefService.getBranch(null).setBoolPref(pref, true); + + var request = Services.qms.reset(); + request.callback = resolve; + + // disable quota manager testing mode + // prefService.getBranch(null).setBoolPref(pref, false); + }); +} + +function run_test() { + do_test_pending(); + do_get_profile(); + + var directoryService = Services.dirsvc; + var profileDir = directoryService.get("ProfD", Ci.nsIFile); + var currentDir = directoryService.get("CurWorkD", Ci.nsIFile); + + var zipFile = currentDir.clone(); + zipFile.append("new_profile.zip"); + if (zipFile.exists()) { + zipFile.remove(false); + } + ok(!zipFile.exists()); + + caches + .open("xpcshell-test") + .then(function (c) { + var request = new Request("http://example.com/index.html"); + var response = new Response("hello world"); + return c.put(request, response); + }) + .then(exactGC) + .then(resetQuotaManager) + .then(function () { + zip_profile(zipFile, profileDir); + dump("### ### created zip at: " + zipFile.path + "\n"); + do_test_finished(); + }) + .catch(function (e) { + do_test_finished(); + ok(false, e); + }); +} diff --git a/dom/cache/test/xpcshell/schema_15_profile.zip b/dom/cache/test/xpcshell/schema_15_profile.zip Binary files differnew file mode 100644 index 0000000000..32cc8f2eeb --- /dev/null +++ b/dom/cache/test/xpcshell/schema_15_profile.zip diff --git a/dom/cache/test/xpcshell/schema_25_profile.zip b/dom/cache/test/xpcshell/schema_25_profile.zip Binary files differnew file mode 100644 index 0000000000..626aa8f625 --- /dev/null +++ b/dom/cache/test/xpcshell/schema_25_profile.zip diff --git a/dom/cache/test/xpcshell/test_bug1425146.js b/dom/cache/test/xpcshell/test_bug1425146.js new file mode 100644 index 0000000000..d678bb3745 --- /dev/null +++ b/dom/cache/test/xpcshell/test_bug1425146.js @@ -0,0 +1,51 @@ +/** + * Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ + */ + +// This test is mainly to verify that we are able to recover from a situation +// in which the padding file and the padding column are missing during origin +// initialization. This was originally reported in bug 1425146 comment 39. +// The situation can be described as follows: +// 1. A profile is used in FF57. The storage version is 2.1. There's no cache +// storage for http://www.mozilla.org +// 2. The same profile is used in FF56. The storage version is still 2.1 which +// doesn't prevent storage system from working since minor upgrades are +// backwards-compatible. The cache storage for http://www.mozilla.org is +// created with schema version 25 (without any padding stuff). +// 3. The profile is used in FF57 again. Zero padding files for existing cache +// storages are not created because storage is already at version 2.1. +// Storage is being initialized and a missing padding file triggers padding +// size computation from the cache database with schema version 25. Since +// the computation happens before any real DOM cache operation, the database +// is not upgraded to schema version 26, so the padding column is missing. + +add_task(async function testSteps() { + // The profile contains one cache storage, a script for cache creation and + // the storage database: + // - storage/default/http+++www.mozilla.org/cache + // - create_cache.js + // - storage.sqlite + // The file create_cache.js in the package was run locally, specifically it + // was temporarily added to xpcshell.ini and then executed: + // mach xpcshell-test --interactive dom/cache/test/xpcshell/create_cache.js + // Note: it must be executed in FF56 and it only creates the directory + // "storage/default/chrome/cache" and the file "storage.sqlite". To make it + // become the profile in the test, additional manual steps are needed. + // 1. Create "http+++www.mozilla.org" folder under the ""storage/default". + // 2. Copy the "cache" folder under the "storage/default/chrome" to + // "storage/default/http+++www.mozilla.org". + // 3. Remove the folder "storage/default/chrome" + // 4. Remove the folder "storage/temporary". + // 5. Add "create_cache.js". + // 6. Replace the "storage.sqlite" created by FF56 (storage v2.0) with the + // "storage.sqlite" created by FF57 (storage v2.1) + create_test_profile("bug1425146_profile.zip"); + + try { + await caches.open("test"); + ok(true, "Should not have thrown"); + } catch (ex) { + ok(false, "Should not have thrown"); + } +}); diff --git a/dom/cache/test/xpcshell/test_empty_directories.js b/dom/cache/test/xpcshell/test_empty_directories.js new file mode 100644 index 0000000000..2e43ba67bd --- /dev/null +++ b/dom/cache/test/xpcshell/test_empty_directories.js @@ -0,0 +1,85 @@ +/** + * This test is mainly to verify cache won't leave emptry directory. + */ + +function resetStorage() { + return new Promise(function (resolve, reject) { + var qms = Services.qms; + var request = qms.reset(); + request.callback = resolve; + }); +} + +async function setUpEnv() { + Services.prefs.setBoolPref("dom.quotaManager.testing", true); + + // We need this for generating the basic profile path + create_test_profile("schema_25_profile.zip"); + + // Trigger storage upgrade + await caches.open("test"); + const cacheDir = getCacheDir(); + let morgueDir = cacheDir.clone(); + morgueDir.append("morgue"); + + // clean the cache directoy + for (let dir of morgueDir.directoryEntries) { + for (let file of dir.directoryEntries) { + file.remove(false); + } + } + + await resetStorage(); +} + +// This function ensure the directory with file shouldn't have been deleted and +// ensure the directory without file should've been deleted. +function verifyResult() { + const cacheDir = getCacheDir(); + let morgueDir = cacheDir.clone(); + morgueDir.append("morgue"); + + let foundEmpty = false; + for (let dir of morgueDir.directoryEntries) { + let empty = true; + // eslint-disable-next-line no-unused-vars + for (let file of dir.directoryEntries) { + empty = false; + } + + foundEmpty = foundEmpty || empty; + } + return !foundEmpty; +} + +add_task(async function testSteps() { + const url = "https://www.mozilla.org"; + + info("Setting up environment"); + + await setUpEnv(); + + info("Test 0 - InitOrigin shouldn't leave an empty directoy"); + + let cache = await caches.open("test"); + let response = await cache.match(url); + ok(!!response, "Upgrade from 25 to 26 do succeed"); + ok(verifyResult(), "InitOrigin should clean all empty directories"); + + info("Test 1 - DeleteOrphanedBodyFiles shouldn't leave an empty directoy"); + + await cache.put(url, response.clone()); + // eslint-disable-next-line no-unused-vars + let r = await cache.match(url); + await cache.delete(url); + await resetStorage(); + + cache = await caches.open("test"); + + // Extra operation to ensure the deletion is completed + await cache.match(url); + + ok(verifyResult(), "Empty directory should be removed"); + + await caches.delete("test"); +}); diff --git a/dom/cache/test/xpcshell/test_migration.js b/dom/cache/test/xpcshell/test_migration.js new file mode 100644 index 0000000000..5a6637eecc --- /dev/null +++ b/dom/cache/test/xpcshell/test_migration.js @@ -0,0 +1,65 @@ +/** + * Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ + * + * All images in schema_15_profile.zip are from https://github.com/mdn/sw-test/ + * and are CC licensed by https://www.flickr.com/photos/legofenris/. + */ + +add_task(async function testSteps() { + create_test_profile("schema_15_profile.zip"); + + const cache = await caches.open("xpcshell-test"); + ok(cache, "cache exists"); + + const requestList = await cache.keys(); + + ok(requestList.length, "should have at least one request in cache"); + for (const request of requestList) { + ok(request, "each request in list should be non-null"); + ok( + request.redirect === "follow", + 'request.redirect should default to "follow"' + ); + ok( + request.cache === "default", + 'request.cache should have been updated to "default"' + request.cache + ); + ok( + request.mode === "navigate", + 'request.mode should have been updated to "navigate"' + ); + ok( + request.referrerPolicy === "no-referrer-when-downgrade", + 'request.referrerPolicy should have been updated to "no-referrer-when-downgrade"' + ); + } + + const responseList = await Promise.all( + requestList.map(function (request) { + return cache.match(request); + }) + ); + + ok(responseList.length, "should have at least one response in cache"); + for (const response of responseList) { + ok(response, "each response should be non-null"); + // reponse.url is a empty string in current test file. It should test for + // not being a empty string once thet test file is updated. + ok( + typeof response.url === "string", + "each response.url should be a string" + ); + // reponse.redirected may be changed once test file is updated. It should + // be false since current reponse.url is a empty string. + ok( + response.redirected === false, + "each response.redirected should be false" + ); + Assert.equal( + response.headers.get("Content-Type"), + "text/plain;charset=UTF-8", + "the response should have the correct header" + ); + } +}); diff --git a/dom/cache/test/xpcshell/test_originInit.js b/dom/cache/test/xpcshell/test_originInit.js new file mode 100644 index 0000000000..395cba8a0f --- /dev/null +++ b/dom/cache/test/xpcshell/test_originInit.js @@ -0,0 +1,249 @@ +/** + * Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ + */ + +add_task(async function testSteps() { + // ToDo: Replace storage and default with a getter function once we expose the + // filenames of them to a IDL file. + const basePath = `${storageDirName}/${defaultPersistenceDirName}/`; + const principal = getPrincipal("https://example.com"); + const originDirName = "https+++example.com"; + + // ToDo: Replace caches.sqlite with a getter function once we expose the + // filename of it to a IDL file. + const cachesDatabase = getRelativeFile( + `${basePath}/${originDirName}/${cacheClientDirName()}/caches.sqlite` + ); + // ToDo: Replace .padding with a getter function once we expose the filename + // of it to a IDL file. + const paddingFile = getRelativeFile( + `${basePath}/${originDirName}/${cacheClientDirName()}/.padding` + ); + // ToDo: Replace .padding-tmp with a getter function once we expose the + // filename of it to a IDL file. + const paddingTempFile = getRelativeFile( + `${basePath}/${originDirName}/${cacheClientDirName()}/.padding-tmp` + ); + // ToDo: Replace morgue with a getter function once we expose the + // filename of it to a IDL file. + const morgueDir = getRelativeFile( + `${basePath}/${originDirName}/${cacheClientDirName()}/morgue` + ); + + const persistentCacheDir = getRelativeFile( + `${storageDirName}/${persistentPersistenceDirName}/${originDirName}/` + + `${cacheClientDirName()}` + ); + + async function createNormalCacheOrigin() { + async function sandboxScript() { + const cache = await caches.open("myCache"); + const request = new Request("https://example.com/index.html"); + const response = new Response("hello world"); + await cache.put(request, response); + } + + const sandbox = new Cu.Sandbox(principal, { + wantGlobalProperties: ["caches", "fetch"], + }); + + const promise = new Promise(function (resolve, reject) { + sandbox.resolve = resolve; + sandbox.reject = reject; + }); + + Cu.evalInSandbox( + sandboxScript.toSource() + " sandboxScript().then(resolve, reject);", + sandbox + ); + await promise; + + let request = reset(); + await requestFinished(request); + } + + async function createPersistentTestOrigin() { + let database = getSimpleDatabase(principal, "persistent"); + + let request = database.open("data"); + await requestFinished(request); + + request = reset(); + await requestFinished(request); + } + + function removeFile(file) { + file.remove(false); + } + + function removeDir(dir) { + dir.remove(true); + } + + function createEmptyFile(file) { + file.create(Ci.nsIFile.NORMAL_FILE_TYPE, 0o644); + } + + function createEmptyDirectory(dir) { + dir.create(Ci.nsIFile.DIRECTORY_TYPE, 0o0755); + } + + async function initTestOrigin() { + let request = initStorage(); + await requestFinished(request); + + request = initTemporaryStorage(); + await requestFinished(request); + + request = initTemporaryOrigin(principal); + await requestFinished(request); + } + + async function initPersistentTestOrigin() { + let request = initStorage(); + await requestFinished(request); + + request = initPersistentOrigin(principal); + await requestFinished(request); + } + + function checkFiles( + expectCachesDatabase, + expectPaddingFile, + expectTempPaddingFile, + expectMorgueDir + ) { + let exists = cachesDatabase.exists(); + if (expectCachesDatabase) { + ok(exists, "caches.sqlite does exist"); + } else { + ok(!exists, "caches.sqlite doesn't exist"); + } + + exists = paddingFile.exists(); + if (expectPaddingFile) { + ok(exists, ".padding does exist"); + } else { + ok(!exists, ".padding doesn't exist"); + } + + exists = paddingTempFile.exists(); + if (expectTempPaddingFile) { + ok(exists, ".padding-tmp does exist"); + } else { + ok(!exists, ".padding-tmp doesn't exist"); + } + + exists = morgueDir.exists(); + if (expectMorgueDir) { + ok(exists, "morgue does exist"); + } else { + ok(!exists, "moruge doesn't exist"); + } + } + + async function clearTestOrigin() { + let request = clearOrigin(principal, defaultPersistence); + await requestFinished(request); + } + + async function clearPersistentTestOrigin() { + let request = clearOrigin(principal, persistentPersistence); + await requestFinished(request); + } + + async function testOriginInit( + createCachesDatabase, + createPaddingFile, + createTempPaddingFile, + createMorgueDir + ) { + info( + `Testing initialization of cache directory when caches.sqlite ` + + `${createCachesDatabase ? "exists" : "doesn't exist"}, .padding ` + + `${createPaddingFile ? "exists" : "doesn't exist"}, .padding-tmp ` + + `${createTempPaddingFile ? "exists" : "doesn't exist"}, morgue ` + + `${createMorgueDir ? "exists" : "doesn't exist"}` + ); + + await createNormalCacheOrigin(); + + checkFiles(true, true, false, true); + + if (!createCachesDatabase) { + removeFile(cachesDatabase); + } + + if (!createPaddingFile) { + removeFile(paddingFile); + } + + if (createTempPaddingFile) { + createEmptyFile(paddingTempFile); + } + + if (!createMorgueDir) { + removeDir(morgueDir); + } + + await initTestOrigin(); + + checkFiles( + createCachesDatabase, + // After the origin is initialized, ".padding" should only exist when + // "caches.sqlite" exists. + createCachesDatabase, + // After the origin is initialized, ".padding-tmp" should have always been + // removed. + false, + createCachesDatabase && createMorgueDir + ); + + await clearTestOrigin(); + } + + // Test all possible combinations. + for (let createCachesDatabase of [false, true]) { + for (let createPaddingFile of [false, true]) { + for (let createTempPaddingFile of [false, true]) { + for (let createMorgueDir of [false, true]) { + await testOriginInit( + createCachesDatabase, + createPaddingFile, + createTempPaddingFile, + createMorgueDir + ); + } + } + } + } + + // Verify that InitializeOrigin doesn't fail when a + // storage/permanent/${origin}/cache exists. + async function testPermanentCacheDir() { + info( + "Testing initialization of cache directory placed in permanent origin " + + "directory" + ); + + await createPersistentTestOrigin(); + + createEmptyDirectory(persistentCacheDir); + + try { + await initPersistentTestOrigin(); + + ok(true, "Should not have thrown"); + } catch (ex) { + ok(false, "Should not have thrown"); + } + + let exists = persistentCacheDir.exists(); + ok(exists, "cache directory in permanent origin directory does exist"); + + await clearPersistentTestOrigin(); + } + + await testPermanentCacheDir(); +}); diff --git a/dom/cache/test/xpcshell/test_padding_error_handle.js b/dom/cache/test/xpcshell/test_padding_error_handle.js new file mode 100644 index 0000000000..1bb1e1804f --- /dev/null +++ b/dom/cache/test/xpcshell/test_padding_error_handle.js @@ -0,0 +1,72 @@ +/** + * This test is mainly to verify cache actions work as usual even there exists + * an unexpected padding file. + */ + +function getTempPaddingFilePath() { + let cacheDir = getCacheDir(); + let temporaryPaddingFile = cacheDir.clone(); + temporaryPaddingFile.append(".padding-tmp"); + return temporaryPaddingFile; +} + +function createTempPaddingFile() { + let temporaryPaddingFile = getTempPaddingFilePath(); + temporaryPaddingFile.create(Ci.nsIFile.NORMAL_FILE_TYPE, parseInt("0644", 8)); + + ok( + temporaryPaddingFile.exists(), + "Temporary padding file does be created by test" + ); +} + +add_task(async function testSteps() { + create_test_profile("schema_25_profile.zip"); + let cache = await caches.open("test"); + + // Step 1: Verify cache.match won't fail when there is a temporary padding + // file + createTempPaddingFile(); + + let response = await cache.match("https://www.mozilla.org"); + ok(!!response, "Upgrade from 25 to 26 do succeed"); + + // Note: Only cache write actions(e.g. cache.put/add/addAll/delete) will + // remove unexpected temporary padding file when writting an opaque response + // into the file-system. Cache read actions(e.g. cache.keys/match) won't. + let temporaryPaddingFile = getTempPaddingFilePath(); + ok( + temporaryPaddingFile.exists(), + "Temporary padding file doesn't be removed by cache.match" + ); + + // Step 2: Verify cache.put won't fail when there is a temporary padding + // file + await cache.put("https://foo.com", response); + ok( + !temporaryPaddingFile.exists(), + "Temporary padding file does be removed by cache.put" + ); + + // Step 3: Verify cache.keys won't fail when there is a temporary padding + // file + createTempPaddingFile(); + + let cacheEntries = await cache.keys("https://foo.com"); + ok(cacheEntries.length === 1, "Cache.put does succeed"); + + ok( + temporaryPaddingFile.exists(), + "Temporary padding file doesn't be removed by cache.keys" + ); + + // Step 4: Verify cache.delete won't fail when there is a temporary padding + // file + await cache.delete("https://foo.com"); + ok( + !temporaryPaddingFile.exists(), + "Temporary padding file does be removed by cache.delete" + ); + + await caches.delete("test"); +}); diff --git a/dom/cache/test/xpcshell/test_schema_26_upgrade.js b/dom/cache/test/xpcshell/test_schema_26_upgrade.js new file mode 100644 index 0000000000..3bdb789e17 --- /dev/null +++ b/dom/cache/test/xpcshell/test_schema_26_upgrade.js @@ -0,0 +1,18 @@ +/** + * The schema_25_profile.zip are made from local Nightly by following step + * 1. Go to any website + * 2. Open web console and type + * caches.open("test") + * .then(c => fetch("https://www.mozilla.org", {mode:"no-cors"}) + * .then(r => c.put("https://www.mozilla.org", r))); + * 3. Go to profile directory and rename the website folder to "chrome" + */ + +add_task(async function testSteps() { + create_test_profile("schema_25_profile.zip"); + + let cache = await caches.open("test"); + let response = await cache.match("https://www.mozilla.org"); + ok(!!response, "Upgrade from 25 to 26 do succeed"); + ok(response.type === "opaque", "The response type does be opaque"); +}); diff --git a/dom/cache/test/xpcshell/xpcshell.ini b/dom/cache/test/xpcshell/xpcshell.ini new file mode 100644 index 0000000000..b1640ce42d --- /dev/null +++ b/dom/cache/test/xpcshell/xpcshell.ini @@ -0,0 +1,21 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +[DEFAULT] +head = head.js +support-files = + bug1425146_profile.zip + schema_15_profile.zip + schema_25_profile.zip + +# dummy test entry to generate profile zip files +[make_profile.js] + skip-if = true + +[test_bug1425146.js] +[test_empty_directories.js] +[test_migration.js] +[test_originInit.js] +[test_padding_error_handle.js] +[test_schema_26_upgrade.js] |